From 7d1208b64232bcbd70fa09c3a8fb6ceab516a437 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 02:36:19 +0200 Subject: [PATCH 001/160] feat(sacc_io): SACC read/write for the standard data-product layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add sp_validation.sacc_io: the writer/reader layer for the two-file SACC layout that becomes the package's standard data-product format. {version}.sacc analysis vector — NZ tracers, coarse xi+/-, pseudo-Cl (EE/BB/EB) with a shared BandpowerWindow, COSEBIs, pure E/B, rho/tau PSF diagnostics; one FullCovariance assembled block-diagonally (zero cross-blocks). {version}_xi_fine COSEBIs/pure-EB integration input — same NZ tracers, fine-grid xi+/-, DiagonalCovariance from TreeCorr varxip/varxim. Covariance order is point-insertion order (SACC preserves it bitwise through FITS). Writers insert in the canonical order — xi+ then xi-, Cl (ee, bb, eb), COSEBIs (all En then all Bn), pure E/B in _EB_KEYS order (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb, matching b_modes.calculate_eb_statistics), rho, then tau — but readers never assume global order: every getter resolves indices through s.indices(dtype, tracers, **tags). assemble_covariance validates that blocks are contiguous, ascending and tile the data vector exactly, failing loud otherwise. Custom data types (pure E/B, rho, tau) all parse under sacc.parse_data_type_name. Tag filters are plain kwargs; the tags={...} form silently selects nothing and is never used. Test suite (test_sacc_io.py, all synthetic and fast): per-writer round-trips (arrays/tags/windows/NZ bitwise), covariance block alignment and zero cross-blocks, assemble_covariance failure modes, DiagonalCovariance round-trip, extract() sub-covariance alignment, a tomographic multi-pair case, reader/writer mirroring on a mixed file, and the end-to-end two-file layout. 20 passed. Co-Authored-By: Claude Opus --- src/sp_validation/sacc_io.py | 537 ++++++++++++++++++++++++ src/sp_validation/tests/test_sacc_io.py | 491 ++++++++++++++++++++++ 2 files changed, 1028 insertions(+) create mode 100644 src/sp_validation/sacc_io.py create mode 100644 src/sp_validation/tests/test_sacc_io.py diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py new file mode 100644 index 00000000..cf03f12c --- /dev/null +++ b/src/sp_validation/sacc_io.py @@ -0,0 +1,537 @@ +"""SACC_IO. + +:Name: sacc_io.py + +:Description: Read/write the standard SACC data-product layout for the + weak-lensing validation package. Two files describe each + catalogue version: + + - ``{version}.sacc`` — the analysis vector: NZ tracers, coarse + ξ±, pseudo-Cℓ (EE/BB/EB) with bandpower windows, COSEBIs, + pure E/B, and ρ/τ PSF diagnostics, all sharing a single + ``FullCovariance`` assembled block-diagonally from the + per-statistic covariances (zero cross-blocks). + - ``{version}_xi_fine.sacc`` — the COSEBIs / pure-EB integration + input: the same NZ tracers, a fine-grid ξ±, and a + ``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim``. + + The covariance order is the point-insertion order (SACC preserves + it bitwise through FITS save/load). Writers below insert in the + canonical order — ξ+ then ξ−, Cℓ (ee, bb, eb), COSEBIs (all Eₙ + then all Bₙ), pure E/B (xip_E, xim_E, xip_B, xim_B, xip_amb, + xim_amb — matching ``b_modes._EB_KEYS``), ρ, then τ — but readers + never assume global order: they resolve indices through + ``Sacc.indices(dtype, tracers, **tags)``. + + Tag filters are plain keyword arguments to ``indices`` / + ``get_data_points`` / ``get_tag``; the ``tags={...}`` form + silently selects nothing and must never be used. +""" + +import numpy as np +import sacc + +PSF_TRACER = "psf_stars" + +# Standard SACC data-type strings. +XI_PLUS = "galaxy_shear_xi_plus" +XI_MINUS = "galaxy_shear_xi_minus" +CL_EE = "galaxy_shear_cl_ee" +CL_BB = "galaxy_shear_cl_bb" +CL_EB = "galaxy_shear_cl_eb" +COSEBI_EE = "galaxy_shear_cosebi_ee" +COSEBI_BB = "galaxy_shear_cosebi_bb" + +# Custom data-type strings (all parse under sacc.parse_data_type_name). +PURE_TYPES = { + "xip_E": "galaxy_shear_xiPureE_plus", + "xim_E": "galaxy_shear_xiPureE_minus", + "xip_B": "galaxy_shear_xiPureB_plus", + "xim_B": "galaxy_shear_xiPureB_minus", + "xip_amb": "galaxy_shear_xiPureAmb_plus", + "xim_amb": "galaxy_shear_xiPureAmb_minus", +} +# Insertion order of the six pure-EB blocks — matches b_modes._EB_KEYS, whose +# order is the [xip_E; xim_E; xip_B; xim_B; xip_amb; xim_amb] layout of the +# treecorr/MC pure-EB covariance (b_modes.calculate_eb_statistics, ~L392). +PURE_KEYS = ("xip_E", "xim_E", "xip_B", "xim_B", "xip_amb", "xim_amb") + +RHO_PLUS = "psf_rho{k}_xi_plus" +RHO_MINUS = "psf_rho{k}_xi_minus" +TAU_PLUS = "galaxyPsf_tau{k}_xi_plus" +TAU_MINUS = "galaxyPsf_tau{k}_xi_minus" + + +def source_name(i): + """SACC tracer name for source redshift bin ``i`` (0-based).""" + return f"source_{i}" + + +def new_sacc(nz, metadata=None): + """Create a Sacc with the survey's NZ (and PSF) tracers. + + Parameters + ---------- + nz : dict or sequence + Redshift distributions, one per source bin. Either a mapping + ``{i: (z, nz)}`` keyed by 0-based bin index, or a sequence of + ``(z, nz)`` array pairs (bin index = position). Tracers are named + ``source_{i}``. + metadata : dict, optional + Key/value pairs stored on ``s.metadata``. + + Returns + ------- + sacc.Sacc + Sacc holding the ``source_{i}`` NZ tracers and the ``psf_stars`` + Misc tracer (needed by ρ/τ diagnostics). + """ + items = nz.items() if isinstance(nz, dict) else enumerate(nz) + s = sacc.Sacc() + for i, (z, nz_i) in items: + s.add_tracer("NZ", source_name(i), np.asarray(z), np.asarray(nz_i)) + s.add_tracer("Misc", PSF_TRACER) + for key, value in (metadata or {}).items(): + s.metadata[key] = value + return s + + +def _pair(bins): + """Resolve a ``(i, j)`` bin pair to the ``(source_i, source_j)`` names.""" + i, j = bins + return (source_name(i), source_name(j)) + + +def add_xi( + s, + bins, + theta, + xip, + xim, + *, + grid, + theta_nom=None, + npairs=None, + weight=None, +): + """Add a real-space shear 2PCF (ξ+ then ξ−) for one tracer pair. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + theta : array_like + Angular separations (arcmin) — TreeCorr ``meanr``. + xip, xim : array_like + ξ+ and ξ− at ``theta``. + grid : {'coarse', 'fine'} + Distinguishes the analysis grid from the fine integration grid; + stored as the ``grid`` tag on every point. + theta_nom : array_like, optional + Nominal bin centres — TreeCorr ``rnom`` — stored as ``theta_nom``. + npairs, weight : array_like, optional + TreeCorr pair counts and weights, stored per point. + """ + tracers = _pair(bins) + for dtype, xi in ((XI_PLUS, xip), (XI_MINUS, xim)): + for n, th in enumerate(theta): + tags = {"theta": float(th), "grid": grid} + if theta_nom is not None: + tags["theta_nom"] = float(theta_nom[n]) + if npairs is not None: + tags["npairs"] = float(npairs[n]) + if weight is not None: + tags["weight"] = float(weight[n]) + s.add_data_point(dtype, tracers, float(xi[n]), **tags) + + +def add_pseudo_cl( + s, + bins, + ell_eff, + cl_ee, + cl_bb, + cl_eb, + *, + window_ells, + window_weights, +): + """Add pseudo-Cℓ (EE, BB, EB) with a shared bandpower window. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + ell_eff : array_like + Effective multipole of each bandpower. + cl_ee, cl_bb, cl_eb : array_like + EE, BB and EB bandpowers at ``ell_eff``. + window_ells : array_like + Multipoles spanned by the bandpower window matrix (shape ``(nell,)``). + window_weights : array_like + Window matrix ``W`` of shape ``(nell, nbp)`` — one column per + bandpower — from NaMaster ``get_bandpower_windows``. One + ``sacc.BandpowerWindow`` is built and shared across EE/BB/EB. + """ + tracers = _pair(bins) + window = sacc.BandpowerWindow(np.asarray(window_ells), np.asarray(window_weights)) + for dtype, cl in ((CL_EE, cl_ee), (CL_BB, cl_bb), (CL_EB, cl_eb)): + s.add_ell_cl( + dtype, *tracers, np.asarray(ell_eff), np.asarray(cl), window=window + ) + + +def add_cosebis(s, bins, En, Bn, scale_cut): + """Add COSEBIs (all Eₙ then all Bₙ) for one scale cut. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + En, Bn : array_like + E- and B-mode COSEBI amplitudes, one per logarithmic mode ``n`` + (1-based). The ``[En; Bn]`` layout matches the COSEBI covariance. + scale_cut : tuple of float + ``(theta_min, theta_max)`` in arcmin, stored on every point as the + ``theta_min``/``theta_max`` tags; multiple cuts coexist in one file, + told apart by these tags. + """ + tracers = _pair(bins) + theta_min, theta_max = scale_cut + for dtype, modes in ((COSEBI_EE, En), (COSEBI_BB, Bn)): + for n, value in enumerate(modes, start=1): + s.add_data_point( + dtype, + tracers, + float(value), + n=n, + theta_min=float(theta_min), + theta_max=float(theta_max), + ) + + +def add_pure_eb(s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb): + """Add pure E/B-mode correlation functions for one tracer pair. + + Six blocks are inserted in ``PURE_KEYS`` order (xip_E, xim_E, xip_B, + xim_B, xip_amb, xim_amb), matching ``b_modes._EB_KEYS`` and the pure-EB + covariance layout. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + theta : array_like + Angular separations (arcmin), shared by all six blocks. + xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb : array_like + The six pure E/B / ambiguous mode arrays at ``theta``. + """ + tracers = _pair(bins) + values = { + "xip_E": xip_E, + "xim_E": xim_E, + "xip_B": xip_B, + "xim_B": xim_B, + "xip_amb": xip_amb, + "xim_amb": xim_amb, + } + for key in PURE_KEYS: + dtype, arr = PURE_TYPES[key], values[key] + for n, th in enumerate(theta): + s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + + +def add_rho(s, k, theta, rho_p, rho_m): + """Add a ρ_k PSF statistic (ρ+ then ρ−) on the ``psf_stars`` tracer. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + k : int + ρ index (0…5). + theta : array_like + Angular separations (arcmin). + rho_p, rho_m : array_like + ρ_k+ and ρ_k− at ``theta``. + """ + tracers = (PSF_TRACER, PSF_TRACER) + for dtype, arr in ((RHO_PLUS.format(k=k), rho_p), (RHO_MINUS.format(k=k), rho_m)): + for n, th in enumerate(theta): + s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + + +def add_tau(s, bins, k, theta, tau_p, tau_m): + """Add a τ_k PSF-leakage statistic (τ+ then τ−). + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin ``i`` and PSF; the τ tracers are ``(source_i, psf_stars)``. + Only ``bins[0]`` is used. + k : int + τ index (0, 2 or 5). + theta : array_like + Angular separations (arcmin). + tau_p, tau_m : array_like + τ_k+ and τ_k− at ``theta``. + """ + tracers = (source_name(bins[0]), PSF_TRACER) + for dtype, arr in ((TAU_PLUS.format(k=k), tau_p), (TAU_MINUS.format(k=k), tau_m)): + for n, th in enumerate(theta): + s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + + +def assemble_covariance(s, blocks): + """Assemble a block-diagonal ``FullCovariance`` from per-statistic blocks. + + Each block is validated against the current insertion order: its indices + must be contiguous and ascending, the blocks must tile ``0…len(s.mean)`` + exactly (no gap, no overlap), and each block must be square with a size + matching its index span. Any violation raises ``ValueError`` naming the + mismatch. Cross-blocks are left zero. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place via ``add_covariance``. + blocks : sequence + Ordered ``(selector, cov)`` pairs (or a mapping of the same). Each + ``selector`` is either an index array, or a ``(data_type, tracers)`` + / ``(data_type, tracers, tags)`` tuple resolved through + ``s.indices``; ``cov`` is the block's dense covariance. + + Returns + ------- + sacc.Sacc + ``s``, with the assembled ``FullCovariance`` attached. + """ + items = blocks.items() if isinstance(blocks, dict) else blocks + ntot = len(s.mean) + full = np.zeros((ntot, ntot)) + cursor = 0 + for selector, cov in items: + idx = _resolve_indices(s, selector) + cov = np.asarray(cov) + if not np.array_equal(idx, np.arange(idx[0], idx[0] + len(idx))): + raise ValueError( + f"covariance block {selector!r} resolves to non-contiguous " + f"or non-ascending indices {idx.tolist()}" + ) + if idx[0] != cursor: + raise ValueError( + f"covariance block {selector!r} starts at index {idx[0]} but " + f"the previous blocks cover through {cursor} — blocks must tile " + "the data vector with no gap or overlap" + ) + if cov.ndim != 2 or cov.shape[0] != cov.shape[1]: + raise ValueError( + f"covariance block {selector!r} must be square; got shape {cov.shape}" + ) + if cov.shape[0] != len(idx): + raise ValueError( + f"covariance block {selector!r} has size {cov.shape[0]} but " + f"spans {len(idx)} data points" + ) + full[np.ix_(idx, idx)] = cov + cursor = idx[-1] + 1 + if cursor != ntot: + raise ValueError( + f"covariance blocks cover {cursor} of {ntot} data points — the " + "blocks must tile the whole data vector" + ) + s.add_covariance(full) + return s + + +def _resolve_indices(s, selector): + """Resolve a covariance-block selector to a sorted index array.""" + if isinstance(selector, (np.ndarray, list, tuple, range)) and not ( + len(selector) in (2, 3) and isinstance(selector[0], str) + ): + return np.asarray(selector, dtype=int) + data_type, tracers = selector[0], selector[1] + tags = selector[2] if len(selector) == 3 else {} + return np.asarray(s.indices(data_type, tuple(tracers), **tags), dtype=int) + + +def add_diagonal_covariance(s, variances): + """Attach a ``DiagonalCovariance`` from a 1-D variance array. + + The 1-D array is passed straight to ``add_covariance`` (never + ``np.diag``), which is what makes SACC store a ``DiagonalCovariance``. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + variances : array_like + Per-point variances, ``len == len(s.mean)``. + + Returns + ------- + sacc.Sacc + ``s``, with the ``DiagonalCovariance`` attached. + """ + s.add_covariance(np.asarray(variances)) + return s + + +def get_nz(s, i): + """Return ``(z, nz)`` for source bin ``i``.""" + tracer = s.tracers[source_name(i)] + return tracer.z, tracer.nz + + +def get_xi(s, bins, *, grid): + """Return ``(theta, xip, xim)`` for one tracer pair and grid.""" + tracers = _pair(bins) + return ( + _sorted_tag(s, XI_PLUS, tracers, "theta", grid=grid), + _sorted_mean(s, XI_PLUS, tracers, grid=grid), + _sorted_mean(s, XI_MINUS, tracers, grid=grid), + ) + + +def get_pseudo_cl(s, bins): + """Return ``(ell_eff, cl_ee, cl_bb, cl_eb, window)`` for one tracer pair. + + ``window`` is the shared ``sacc.BandpowerWindow`` recovered via + ``get_bandpower_windows``. + """ + tracers = _pair(bins) + ell = _sorted_tag(s, CL_EE, tracers, "ell") + window = s.get_bandpower_windows(s.indices(CL_EE, tracers)) + return ( + ell, + _sorted_mean(s, CL_EE, tracers, _sort_tag="ell"), + _sorted_mean(s, CL_BB, tracers, _sort_tag="ell"), + _sorted_mean(s, CL_EB, tracers, _sort_tag="ell"), + window, + ) + + +def get_cosebis(s, bins, scale_cut=None): + """Return ``(n, En, Bn)`` for one tracer pair. + + Parameters + ---------- + scale_cut : tuple of float, optional + ``(theta_min, theta_max)`` to select when several cuts share the file. + """ + tracers = _pair(bins) + tags = ( + {"theta_min": float(scale_cut[0]), "theta_max": float(scale_cut[1])} + if scale_cut is not None + else {} + ) + modes = _sorted_tag(s, COSEBI_EE, tracers, "n", **tags) + return ( + modes.astype(int), + _sorted_mean(s, COSEBI_EE, tracers, **tags, _sort_tag="n"), + _sorted_mean(s, COSEBI_BB, tracers, **tags, _sort_tag="n"), + ) + + +def get_pure_eb(s, bins): + """Return ``(theta, {key: array})`` for the six pure-EB blocks. + + The dict is keyed by ``PURE_KEYS`` (xip_E, xim_E, …). + """ + tracers = _pair(bins) + theta = _sorted_tag(s, PURE_TYPES["xip_E"], tracers, "theta") + arrays = {key: _sorted_mean(s, PURE_TYPES[key], tracers) for key in PURE_KEYS} + return theta, arrays + + +def get_rho(s, k): + """Return ``(theta, rho_p, rho_m)`` for ρ index ``k``.""" + tracers = (PSF_TRACER, PSF_TRACER) + dt_p, dt_m = RHO_PLUS.format(k=k), RHO_MINUS.format(k=k) + return ( + _sorted_tag(s, dt_p, tracers, "theta"), + _sorted_mean(s, dt_p, tracers), + _sorted_mean(s, dt_m, tracers), + ) + + +def get_tau(s, bins, k): + """Return ``(theta, tau_p, tau_m)`` for τ index ``k`` and source bin.""" + tracers = (source_name(bins[0]), PSF_TRACER) + dt_p, dt_m = TAU_PLUS.format(k=k), TAU_MINUS.format(k=k) + return ( + _sorted_tag(s, dt_p, tracers, "theta"), + _sorted_mean(s, dt_p, tracers), + _sorted_mean(s, dt_m, tracers), + ) + + +def _order(s, data_type, tracers, sort_tag, **tag_filters): + """Indices for a selection, ordered by ascending ``sort_tag``.""" + idx = np.asarray(s.indices(data_type, tracers, **tag_filters), dtype=int) + key = np.array([s.data[i].tags[sort_tag] for i in idx]) + return idx[np.argsort(key)] + + +def _sorted_mean(s, data_type, tracers, _sort_tag="theta", **tag_filters): + """Mean values for a selection, ordered by ``_sort_tag`` ascending.""" + idx = _order(s, data_type, tracers, _sort_tag, **tag_filters) + return s.mean[idx] + + +def _sorted_tag(s, data_type, tracers, tag, **tag_filters): + """Values of ``tag`` for a selection, ordered by that tag ascending.""" + idx = _order(s, data_type, tracers, tag, **tag_filters) + return np.array([s.data[i].tags[tag] for i in idx]) + + +def extract(s, data_type=None, tracers=None, **tag_filters): + """Extract a sub-Sacc (points + aligned covariance sub-block). + + A copy is made and everything *not* matching the selection is removed, so + the covariance sub-block comes out correctly aligned and the original is + untouched. + + Parameters + ---------- + s : sacc.Sacc + Source, left unmodified. + data_type : str, optional + Data type to keep. + tracers : tuple, optional + Tracer pair to keep. + **tag_filters + Tag filters (plain kwargs, e.g. ``grid='fine'``). + + Returns + ------- + sacc.Sacc + New Sacc holding only the selected points. + """ + sub = s.copy() + selection = {} + if tracers is not None: + selection["tracers"] = tuple(tracers) + selection.update(tag_filters) + sub.keep_selection(data_type, **selection) + return sub + + +def save(s, path): + """Write ``s`` to ``path`` (FITS), overwriting any existing file.""" + s.save_fits(path, overwrite=True) + + +def load(path): + """Load a Sacc from ``path`` (FITS).""" + return sacc.Sacc.load_fits(path) diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py new file mode 100644 index 00000000..883a2af0 --- /dev/null +++ b/src/sp_validation/tests/test_sacc_io.py @@ -0,0 +1,491 @@ +"""Tests for :mod:`sp_validation.sacc_io`. + +All synthetic, all fast: build in memory, round-trip through ``tmp_path``, +and assert arrays/tags/windows/covariances come back bitwise-identical and +correctly aligned. No cluster paths. +""" + +import numpy as np +import pytest + +from sp_validation import sacc_io as sio + + +# --------------------------------------------------------------------------- # +# Synthetic builders +# --------------------------------------------------------------------------- # +def _nz(seed, n=50): + rng = np.random.default_rng(seed) + z = np.linspace(0.01, 2.0, n) + return z, rng.uniform(0.1, 1.0, n) + + +def _theta(nbins=6): + return np.geomspace(1.0, 100.0, nbins) + + +def _spd(n, seed): + """Symmetric positive-definite matrix of size ``n``.""" + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _base_sacc(nbins=1): + """A Sacc with ``nbins`` NZ source tracers plus the PSF tracer.""" + return sio.new_sacc({i: _nz(i) for i in range(nbins)}) + + +# --------------------------------------------------------------------------- # +# 1. Per-writer round-trip (arrays / tags / windows / NZ bitwise) +# --------------------------------------------------------------------------- # +def _roundtrip(s, tmp_path, name="rt"): + path = tmp_path / f"{name}.sacc" + sio.save(s, str(path)) + return sio.load(str(path)) + + +def test_nz_roundtrip(tmp_path): + z, nz = _nz(3) + s = sio.new_sacc({0: (z, nz)}, metadata={"version": "v1.4.6.3"}) + s2 = _roundtrip(s, tmp_path, "nz") + z2, nz2 = sio.get_nz(s2, 0) + assert np.array_equal(z2, z) + assert np.array_equal(nz2, nz) + assert s2.metadata["version"] == "v1.4.6.3" + assert sio.PSF_TRACER in s2.tracers + + +def test_xi_roundtrip(tmp_path): + theta = _theta() + xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 + npairs, weight = np.arange(6) * 1e3, np.arange(6) * 1.5 + s = _base_sacc() + sio.add_xi( + s, + (0, 0), + theta, + xip, + xim, + grid="coarse", + theta_nom=theta * 1.01, + npairs=npairs, + weight=weight, + ) + s2 = _roundtrip(s, tmp_path, "xi") + th, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(p, xip) + assert np.array_equal(m, xim) + # extra tags survive + idx = s2.indices(sio.XI_PLUS, ("source_0", "source_0"), grid="coarse") + tags = s2.data[idx[0]].tags + assert tags["grid"] == "coarse" + assert set(tags) >= {"theta", "theta_nom", "npairs", "weight", "grid"} + + +def test_pseudo_cl_roundtrip(tmp_path): + ell_eff = np.array([30.0, 120.0, 210.0, 300.0]) + nell, nbp = 50, len(ell_eff) + window_ells = np.arange(2, 2 + nell).astype(float) + W = np.random.default_rng(5).uniform(size=(nell, nbp)) + ee, bb, eb = np.arange(nbp) * 1e-9, np.arange(nbp) * 2e-9, np.arange(nbp) * 3e-9 + s = _base_sacc() + sio.add_pseudo_cl( + s, + (0, 0), + ell_eff, + ee, + bb, + eb, + window_ells=window_ells, + window_weights=W, + ) + s2 = _roundtrip(s, tmp_path, "cl") + ell, cl_ee, cl_bb, cl_eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell, ell_eff) + assert np.array_equal(cl_ee, ee) + assert np.array_equal(cl_bb, bb) + assert np.array_equal(cl_eb, eb) + assert np.array_equal(window.weight, W) + assert np.array_equal(window.values, window_ells) + + +def test_cosebis_roundtrip(tmp_path): + En, Bn = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 + s = _base_sacc() + sio.add_cosebis(s, (0, 0), En, Bn, (1.0, 100.0)) + s2 = _roundtrip(s, tmp_path, "cosebi") + n, E, B = sio.get_cosebis(s2, (0, 0)) + assert np.array_equal(n, np.arange(1, 11)) + assert np.array_equal(E, En) + assert np.array_equal(B, Bn) + idx = s2.indices(sio.COSEBI_EE, ("source_0", "source_0")) + assert s2.data[idx[0]].tags["theta_min"] == 1.0 + assert s2.data[idx[0]].tags["theta_max"] == 100.0 + + +def test_pure_eb_roundtrip(tmp_path): + theta = _theta() + arrays = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} + s = _base_sacc() + sio.add_pure_eb(s, (0, 0), theta, **arrays) + s2 = _roundtrip(s, tmp_path, "pureeb") + th, back = sio.get_pure_eb(s2, (0, 0)) + assert np.array_equal(th, theta) + for key in sio.PURE_KEYS: + assert np.array_equal(back[key], arrays[key]) + + +def test_rho_roundtrip(tmp_path): + theta = _theta() + s = _base_sacc() + for k in range(6): + sio.add_rho( + s, k, theta, np.arange(6) * (k + 1) * 1e-6, np.arange(6) * (k + 1) * 2e-6 + ) + s2 = _roundtrip(s, tmp_path, "rho") + for k in range(6): + th, p, m = sio.get_rho(s2, k) + assert np.array_equal(th, theta) + assert np.array_equal(p, np.arange(6) * (k + 1) * 1e-6) + assert np.array_equal(m, np.arange(6) * (k + 1) * 2e-6) + + +def test_tau_roundtrip(tmp_path): + theta = _theta() + s = _base_sacc() + for k in (0, 2, 5): + sio.add_tau( + s, + (0, 0), + k, + theta, + np.arange(6) * (k + 1) * 1e-6, + np.arange(6) * (k + 1) * 2e-6, + ) + s2 = _roundtrip(s, tmp_path, "tau") + for k in (0, 2, 5): + th, p, m = sio.get_tau(s2, (0, 0), k) + assert np.array_equal(th, theta) + assert np.array_equal(p, np.arange(6) * (k + 1) * 1e-6) + assert np.array_equal(m, np.arange(6) * (k + 1) * 2e-6) + assert s2.data[s2.indices(sio.TAU_PLUS.format(k=k))[0]].tracers == ( + "source_0", + sio.PSF_TRACER, + ) + + +# --------------------------------------------------------------------------- # +# 2. Covariance alignment +# --------------------------------------------------------------------------- # +def _multi_statistic_sacc(): + """Sacc with ξ+/ξ−, Cℓ (ee/bb/eb) and COSEBIs, ready for a covariance.""" + theta = _theta() + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + ell = np.array([30.0, 120.0, 210.0]) + W = np.random.default_rng(0).uniform(size=(20, 3)) + sio.add_pseudo_cl( + s, + (0, 0), + ell, + np.arange(3) * 1e-9, + np.arange(3) * 2e-9, + np.arange(3) * 3e-9, + window_ells=np.arange(2, 22).astype(float), + window_weights=W, + ) + sio.add_cosebis( + s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) + ) + return s + + +def test_assemble_covariance_alignment(tmp_path): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + # Block selectors in canonical (insertion) order. + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + cov_xi, cov_cl, cov_co = _spd(len(xi), 1), _spd(len(cl), 2), _spd(len(co), 3) + sio.assemble_covariance(s, [(xi, cov_xi), (cl, cov_cl), (co, cov_co)]) + s2 = _roundtrip(s, tmp_path, "cov") + assert type(s2.covariance).__name__ == "FullCovariance" + dense = s2.covariance.dense + # each block's sub-covariance is exactly what went in + assert np.array_equal(dense[np.ix_(xi, xi)], cov_xi) + assert np.array_equal(dense[np.ix_(cl, cl)], cov_cl) + assert np.array_equal(dense[np.ix_(co, co)], cov_co) + # zero cross-blocks + assert np.array_equal(dense[np.ix_(xi, cl)], np.zeros((len(xi), len(cl)))) + assert np.array_equal(dense[np.ix_(xi, co)], np.zeros((len(xi), len(co)))) + assert np.array_equal(dense[np.ix_(cl, co)], np.zeros((len(cl), len(co)))) + + +def test_assemble_covariance_selector_tuples(): + """Blocks addressed by (data_type, tracers) tuples, not raw indices.""" + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + sio.assemble_covariance( + s, + [ + (xi, _spd(len(xi), 1)), + (cl, _spd(len(cl), 2)), + ((sio.COSEBI_EE, tr), _spd(len(s.indices(sio.COSEBI_EE, tr)), 3)), + ((sio.COSEBI_BB, tr), _spd(len(s.indices(sio.COSEBI_BB, tr)), 4)), + ], + ) + assert type(s.covariance).__name__ == "FullCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) + + +# --------------------------------------------------------------------------- # +# 3. assemble_covariance failure modes +# --------------------------------------------------------------------------- # +def test_assemble_covariance_wrong_dimension(): + s = _base_sacc() + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + idx = np.arange(len(s.mean)) + with pytest.raises(ValueError, match="span"): + sio.assemble_covariance(s, [(idx, _spd(len(idx) - 1, 1))]) + + +def test_assemble_covariance_non_contiguous(): + s = _base_sacc() + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + idx = np.array([0, 2, 4, 6, 8, 10, 1, 3]) # not contiguous/ascending + with pytest.raises(ValueError, match="non-contiguous"): + sio.assemble_covariance(s, [(idx, _spd(len(idx), 1))]) + + +def test_assemble_covariance_missing_coverage(): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + with pytest.raises(ValueError, match="tile"): + sio.assemble_covariance( + s, [(xi, _spd(len(xi), 1))] + ) # leaves cl+cosebi uncovered + + +def test_assemble_covariance_non_square(): + s = _base_sacc() + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + idx = np.arange(len(s.mean)) + with pytest.raises(ValueError, match="square"): + sio.assemble_covariance(s, [(idx, np.ones((len(idx), len(idx) - 1)))]) + + +def test_assemble_covariance_overlap(): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + # second block starts before the first ended -> gap/overlap error + with pytest.raises(ValueError, match="tile|gap|overlap"): + sio.assemble_covariance(s, [(xi, _spd(len(xi), 1)), (xi, _spd(len(xi), 2))]) + + +# --------------------------------------------------------------------------- # +# 4. Fine file: DiagonalCovariance from 1-D variances +# --------------------------------------------------------------------------- # +def test_diagonal_covariance_roundtrip(tmp_path): + theta = np.geomspace(0.1, 250.0, 40) + n = len(theta) + xip, xim = np.arange(n) * 1e-5, np.arange(n) * 2e-5 + varxip, varxim = np.arange(1, n + 1) * 1e-12, np.arange(1, n + 1) * 2e-12 + s = _base_sacc() + sio.add_xi(s, (0, 0), theta, xip, xim, grid="fine") + variances = np.concatenate([varxip, varxim]) # [xip; xim] order + sio.add_diagonal_covariance(s, variances) + assert type(s.covariance).__name__ == "DiagonalCovariance" + s2 = _roundtrip(s, tmp_path, "fine") + assert type(s2.covariance).__name__ == "DiagonalCovariance" + assert np.array_equal(np.diag(s2.covariance.dense), variances) + + +# --------------------------------------------------------------------------- # +# 5. extract(): sub-covariance alignment; original untouched; tag filter +# --------------------------------------------------------------------------- # +def test_extract_subblock_and_original_untouched(): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + cov_co = _spd(len(co), 3) + sio.assemble_covariance( + s, [(xi, _spd(len(xi), 1)), (cl, _spd(len(cl), 2)), (co, cov_co)] + ) + n_before = len(s.mean) + sub = sio.extract(s, data_type=sio.COSEBI_EE, tracers=tr) + # original untouched + assert len(s.mean) == n_before + # subset covariance equals the COSEBI-EE diagonal sub-block + ee_local = np.arange(len(s.indices(sio.COSEBI_EE, tr))) + assert np.array_equal(sub.covariance.dense, cov_co[np.ix_(ee_local, ee_local)]) + + +def test_extract_tag_filter(): + theta = _theta() + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + sio.add_xi(s, (0, 0), theta, np.arange(6) * 3e-5, np.arange(6) * 4e-5, grid="fine") + sub = sio.extract( + s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0"), grid="fine" + ) + assert len(sub.mean) == len(theta) + assert set(sub.get_tag("grid", sio.XI_PLUS)) == {"fine"} + + +# --------------------------------------------------------------------------- # +# 6. Tomographic case: >=2 bins, >=3 pairs, per-pair selection +# --------------------------------------------------------------------------- # +def test_tomographic_per_pair_selection(tmp_path): + theta = _theta() + s = _base_sacc(nbins=2) + pairs = [(0, 0), (0, 1), (1, 1)] + for k, (i, j) in enumerate(pairs): + sio.add_xi( + s, + (i, j), + theta, + np.arange(6) * (k + 1) * 1e-5, + np.arange(6) * (k + 1) * 2e-5, + grid="coarse", + ) + s2 = _roundtrip(s, tmp_path, "tomo") + for k, (i, j) in enumerate(pairs): + th, p, m = sio.get_xi(s2, (i, j), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(p, np.arange(6) * (k + 1) * 1e-5) + assert np.array_equal(m, np.arange(6) * (k + 1) * 2e-5) + # selecting one pair does not bleed into another + assert len(s2.indices(sio.XI_PLUS, ("source_0", "source_1"))) == len(theta) + assert len(s2.indices(sio.XI_PLUS, ("source_1", "source_1"))) == len(theta) + + +# --------------------------------------------------------------------------- # +# 7. Readers mirror writers on a mixed file +# --------------------------------------------------------------------------- # +def test_readers_on_mixed_file(tmp_path): + theta = _theta() + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + ell = np.array([30.0, 120.0, 210.0]) + W = np.random.default_rng(0).uniform(size=(20, 3)) + sio.add_pseudo_cl( + s, + (0, 0), + ell, + np.arange(3) * 1e-9, + np.arange(3) * 2e-9, + np.arange(3) * 3e-9, + window_ells=np.arange(2, 22).astype(float), + window_weights=W, + ) + sio.add_cosebis( + s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) + ) + pure = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} + sio.add_pure_eb(s, (0, 0), theta, **pure) + for k in range(6): + sio.add_rho( + s, k, theta, np.arange(6) * (k + 1) * 1e-7, np.arange(6) * (k + 1) * 2e-7 + ) + for k in (0, 2, 5): + sio.add_tau( + s, + (0, 0), + k, + theta, + np.arange(6) * (k + 1) * 1e-8, + np.arange(6) * (k + 1) * 2e-8, + ) + s2 = _roundtrip(s, tmp_path, "mixed") + + _, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + assert np.array_equal(p, np.arange(6) * 1e-5) and np.array_equal( + m, np.arange(6) * 2e-5 + ) + ell_r, ee, bb, eb, win = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell_r, ell) and np.array_equal(win.weight, W) + n, E, B = sio.get_cosebis(s2, (0, 0)) + assert np.array_equal(E, np.arange(1, 6) * 1e-6) + _, back = sio.get_pure_eb(s2, (0, 0)) + for key in sio.PURE_KEYS: + assert np.array_equal(back[key], pure[key]) + for k in range(6): + _, rp, rm = sio.get_rho(s2, k) + assert np.array_equal(rp, np.arange(6) * (k + 1) * 1e-7) + for k in (0, 2, 5): + _, tp, tm = sio.get_tau(s2, (0, 0), k) + assert np.array_equal(tp, np.arange(6) * (k + 1) * 1e-8) + + +# --------------------------------------------------------------------------- # +# 8. End-to-end two-file layout for a synthetic catalogue version +# --------------------------------------------------------------------------- # +def test_end_to_end_two_file_layout(tmp_path): + version = "vSYNTH" + theta_c = _theta(20) + theta_f = np.geomspace(0.1, 250.0, 200) + + # analysis file + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta_c, np.arange(20) * 1e-5, np.arange(20) * 2e-5, grid="coarse" + ) + sio.add_cosebis( + s, (0, 0), np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7, (1.0, 100.0) + ) + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + sio.assemble_covariance(s, [(xi, _spd(len(xi), 1)), (co, _spd(len(co), 2))]) + sio.save(s, str(tmp_path / f"{version}.sacc")) + + # fine file + sf = _base_sacc() + sio.add_xi( + sf, (0, 0), theta_f, np.arange(200) * 1e-5, np.arange(200) * 2e-5, grid="fine" + ) + variances = np.concatenate([np.arange(1, 201) * 1e-12, np.arange(1, 201) * 2e-12]) + sio.add_diagonal_covariance(sf, variances) + sio.save(sf, str(tmp_path / f"{version}_xi_fine.sacc")) + + # reload both, verify everything + a = sio.load(str(tmp_path / f"{version}.sacc")) + f = sio.load(str(tmp_path / f"{version}_xi_fine.sacc")) + + th_c, p_c, _ = sio.get_xi(a, (0, 0), grid="coarse") + assert np.array_equal(th_c, theta_c) and np.array_equal(p_c, np.arange(20) * 1e-5) + n, E, B = sio.get_cosebis(a, (0, 0)) + assert np.array_equal(n, np.arange(1, 11)) + assert type(a.covariance).__name__ == "FullCovariance" + assert a.covariance.dense.shape == (len(a.mean), len(a.mean)) + + th_f, p_f, _ = sio.get_xi(f, (0, 0), grid="fine") + assert np.array_equal(th_f, theta_f) and np.array_equal(p_f, np.arange(200) * 1e-5) + assert type(f.covariance).__name__ == "DiagonalCovariance" + assert np.array_equal(np.diag(f.covariance.dense), variances) From 16e9c300454c06bbcf87de68e76f70b02d0761da Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 02:50:52 +0200 Subject: [PATCH 002/160] fix(sacc_io): enforce ascending grids; readers in insertion order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fresh-eyes review caught a correctness bug: readers re-sorted selections by theta/ell/n, but covariance blocks and bandpower windows stay in insertion order. On a non-ascending grid the reader output silently desynchronised from its covariance, and get_pseudo_cl returned sorted cl arrays against unsorted window columns — internally inconsistent within one return tuple. Fix by construction, not by sort: - Writers validate their grids. add_xi/add_pure_eb/add_rho/add_tau require strictly ascending theta; add_pseudo_cl requires strictly ascending ell_eff (add_cosebis is inherently safe — it enumerates the mode index). Out-of-order grids raise a loud ValueError naming the argument. - Readers drop the sort entirely and return in s.indices (insertion) order, so every getter is covariance- and window-aligned for ANY file, and ascending for canonical files. The _sorted_* helpers are replaced by plain insertion-order accessors (_mean/_tag). Also: - _pair normalises (i, j) -> sorted, so get_xi(s, (1, 0)) addresses the same symmetric shear-shear pair as (0, 1) instead of a silent empty read. - Module docstring documents the tomographic ξ covariance ordering: insertion is pair-major ([pair0 xip; pair0 xim; pair1 xip; …]), supplied to assemble_covariance as one contiguous block matching add_xi call order; type-major converters (DES 2pt-FITS) permute explicitly via s.indices. - extract() docstring states tracers takes SACC names, not integer bins. New tests (26 total, was 20): writers reject non-ascending theta/ell; (1, 0) == (0, 1) round-trip; a 3-pair tomographic ξ covariance assembled as one contiguous pair-major block with per-pair sub-blocks recovered via extract(); get_pseudo_cl window column j <-> ell_eff[j] via window_ind tags. Co-Authored-By: Claude Opus --- src/sp_validation/sacc_io.py | 114 +++++++++++++------ src/sp_validation/tests/test_sacc_io.py | 140 ++++++++++++++++++++++++ 2 files changed, 218 insertions(+), 36 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index cf03f12c..4ed85d0f 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -26,6 +26,18 @@ Tag filters are plain keyword arguments to ``indices`` / ``get_data_points`` / ``get_tag``; the ``tags={...}`` form silently selects nothing and must never be used. + + Tomographic ξ ordering: each ``add_xi`` call inserts one tracer + pair as ``[xip; xim]``, so a multi-pair vector is *pair-major* + — ``[pair_0 xip; pair_0 xim; pair_1 xip; …]`` — not type-major + (``[all xip; all xim]``). A tomographic ξ covariance with + cross-pair correlations is therefore supplied to + ``assemble_covariance`` as ONE contiguous block spanning the + consecutive ``add_xi`` calls, ordered pair-by-pair to match + insertion. Writers must call ``add_xi`` in the same pair order + the covariance was built in. Converters that need a type-major + layout (e.g. the DES 2pt-FITS convention) permute explicitly via + ``s.indices`` rather than assuming global order. """ import numpy as np @@ -97,11 +109,32 @@ def new_sacc(nz, metadata=None): def _pair(bins): - """Resolve a ``(i, j)`` bin pair to the ``(source_i, source_j)`` names.""" - i, j = bins + """Resolve a ``(i, j)`` bin pair to the ``(source_i, source_j)`` names. + + The pair is normalised to ``i <= j``: shear-shear statistics are symmetric + in the tracer pair, and SACC stores each pair under one ordering, so + ``(1, 0)`` must address the same points as ``(0, 1)``. + """ + i, j = sorted(bins) return (source_name(i), source_name(j)) +def _check_ascending(name, values): + """Require ``values`` to be strictly ascending; else raise ValueError. + + Insertion order is the covariance (and bandpower-window) order, and readers + return points in insertion order, so an out-of-order grid would silently + desynchronise a data vector from its covariance. Enforce monotonicity at + write time instead. + """ + values = np.asarray(values) + if not np.all(np.diff(values) > 0): + raise ValueError( + f"{name} must be strictly ascending (insertion order is the " + f"covariance order); got {values.tolist()}" + ) + + def add_xi( s, bins, @@ -134,6 +167,7 @@ def add_xi( npairs, weight : array_like, optional TreeCorr pair counts and weights, stored per point. """ + _check_ascending("theta", theta) tracers = _pair(bins) for dtype, xi in ((XI_PLUS, xip), (XI_MINUS, xim)): for n, th in enumerate(theta): @@ -177,6 +211,7 @@ def add_pseudo_cl( bandpower — from NaMaster ``get_bandpower_windows``. One ``sacc.BandpowerWindow`` is built and shared across EE/BB/EB. """ + _check_ascending("ell_eff", ell_eff) tracers = _pair(bins) window = sacc.BandpowerWindow(np.asarray(window_ells), np.asarray(window_weights)) for dtype, cl in ((CL_EE, cl_ee), (CL_BB, cl_bb), (CL_EB, cl_eb)): @@ -234,6 +269,7 @@ def add_pure_eb(s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb): xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb : array_like The six pure E/B / ambiguous mode arrays at ``theta``. """ + _check_ascending("theta", theta) tracers = _pair(bins) values = { "xip_E": xip_E, @@ -263,6 +299,7 @@ def add_rho(s, k, theta, rho_p, rho_m): rho_p, rho_m : array_like ρ_k+ and ρ_k− at ``theta``. """ + _check_ascending("theta", theta) tracers = (PSF_TRACER, PSF_TRACER) for dtype, arr in ((RHO_PLUS.format(k=k), rho_p), (RHO_MINUS.format(k=k), rho_m)): for n, th in enumerate(theta): @@ -286,6 +323,7 @@ def add_tau(s, bins, k, theta, tau_p, tau_m): tau_p, tau_m : array_like τ_k+ and τ_k− at ``theta``. """ + _check_ascending("theta", theta) tracers = (source_name(bins[0]), PSF_TRACER) for dtype, arr in ((TAU_PLUS.format(k=k), tau_p), (TAU_MINUS.format(k=k), tau_m)): for n, th in enumerate(theta): @@ -397,9 +435,9 @@ def get_xi(s, bins, *, grid): """Return ``(theta, xip, xim)`` for one tracer pair and grid.""" tracers = _pair(bins) return ( - _sorted_tag(s, XI_PLUS, tracers, "theta", grid=grid), - _sorted_mean(s, XI_PLUS, tracers, grid=grid), - _sorted_mean(s, XI_MINUS, tracers, grid=grid), + _tag(s, XI_PLUS, tracers, "theta", grid=grid), + _mean(s, XI_PLUS, tracers, grid=grid), + _mean(s, XI_MINUS, tracers, grid=grid), ) @@ -407,16 +445,17 @@ def get_pseudo_cl(s, bins): """Return ``(ell_eff, cl_ee, cl_bb, cl_eb, window)`` for one tracer pair. ``window`` is the shared ``sacc.BandpowerWindow`` recovered via - ``get_bandpower_windows``. + ``get_bandpower_windows``; its columns are in the same insertion order as + the returned ``ell_eff``/``cl`` arrays, so window column ``j`` corresponds + to ``ell_eff[j]``. """ tracers = _pair(bins) - ell = _sorted_tag(s, CL_EE, tracers, "ell") window = s.get_bandpower_windows(s.indices(CL_EE, tracers)) return ( - ell, - _sorted_mean(s, CL_EE, tracers, _sort_tag="ell"), - _sorted_mean(s, CL_BB, tracers, _sort_tag="ell"), - _sorted_mean(s, CL_EB, tracers, _sort_tag="ell"), + _tag(s, CL_EE, tracers, "ell"), + _mean(s, CL_EE, tracers), + _mean(s, CL_BB, tracers), + _mean(s, CL_EB, tracers), window, ) @@ -435,11 +474,11 @@ def get_cosebis(s, bins, scale_cut=None): if scale_cut is not None else {} ) - modes = _sorted_tag(s, COSEBI_EE, tracers, "n", **tags) + modes = _tag(s, COSEBI_EE, tracers, "n", **tags) return ( modes.astype(int), - _sorted_mean(s, COSEBI_EE, tracers, **tags, _sort_tag="n"), - _sorted_mean(s, COSEBI_BB, tracers, **tags, _sort_tag="n"), + _mean(s, COSEBI_EE, tracers, **tags), + _mean(s, COSEBI_BB, tracers, **tags), ) @@ -449,8 +488,8 @@ def get_pure_eb(s, bins): The dict is keyed by ``PURE_KEYS`` (xip_E, xim_E, …). """ tracers = _pair(bins) - theta = _sorted_tag(s, PURE_TYPES["xip_E"], tracers, "theta") - arrays = {key: _sorted_mean(s, PURE_TYPES[key], tracers) for key in PURE_KEYS} + theta = _tag(s, PURE_TYPES["xip_E"], tracers, "theta") + arrays = {key: _mean(s, PURE_TYPES[key], tracers) for key in PURE_KEYS} return theta, arrays @@ -459,9 +498,9 @@ def get_rho(s, k): tracers = (PSF_TRACER, PSF_TRACER) dt_p, dt_m = RHO_PLUS.format(k=k), RHO_MINUS.format(k=k) return ( - _sorted_tag(s, dt_p, tracers, "theta"), - _sorted_mean(s, dt_p, tracers), - _sorted_mean(s, dt_m, tracers), + _tag(s, dt_p, tracers, "theta"), + _mean(s, dt_p, tracers), + _mean(s, dt_m, tracers), ) @@ -470,28 +509,26 @@ def get_tau(s, bins, k): tracers = (source_name(bins[0]), PSF_TRACER) dt_p, dt_m = TAU_PLUS.format(k=k), TAU_MINUS.format(k=k) return ( - _sorted_tag(s, dt_p, tracers, "theta"), - _sorted_mean(s, dt_p, tracers), - _sorted_mean(s, dt_m, tracers), + _tag(s, dt_p, tracers, "theta"), + _mean(s, dt_p, tracers), + _mean(s, dt_m, tracers), ) -def _order(s, data_type, tracers, sort_tag, **tag_filters): - """Indices for a selection, ordered by ascending ``sort_tag``.""" - idx = np.asarray(s.indices(data_type, tracers, **tag_filters), dtype=int) - key = np.array([s.data[i].tags[sort_tag] for i in idx]) - return idx[np.argsort(key)] +def _mean(s, data_type, tracers, **tag_filters): + """Mean values for a selection, in ``s.indices`` (insertion) order. - -def _sorted_mean(s, data_type, tracers, _sort_tag="theta", **tag_filters): - """Mean values for a selection, ordered by ``_sort_tag`` ascending.""" - idx = _order(s, data_type, tracers, _sort_tag, **tag_filters) - return s.mean[idx] + Never re-sort: insertion order is the covariance and bandpower-window + order, so returning in ``s.indices`` order keeps every reader aligned with + the covariance for any file (and ascending for canonically-written files, + which the writers enforce). + """ + return s.mean[s.indices(data_type, tracers, **tag_filters)] -def _sorted_tag(s, data_type, tracers, tag, **tag_filters): - """Values of ``tag`` for a selection, ordered by that tag ascending.""" - idx = _order(s, data_type, tracers, tag, **tag_filters) +def _tag(s, data_type, tracers, tag, **tag_filters): + """Values of ``tag`` for a selection, in insertion order.""" + idx = s.indices(data_type, tracers, **tag_filters) return np.array([s.data[i].tags[tag] for i in idx]) @@ -509,7 +546,12 @@ def extract(s, data_type=None, tracers=None, **tag_filters): data_type : str, optional Data type to keep. tracers : tuple, optional - Tracer pair to keep. + Tracer pair to keep, as SACC tracer **names** (e.g. + ``("source_0", "source_0")`` or ``("source_0", "psf_stars")``) — *not* + integer bin indices. This differs deliberately from the ``add_*`` / + ``get_*`` interface, whose ``bins`` argument takes integer pairs: + ``extract`` is the generic selection escape hatch, mirroring + ``Sacc.keep_selection`` and addressing non-source tracers uniformly. **tag_filters Tag filters (plain kwargs, e.g. ``grid='fine'``). diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 883a2af0..28bd6627 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -489,3 +489,143 @@ def test_end_to_end_two_file_layout(tmp_path): assert np.array_equal(th_f, theta_f) and np.array_equal(p_f, np.arange(200) * 1e-5) assert type(f.covariance).__name__ == "DiagonalCovariance" assert np.array_equal(np.diag(f.covariance.dense), variances) + + +# --------------------------------------------------------------------------- # +# 9. Ascending-grid enforcement (writers reject out-of-order grids) +# --------------------------------------------------------------------------- # +def test_add_xi_rejects_non_ascending_theta(): + s = _base_sacc() + theta = _theta()[::-1] # descending + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + + +def test_add_pseudo_cl_rejects_non_ascending_ell(): + s = _base_sacc() + ell = np.array([210.0, 30.0, 120.0]) # not ascending + W = np.random.default_rng(0).uniform(size=(20, 3)) + with pytest.raises(ValueError, match="ell_eff must be strictly ascending"): + sio.add_pseudo_cl( + s, + (0, 0), + ell, + np.arange(3) * 1e-9, + np.arange(3) * 2e-9, + np.arange(3) * 3e-9, + window_ells=np.arange(2, 22).astype(float), + window_weights=W, + ) + + +def test_add_pure_eb_rho_tau_reject_non_ascending_theta(): + s = _base_sacc() + theta = _theta()[::-1] + pure = {key: np.arange(6) * 1e-6 for key in sio.PURE_KEYS} + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_pure_eb(s, (0, 0), theta, **pure) + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_rho(s, 0, theta, np.arange(6) * 1e-6, np.arange(6) * 2e-6) + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_tau(s, (0, 0), 0, theta, np.arange(6) * 1e-6, np.arange(6) * 2e-6) + + +# --------------------------------------------------------------------------- # +# 10. Bin-pair normalisation: (1, 0) addresses the same points as (0, 1) +# --------------------------------------------------------------------------- # +def test_bin_pair_normalisation(): + theta = _theta() + xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 + s = _base_sacc(nbins=2) + sio.add_xi(s, (0, 1), theta, xip, xim, grid="coarse") + th01, p01, m01 = sio.get_xi(s, (0, 1), grid="coarse") + th10, p10, m10 = sio.get_xi(s, (1, 0), grid="coarse") # reversed order + assert np.array_equal(th01, th10) + assert np.array_equal(p01, p10) and np.array_equal(p10, xip) + assert np.array_equal(m01, m10) and np.array_equal(m10, xim) + # writing under (1, 0) lands in the same tracer pair, not a new one + s2 = _base_sacc(nbins=2) + sio.add_xi(s2, (1, 0), theta, xip, xim, grid="coarse") + assert len(s2.indices(sio.XI_PLUS, ("source_0", "source_1"))) == len(theta) + + +# --------------------------------------------------------------------------- # +# 11. Reader/covariance alignment holds for ANY insertion order (the core +# regression the review caught): a tomographic multi-pair ξ covariance +# assembled as ONE contiguous pair-major block, per-pair sub-blocks +# recovered via extract(). +# --------------------------------------------------------------------------- # +def test_tomographic_xi_covariance_one_contiguous_block(): + theta = _theta() + nth = len(theta) + pairs = [(0, 0), (0, 1), (1, 1)] + s = _base_sacc(nbins=2) + for k, (i, j) in enumerate(pairs): + sio.add_xi( + s, + (i, j), + theta, + np.arange(nth) * (k + 1) * 1e-5, + np.arange(nth) * (k + 1) * 2e-5, + grid="coarse", + ) + # All ξ points as one contiguous block in insertion (pair-major) order. + xi_idx = np.arange(len(s.mean)) + assert np.array_equal(xi_idx, np.arange(3 * 2 * nth)) # 3 pairs x [xip; xim] + cov = _spd(len(xi_idx), 11) # dense, cross-pair correlations + sio.assemble_covariance(s, [(xi_idx, cov)]) + + # Per-pair xip sub-block: resolve indices, extract, compare to input. + for i, j in pairs: + idx_p = s.indices( + sio.XI_PLUS, ("source_" + str(min(i, j)), "source_" + str(max(i, j))) + ) + sub = sio.extract( + s, + data_type=sio.XI_PLUS, + tracers=("source_" + str(min(i, j)), "source_" + str(max(i, j))), + ) + assert np.array_equal(sub.covariance.dense, cov[np.ix_(idx_p, idx_p)]) + # readers stay covariance-aligned: get_xi returns in the same order + th, xip, _ = sio.get_xi(s, (i, j), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(s.mean[idx_p], xip) + + +# --------------------------------------------------------------------------- # +# 12. get_pseudo_cl window/cl column correspondence: window column j maps to +# the returned ell_eff[j] (verified through window_ind tags). +# --------------------------------------------------------------------------- # +def test_pseudo_cl_window_column_correspondence(tmp_path): + ell_eff = np.array([30.0, 120.0, 210.0, 300.0]) + nell, nbp = 40, len(ell_eff) + window_ells = np.arange(2, 2 + nell).astype(float) + # Distinct columns so a permutation would be detectable. + W = np.zeros((nell, nbp)) + for b in range(nbp): + W[b * 5 : (b + 1) * 5, b] = 1.0 + ee, bb, eb = np.arange(nbp) * 1e-9, np.arange(nbp) * 2e-9, np.arange(nbp) * 3e-9 + s = _base_sacc() + sio.add_pseudo_cl( + s, + (0, 0), + ell_eff, + ee, + bb, + eb, + window_ells=window_ells, + window_weights=W, + ) + s2 = _roundtrip(s, tmp_path, "clwin") + ell, cl_ee, _, _, window = sio.get_pseudo_cl(s2, (0, 0)) + # returned ell array is in insertion order + assert np.array_equal(ell, ell_eff) + assert np.array_equal(cl_ee, ee) + # window_ind tag on each EE point indexes the matching window column + idx = s2.indices(sio.CL_EE, ("source_0", "source_0")) + for pos, i in enumerate(idx): + col = s2.data[i].tags["window_ind"] + assert col == pos # insertion order preserved => column j <-> ell[j] + assert np.array_equal(window.weight[:, col], W[:, pos]) From 7eb8595c453b9732dd455e00e483ca7d14bb1d64 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 11 Jul 2026 00:09:37 +0200 Subject: [PATCH 003/160] One SACC file per catalogue version: fine xi rides as grid='fine' points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two-file split (analysis + {version}_xi_fine.sacc) was premised on a 10000-bin fine grid; the production operating point (Paper II B-modes) is 1000 bins, where a dense per-pair fine covariance block is ~32 MB and the CosmoCov integration-binning covariance — which feeds pure-E/B and COSEBIs error propagation — has a natural home as a BlockDiagonal block alongside the analysis blocks. Layout test replaced with the one-file end-to-end case (dense fine block, extract() sub-covariance alignment, zero cross-blocks) plus a varxi-diagonal fallback test. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019R5eiy11Lihkgn4MKufXSp --- src/sp_validation/sacc_io.py | 24 +++++--- src/sp_validation/tests/test_sacc_io.py | 82 +++++++++++++++++++------ 2 files changed, 77 insertions(+), 29 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 4ed85d0f..22fe5aa6 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -3,17 +3,23 @@ :Name: sacc_io.py :Description: Read/write the standard SACC data-product layout for the - weak-lensing validation package. Two files describe each + weak-lensing validation package. One file describes each catalogue version: - - ``{version}.sacc`` — the analysis vector: NZ tracers, coarse - ξ±, pseudo-Cℓ (EE/BB/EB) with bandpower windows, COSEBIs, - pure E/B, and ρ/τ PSF diagnostics, all sharing a single - ``FullCovariance`` assembled block-diagonally from the - per-statistic covariances (zero cross-blocks). - - ``{version}_xi_fine.sacc`` — the COSEBIs / pure-EB integration - input: the same NZ tracers, a fine-grid ξ±, and a - ``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim``. + - ``{version}.sacc`` — NZ tracers, coarse ξ±, pseudo-Cℓ + (EE/BB/EB) with bandpower windows, COSEBIs, pure E/B, ρ/τ PSF + diagnostics, and the fine-grid ξ± integration input for + COSEBIs / pure-EB (``grid='fine'`` tagged points). The + covariance is assembled block-diagonally from the + per-statistic covariances (zero cross-blocks): the analysis + blocks first, then a dense per-pair fine-ξ block (the + CosmoCov integration-binning covariance when it exists — it + feeds derived-statistic error propagation — or the TreeCorr + ``varxip``/``varxim`` diagonal as degraded fallback). At the + production fine binning (1000 θ bins) a dense fine block is + ~32 MB per pair; extreme convergence-check grids (10k bins) + degrade to the diagonal fallback rather than forking the + layout. The covariance order is the point-insertion order (SACC preserves it bitwise through FITS save/load). Writers below insert in the diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 28bd6627..cb9c5a37 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -444,14 +444,14 @@ def test_readers_on_mixed_file(tmp_path): # --------------------------------------------------------------------------- # -# 8. End-to-end two-file layout for a synthetic catalogue version +# 8. End-to-end one-file layout for a synthetic catalogue version # --------------------------------------------------------------------------- # -def test_end_to_end_two_file_layout(tmp_path): +def test_end_to_end_one_file_layout(tmp_path): version = "vSYNTH" theta_c = _theta(20) theta_f = np.geomspace(0.1, 250.0, 200) - # analysis file + # one file: analysis products first, fine-grid integration input last s = _base_sacc() sio.add_xi( s, (0, 0), theta_c, np.arange(20) * 1e-5, np.arange(20) * 2e-5, grid="coarse" @@ -459,36 +459,78 @@ def test_end_to_end_two_file_layout(tmp_path): sio.add_cosebis( s, (0, 0), np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7, (1.0, 100.0) ) + sio.add_xi( + s, (0, 0), theta_f, np.arange(200) * 1e-5, np.arange(200) * 2e-5, grid="fine" + ) tr = ("source_0", "source_0") - xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + xi_c = np.concatenate( + [ + s.indices(sio.XI_PLUS, tr, grid="coarse"), + s.indices(sio.XI_MINUS, tr, grid="coarse"), + ] + ) co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) - sio.assemble_covariance(s, [(xi, _spd(len(xi), 1)), (co, _spd(len(co), 2))]) - sio.save(s, str(tmp_path / f"{version}.sacc")) - - # fine file - sf = _base_sacc() - sio.add_xi( - sf, (0, 0), theta_f, np.arange(200) * 1e-5, np.arange(200) * 2e-5, grid="fine" + xi_f = np.concatenate( + [ + s.indices(sio.XI_PLUS, tr, grid="fine"), + s.indices(sio.XI_MINUS, tr, grid="fine"), + ] + ) + # dense fine block (CosmoCov integration covariance in production) + fine_block = _spd(len(xi_f), 3) + sio.assemble_covariance( + s, + [(xi_c, _spd(len(xi_c), 1)), (co, _spd(len(co), 2)), (xi_f, fine_block)], ) - variances = np.concatenate([np.arange(1, 201) * 1e-12, np.arange(1, 201) * 2e-12]) - sio.add_diagonal_covariance(sf, variances) - sio.save(sf, str(tmp_path / f"{version}_xi_fine.sacc")) + sio.save(s, str(tmp_path / f"{version}.sacc")) - # reload both, verify everything a = sio.load(str(tmp_path / f"{version}.sacc")) - f = sio.load(str(tmp_path / f"{version}_xi_fine.sacc")) th_c, p_c, _ = sio.get_xi(a, (0, 0), grid="coarse") assert np.array_equal(th_c, theta_c) and np.array_equal(p_c, np.arange(20) * 1e-5) n, E, B = sio.get_cosebis(a, (0, 0)) assert np.array_equal(n, np.arange(1, 11)) - assert type(a.covariance).__name__ == "FullCovariance" assert a.covariance.dense.shape == (len(a.mean), len(a.mean)) - th_f, p_f, _ = sio.get_xi(f, (0, 0), grid="fine") + th_f, p_f, _ = sio.get_xi(a, (0, 0), grid="fine") assert np.array_equal(th_f, theta_f) and np.array_equal(p_f, np.arange(200) * 1e-5) - assert type(f.covariance).__name__ == "DiagonalCovariance" - assert np.array_equal(np.diag(f.covariance.dense), variances) + + # extract() of the fine selection pulls the aligned dense sub-covariance + fine = sio.extract(a, sio.XI_PLUS, tr, grid="fine") + idx_p = a.indices(sio.XI_PLUS, tr, grid="fine") + assert np.allclose(fine.covariance.dense, a.covariance.dense[np.ix_(idx_p, idx_p)]) + # zero cross-blocks between analysis and fine points + assert np.all(a.covariance.dense[np.ix_(xi_c, xi_f)] == 0) + + +def test_one_file_layout_diagonal_fine_fallback(tmp_path): + """No CosmoCov covariance: the fine block is np.diag(varxip/varxim).""" + s = _base_sacc() + theta_f = np.geomspace(0.1, 250.0, 50) + sio.add_xi( + s, (0, 0), _theta(6), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + sio.add_xi( + s, (0, 0), theta_f, np.arange(50) * 1e-5, np.arange(50) * 2e-5, grid="fine" + ) + tr = ("source_0", "source_0") + xi_c = np.concatenate( + [ + s.indices(sio.XI_PLUS, tr, grid="coarse"), + s.indices(sio.XI_MINUS, tr, grid="coarse"), + ] + ) + xi_f = np.concatenate( + [ + s.indices(sio.XI_PLUS, tr, grid="fine"), + s.indices(sio.XI_MINUS, tr, grid="fine"), + ] + ) + variances = np.concatenate([np.arange(1, 51) * 1e-12, np.arange(1, 51) * 2e-12]) + sio.assemble_covariance(s, [(xi_c, _spd(len(xi_c), 1)), (xi_f, np.diag(variances))]) + sio.save(s, str(tmp_path / "vDIAG.sacc")) + a = sio.load(str(tmp_path / "vDIAG.sacc")) + assert np.array_equal(np.diag(a.covariance.dense[np.ix_(xi_f, xi_f)]), variances) # --------------------------------------------------------------------------- # From 85771bab646dbf98602d3a600c09e49b0619107e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 11 Jul 2026 11:55:55 +0200 Subject: [PATCH 004/160] Word fine-block covariance source tool-agnostically (OneCovariance go-forward) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019R5eiy11Lihkgn4MKufXSp --- src/sp_validation/sacc_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 22fe5aa6..157f409e 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -13,7 +13,7 @@ covariance is assembled block-diagonally from the per-statistic covariances (zero cross-blocks): the analysis blocks first, then a dense per-pair fine-ξ block (the - CosmoCov integration-binning covariance when it exists — it + analytic integration-binning covariance when it exists — it feeds derived-statistic error propagation — or the TreeCorr ``varxip``/``varxim`` diagonal as degraded fallback). At the production fine binning (1000 θ bins) a dense fine block is From 9b7e78785dc0d72603c254291f8e3136da59bef8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 00:37:49 +0200 Subject: [PATCH 005/160] feat(sacc_io): type=data|mock stamp with fail-closed load; merge + update_statistic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PRD #241 §4 (Mocks vs data): save() requires type='data'|'mock' and stamps it into metadata; load() raises on type='data' files lacking the concealed=True blinding stamp, so skipping the blind can never silently expose real data. Mocks load freely. allow_unblinded=True is the loud escape hatch reserved for the blinding/unblinding tooling. merge() wraps sacc.concatenate_data_sets thinly: per-statistic files combine in order, shared tracers stored once, covariance block-diagonal (library-enforced all-or-none), metadata union with loud conflicts. update_statistic() is the value-only merge-back for the extract -> conceal -> merge blinding flow: matches each sub point by (data_type, tracers, tags) and overwrites the value, leaving order and covariance untouched. Tests: all four data/mock x concealed/unconcealed quadrants, the escape hatch, stamp validation, merge (points, covariance, metadata conflict, mixed-covariance failure) and update_statistic (values-only, unique-match). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QWF72ofwJh6ekgnCt9Xx6C --- src/sp_validation/sacc_io.py | 151 ++++++++++++++++++- src/sp_validation/tests/test_sacc_io.py | 184 +++++++++++++++++++++++- 2 files changed, 326 insertions(+), 9 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 157f409e..dae6e8d0 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -575,11 +575,152 @@ def extract(s, data_type=None, tracers=None, **tag_filters): return sub -def save(s, path): - """Write ``s`` to ``path`` (FITS), overwriting any existing file.""" +def merge(saccs): + """Merge several per-statistic Sacc objects into one file's worth. + + A thin wrapper around ``sacc.concatenate_data_sets``: data points + concatenate in input order, tracers shared by several inputs (the + ``source_i`` NZ tracers, ``psf_stars``) are stored once, and the + covariance combines block-diagonally in the same order — the library + requires either **all** inputs to carry a covariance or **none**, and + raises otherwise (cross-statistic covariance assembly beyond + block-diagonal is out of scope here; see ``assemble_covariance``). + + Metadata must be consistent: keys present in several inputs must carry + equal values (a ``type: data`` file cannot merge with a ``type: mock`` + file), and the union lands on the result. This deliberately replaces the + library's clash behaviour, which mangles clashing keys by appending + labels. + + Parameters + ---------- + saccs : sequence of sacc.Sacc + The per-statistic data sets, in the insertion order the merged file + should have. Inputs are left unmodified. + + Returns + ------- + sacc.Sacc + The merged data set. + """ + saccs = list(saccs) + metadata = {} + for s in saccs: + for key, value in s.metadata.items(): + if key in metadata and metadata[key] != value: + raise ValueError( + f"conflicting metadata across merge inputs: {key!r} is " + f"{metadata[key]!r} in one input and {value!r} in another" + ) + metadata[key] = value + # Strip metadata before concatenating (the library "resolves" clashing + # keys by renaming them), then restore the validated union. + stripped = [] + for s in saccs: + s = s.copy() + s.metadata.clear() + stripped.append(s) + seen, shared = set(), set() # tracers appearing in more than one input + for s in saccs: + shared |= seen & set(s.tracers) + seen |= set(s.tracers) + same_tracers = sorted(shared) + merged = sacc.concatenate_data_sets(*stripped, same_tracers=same_tracers) + for key, value in metadata.items(): + merged.metadata[key] = value + return merged + + +def update_statistic(s, sub): + """Overwrite the values of ``s``'s points that match ``sub``'s, in place. + + The merge-back half of the extract → conceal → merge blinding flow + (PRD #241 §4): each point of ``sub`` is matched to exactly one point of + ``s`` by ``(data_type, tracers, tags)``, and that point's *value* is + replaced. Nothing else changes — insertion order, tags, windows and the + covariance are untouched (blinding shifts the mean only), so ``sub``'s + own covariance (e.g. the sub-block ``extract`` attaches) is deliberately + not consulted. A ``sub`` point with no match, or with several, raises + ``ValueError``. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + sub : sacc.Sacc + The replacement block, e.g. ``extract(s, ...)`` after concealment. + """ + for point in sub.data: + idx = s.indices(point.data_type, point.tracers, **point.tags) + if len(idx) != 1: + raise ValueError( + f"update_statistic: {len(idx)} points in the target match " + f"({point.data_type}, {point.tracers}, {point.tags}) — need " + "exactly one" + ) + s.data[idx[0]].value = point.value + + +def save(s, path, *, type): + """Write ``s`` to ``path`` (FITS), overwriting any existing file. + + Parameters + ---------- + s : sacc.Sacc + Data set to write; its metadata is stamped in place. + type : {'data', 'mock'} + Provenance of the underlying catalogue, stored as the required + ``type`` metadata tag (PRD #241 §4, "Mocks vs data"). The caller — + the pipeline computing the data vector — knows whether its input + catalogue is a mock; there is deliberately no default. ``load`` + refuses ``type='data'`` files that are not blinded. + """ + if type not in ("data", "mock"): + raise ValueError(f"type must be 'data' or 'mock'; got {type!r}") + if s.metadata.get("type", type) != type: + raise ValueError( + f"Sacc metadata already carries type={s.metadata['type']!r}; " + f"refusing to re-stamp as {type!r}" + ) + s.metadata["type"] = type s.save_fits(path, overwrite=True) -def load(path): - """Load a Sacc from ``path`` (FITS).""" - return sacc.Sacc.load_fits(path) +def load(path, *, allow_unblinded=False): + """Load a Sacc from ``path`` (FITS), failing closed on unblinded data. + + Every sacc_io file carries a ``type: data|mock`` metadata tag (stamped by + ``save``); blinded files are additionally stamped ``concealed=True`` by + Smokescreen. A ``type='data'`` file without that stamp is real, unblinded + data, and loading it raises — skipping the blind can never silently + expose the measured vector (PRD #241 §4). Mocks load freely, blinded or + not. + + Parameters + ---------- + path : str + File to load. + allow_unblinded : bool, optional + Escape hatch for the two legitimate consumers of unblinded data: + the blinding step itself (which must read the true vector to conceal + it) and the unblinding/verification tooling. Nothing else — no + analysis, plotting or inference code — may pass ``True``. + + Returns + ------- + sacc.Sacc + The loaded data set. + """ + s = sacc.Sacc.load_fits(path) + if ( + s.metadata["type"] == "data" + and not s.metadata.get("concealed", False) + and not allow_unblinded + ): + raise ValueError( + f"{path} holds real data (type='data') without the " + "concealed=True blinding stamp — refusing to load an unblinded " + "data vector. Only the blinding/unblinding tooling may pass " + "allow_unblinded=True." + ) + return s diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index cb9c5a37..43ad2c59 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -40,7 +40,7 @@ def _base_sacc(nbins=1): # --------------------------------------------------------------------------- # def _roundtrip(s, tmp_path, name="rt"): path = tmp_path / f"{name}.sacc" - sio.save(s, str(path)) + sio.save(s, str(path), type="mock") return sio.load(str(path)) @@ -482,7 +482,7 @@ def test_end_to_end_one_file_layout(tmp_path): s, [(xi_c, _spd(len(xi_c), 1)), (co, _spd(len(co), 2)), (xi_f, fine_block)], ) - sio.save(s, str(tmp_path / f"{version}.sacc")) + sio.save(s, str(tmp_path / f"{version}.sacc"), type="mock") a = sio.load(str(tmp_path / f"{version}.sacc")) @@ -528,7 +528,7 @@ def test_one_file_layout_diagonal_fine_fallback(tmp_path): ) variances = np.concatenate([np.arange(1, 51) * 1e-12, np.arange(1, 51) * 2e-12]) sio.assemble_covariance(s, [(xi_c, _spd(len(xi_c), 1)), (xi_f, np.diag(variances))]) - sio.save(s, str(tmp_path / "vDIAG.sacc")) + sio.save(s, str(tmp_path / "vDIAG.sacc"), type="mock") a = sio.load(str(tmp_path / "vDIAG.sacc")) assert np.array_equal(np.diag(a.covariance.dense[np.ix_(xi_f, xi_f)]), variances) @@ -637,7 +637,183 @@ def test_tomographic_xi_covariance_one_contiguous_block(): # --------------------------------------------------------------------------- # -# 12. get_pseudo_cl window/cl column correspondence: window column j maps to +# 12. type stamping + fail-closed load (data/mock x concealed/not; escape +# hatch for the blinding/unblinding tooling) +# --------------------------------------------------------------------------- # +def _saved(tmp_path, name, *, type, concealed=None): + s = _base_sacc() + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + if concealed is not None: + s.metadata["concealed"] = concealed + path = str(tmp_path / f"{name}.sacc") + sio.save(s, path, type=type) + return path + + +def test_load_mock_unconcealed(tmp_path): + s = sio.load(_saved(tmp_path, "m0", type="mock")) + assert s.metadata["type"] == "mock" + + +def test_load_mock_concealed(tmp_path): + s = sio.load(_saved(tmp_path, "m1", type="mock", concealed=True)) + assert s.metadata["concealed"] + + +def test_load_data_concealed(tmp_path): + s = sio.load(_saved(tmp_path, "d1", type="data", concealed=True)) + assert s.metadata["type"] == "data" + + +def test_load_data_unconcealed_fails_closed(tmp_path): + path = _saved(tmp_path, "d0", type="data") + with pytest.raises(ValueError, match="unblinded"): + sio.load(path) + # concealed=False is as unblinded as no stamp at all + path_f = _saved(tmp_path, "d0f", type="data", concealed=False) + with pytest.raises(ValueError, match="unblinded"): + sio.load(path_f) + + +def test_load_data_unconcealed_escape_hatch(tmp_path): + s = sio.load(_saved(tmp_path, "d0h", type="data"), allow_unblinded=True) + assert s.metadata["type"] == "data" + + +def test_save_requires_valid_type(tmp_path): + s = _base_sacc() + with pytest.raises(TypeError): + sio.save(s, str(tmp_path / "x.sacc")) # type is required + with pytest.raises(ValueError, match="'data' or 'mock'"): + sio.save(s, str(tmp_path / "x.sacc"), type="simulation") + + +def test_save_refuses_type_restamp(tmp_path): + s = _base_sacc() + sio.save(s, str(tmp_path / "x.sacc"), type="mock") + with pytest.raises(ValueError, match="re-stamp"): + sio.save(s, str(tmp_path / "x.sacc"), type="data") + + +def test_load_requires_type_tag(tmp_path): + import sacc as sacc_lib + + s = _base_sacc() # never stamped + path = str(tmp_path / "untyped.sacc") + s.save_fits(path, overwrite=True) + with pytest.raises(KeyError): + sio.load(path) + assert sacc_lib.Sacc.load_fits(path) is not None # raw loader still works + + +# --------------------------------------------------------------------------- # +# 13. merge(): per-statistic files combine into one; shared tracers stored +# once; covariance block-diagonal (all-or-none); metadata union with +# loud conflicts. update_statistic(): value-only merge-back. +# --------------------------------------------------------------------------- # +def _xi_sacc(metadata=None): + s = sio.new_sacc({0: _nz(0)}, metadata=metadata) + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + return s + + +def _cosebi_sacc(metadata=None): + s = sio.new_sacc({0: _nz(0)}, metadata=metadata) + sio.add_cosebis( + s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) + ) + return s + + +def test_merge_per_statistic_files(tmp_path): + meta = {"version": "vM", "type": "mock"} + s_xi, s_co = _xi_sacc(meta), _cosebi_sacc(meta) + merged = sio.merge([s_xi, s_co]) + # shared tracers stored once; all points present, xi first + assert set(merged.tracers) == {"source_0", sio.PSF_TRACER} + assert len(merged.mean) == len(s_xi.mean) + len(s_co.mean) + assert np.array_equal(merged.mean, np.concatenate([s_xi.mean, s_co.mean])) + assert merged.metadata["version"] == "vM" and merged.metadata["type"] == "mock" + # inputs untouched + assert s_xi.metadata["version"] == "vM" + # readers work on the merged file after a round-trip + sio.save(merged, str(tmp_path / "vM.sacc"), type="mock") + merged_rt = sio.load(str(tmp_path / "vM.sacc")) + _, p, _ = sio.get_xi(merged_rt, (0, 0), grid="coarse") + assert np.array_equal(p, np.arange(6) * 1e-5) + _, E, _ = sio.get_cosebis(merged_rt, (0, 0)) + assert np.array_equal(E, np.arange(1, 6) * 1e-6) + + +def test_merge_covariance_block_diagonal(): + s_xi, s_co = _xi_sacc(), _cosebi_sacc() + cov_xi, cov_co = _spd(len(s_xi.mean), 1), _spd(len(s_co.mean), 2) + s_xi.add_covariance(cov_xi) + s_co.add_covariance(cov_co) + merged = sio.merge([s_xi, s_co]) + dense = merged.covariance.dense + n_xi = len(s_xi.mean) + assert np.array_equal(dense[:n_xi, :n_xi], cov_xi) + assert np.array_equal(dense[n_xi:, n_xi:], cov_co) + assert np.all(dense[:n_xi, n_xi:] == 0) + + +def test_merge_mixed_covariance_fails(): + s_xi, s_co = _xi_sacc(), _cosebi_sacc() + s_xi.add_covariance(_spd(len(s_xi.mean), 1)) # s_co has none + with pytest.raises(Exception): + sio.merge([s_xi, s_co]) + + +def test_merge_conflicting_metadata_fails(): + s_xi = _xi_sacc({"type": "data"}) + s_co = _cosebi_sacc({"type": "mock"}) + with pytest.raises(ValueError, match="conflicting metadata"): + sio.merge([s_xi, s_co]) + + +def test_update_statistic_values_only(): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + cov = [(xi, _spd(len(xi), 1)), (cl, _spd(len(cl), 2)), (co, _spd(len(co), 3))] + sio.assemble_covariance(s, cov) + dense_before = s.covariance.dense.copy() + mean_before = s.mean.copy() + + # extract -> shift (a stand-in for conceal) -> merge back + sub = sio.extract(s, data_type=sio.XI_PLUS, tracers=tr) + for point in sub.data: + point.value += 1e-4 + sio.update_statistic(s, sub) + + idx_p = s.indices(sio.XI_PLUS, tr) + assert np.allclose(s.mean[idx_p], mean_before[idx_p] + 1e-4) + untouched = np.setdiff1d(np.arange(len(s.mean)), idx_p) + assert np.array_equal(s.mean[untouched], mean_before[untouched]) + # covariance and insertion order untouched + assert np.array_equal(s.covariance.dense, dense_before) + + +def test_update_statistic_requires_unique_match(): + s = _xi_sacc() + sub = sio.extract(s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0")) + missing = sub.copy() + missing.data[0].tags["theta"] = 999.0 # matches nothing in s + with pytest.raises(ValueError, match="0 points"): + sio.update_statistic(s, missing) + + +# --------------------------------------------------------------------------- # +# 14. get_pseudo_cl window/cl column correspondence: window column j maps to # the returned ell_eff[j] (verified through window_ind tags). # --------------------------------------------------------------------------- # def test_pseudo_cl_window_column_correspondence(tmp_path): From 87b359878b2597b5a680ae7d6554df65e5b65f66 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 02:19:56 +0200 Subject: [PATCH 006/160] Address review: rename grid values to reporting/integration; clarify comments - grid tag values: coarse -> "reporting", fine -> "integration" (descriptive, not relative); tag kept because both grids share data type and tracer pair, so the tag is the sole disambiguator. Swept module docstrings and tests. - Module docstring now spells out why insertion order is load-bearing: covariance row/column i refers to the i-th inserted data point. - new_sacc: comment that psf_stars rides in the same tracer list as the NZ tracers only because a Sacc has one flat tracer namespace (bookkeeping, not physics). - source_name/_pair helpers kept (4 and 8 call sites) with one-line justifications of the conventions they centralize. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QWF72ofwJh6ekgnCt9Xx6C --- src/sp_validation/sacc_io.py | 50 ++++++++---- src/sp_validation/tests/test_sacc_io.py | 100 +++++++++++++----------- 2 files changed, 89 insertions(+), 61 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index dae6e8d0..4168e6c6 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -6,27 +6,32 @@ weak-lensing validation package. One file describes each catalogue version: - - ``{version}.sacc`` — NZ tracers, coarse ξ±, pseudo-Cℓ + - ``{version}.sacc`` — NZ tracers, reporting-grid ξ±, pseudo-Cℓ (EE/BB/EB) with bandpower windows, COSEBIs, pure E/B, ρ/τ PSF - diagnostics, and the fine-grid ξ± integration input for - COSEBIs / pure-EB (``grid='fine'`` tagged points). The + diagnostics, and the fine ξ± integration input for + COSEBIs / pure-EB (``grid='integration'`` tagged points). The covariance is assembled block-diagonally from the per-statistic covariances (zero cross-blocks): the analysis - blocks first, then a dense per-pair fine-ξ block (the + blocks first, then a dense per-pair integration-ξ block (the analytic integration-binning covariance when it exists — it feeds derived-statistic error propagation — or the TreeCorr ``varxip``/``varxim`` diagonal as degraded fallback). At the - production fine binning (1000 θ bins) a dense fine block is + production integration binning (1000 θ bins) a dense block is ~32 MB per pair; extreme convergence-check grids (10k bins) degrade to the diagonal fallback rather than forking the layout. - The covariance order is the point-insertion order (SACC preserves - it bitwise through FITS save/load). Writers below insert in the - canonical order — ξ+ then ξ−, Cℓ (ee, bb, eb), COSEBIs (all Eₙ - then all Bₙ), pure E/B (xip_E, xim_E, xip_B, xim_B, xip_amb, - xim_amb — matching ``b_modes._EB_KEYS``), ρ, then τ — but readers - never assume global order: they resolve indices through + Insertion order is load-bearing. A Sacc is a flat list of data + points in the order ``add_data_point`` was called, and row/column + ``i`` of the covariance refers to the ``i``-th inserted point — + there is no other linkage between a point and its covariance + entry. SACC preserves that order bitwise through FITS save/load, + so writers define the covariance layout by their insertion + sequence. Writers below insert in the canonical order — ξ+ then + ξ−, Cℓ (ee, bb, eb), COSEBIs (all Eₙ then all Bₙ), pure E/B + (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb — matching + ``b_modes._EB_KEYS``), ρ, then τ — but readers never assume + global order: they resolve indices through ``Sacc.indices(dtype, tracers, **tags)``. Tag filters are plain keyword arguments to ``indices`` / @@ -81,7 +86,11 @@ def source_name(i): - """SACC tracer name for source redshift bin ``i`` (0-based).""" + """SACC tracer name for source redshift bin ``i`` (0-based). + + Kept as the single definition of the ``source_{i}`` naming contract — + external consumers address tracers through it rather than the f-string. + """ return f"source_{i}" @@ -108,6 +117,10 @@ def new_sacc(nz, metadata=None): s = sacc.Sacc() for i, (z, nz_i) in items: s.add_tracer("NZ", source_name(i), np.asarray(z), np.asarray(nz_i)) + # The PSF star sample sits alongside the source bins because a Sacc has a + # single tracer namespace — every data point references tracers from one + # flat list. This is bookkeeping, not physics: psf_stars is a Misc tracer + # (no n(z)) that exists only so ρ/τ points have something to reference. s.add_tracer("Misc", PSF_TRACER) for key, value in (metadata or {}).items(): s.metadata[key] = value @@ -119,7 +132,8 @@ def _pair(bins): The pair is normalised to ``i <= j``: shear-shear statistics are symmetric in the tracer pair, and SACC stores each pair under one ordering, so - ``(1, 0)`` must address the same points as ``(0, 1)``. + ``(1, 0)`` must address the same points as ``(0, 1)``. Kept as a helper: + every ``add_*``/``get_*`` function relies on this normalisation. """ i, j = sorted(bins) return (source_name(i), source_name(j)) @@ -165,9 +179,11 @@ def add_xi( Angular separations (arcmin) — TreeCorr ``meanr``. xip, xim : array_like ξ+ and ξ− at ``theta``. - grid : {'coarse', 'fine'} - Distinguishes the analysis grid from the fine integration grid; - stored as the ``grid`` tag on every point. + grid : {'reporting', 'integration'} + Stored as the ``grid`` tag on every point. The reporting (analysis) + ξ± and the fine COSEBIs/pure-EB integration input share the same + data type and tracer pair, so this tag is the only thing that + disambiguates them in the file. theta_nom : array_like, optional Nominal bin centres — TreeCorr ``rnom`` — stored as ``theta_nom``. npairs, weight : array_like, optional @@ -559,7 +575,7 @@ def extract(s, data_type=None, tracers=None, **tag_filters): ``extract`` is the generic selection escape hatch, mirroring ``Sacc.keep_selection`` and addressing non-source tracers uniformly. **tag_filters - Tag filters (plain kwargs, e.g. ``grid='fine'``). + Tag filters (plain kwargs, e.g. ``grid='integration'``). Returns ------- diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 43ad2c59..ec574f83 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -66,20 +66,20 @@ def test_xi_roundtrip(tmp_path): theta, xip, xim, - grid="coarse", + grid="reporting", theta_nom=theta * 1.01, npairs=npairs, weight=weight, ) s2 = _roundtrip(s, tmp_path, "xi") - th, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + th, p, m = sio.get_xi(s2, (0, 0), grid="reporting") assert np.array_equal(th, theta) assert np.array_equal(p, xip) assert np.array_equal(m, xim) # extra tags survive - idx = s2.indices(sio.XI_PLUS, ("source_0", "source_0"), grid="coarse") + idx = s2.indices(sio.XI_PLUS, ("source_0", "source_0"), grid="reporting") tags = s2.data[idx[0]].tags - assert tags["grid"] == "coarse" + assert tags["grid"] == "reporting" assert set(tags) >= {"theta", "theta_nom", "npairs", "weight", "grid"} @@ -183,7 +183,7 @@ def _multi_statistic_sacc(): theta = _theta() s = _base_sacc() sio.add_xi( - s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) ell = np.array([30.0, 120.0, 210.0]) W = np.random.default_rng(0).uniform(size=(20, 3)) @@ -254,7 +254,7 @@ def test_assemble_covariance_selector_tuples(): def test_assemble_covariance_wrong_dimension(): s = _base_sacc() sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) idx = np.arange(len(s.mean)) with pytest.raises(ValueError, match="span"): @@ -264,7 +264,7 @@ def test_assemble_covariance_wrong_dimension(): def test_assemble_covariance_non_contiguous(): s = _base_sacc() sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) idx = np.array([0, 2, 4, 6, 8, 10, 1, 3]) # not contiguous/ascending with pytest.raises(ValueError, match="non-contiguous"): @@ -284,7 +284,7 @@ def test_assemble_covariance_missing_coverage(): def test_assemble_covariance_non_square(): s = _base_sacc() sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) idx = np.arange(len(s.mean)) with pytest.raises(ValueError, match="square"): @@ -309,11 +309,11 @@ def test_diagonal_covariance_roundtrip(tmp_path): xip, xim = np.arange(n) * 1e-5, np.arange(n) * 2e-5 varxip, varxim = np.arange(1, n + 1) * 1e-12, np.arange(1, n + 1) * 2e-12 s = _base_sacc() - sio.add_xi(s, (0, 0), theta, xip, xim, grid="fine") + sio.add_xi(s, (0, 0), theta, xip, xim, grid="integration") variances = np.concatenate([varxip, varxim]) # [xip; xim] order sio.add_diagonal_covariance(s, variances) assert type(s.covariance).__name__ == "DiagonalCovariance" - s2 = _roundtrip(s, tmp_path, "fine") + s2 = _roundtrip(s, tmp_path, "integration") assert type(s2.covariance).__name__ == "DiagonalCovariance" assert np.array_equal(np.diag(s2.covariance.dense), variances) @@ -346,14 +346,16 @@ def test_extract_tag_filter(): theta = _theta() s = _base_sacc() sio.add_xi( - s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" + ) + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 3e-5, np.arange(6) * 4e-5, grid="integration" ) - sio.add_xi(s, (0, 0), theta, np.arange(6) * 3e-5, np.arange(6) * 4e-5, grid="fine") sub = sio.extract( - s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0"), grid="fine" + s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0"), grid="integration" ) assert len(sub.mean) == len(theta) - assert set(sub.get_tag("grid", sio.XI_PLUS)) == {"fine"} + assert set(sub.get_tag("grid", sio.XI_PLUS)) == {"integration"} # --------------------------------------------------------------------------- # @@ -370,11 +372,11 @@ def test_tomographic_per_pair_selection(tmp_path): theta, np.arange(6) * (k + 1) * 1e-5, np.arange(6) * (k + 1) * 2e-5, - grid="coarse", + grid="reporting", ) s2 = _roundtrip(s, tmp_path, "tomo") for k, (i, j) in enumerate(pairs): - th, p, m = sio.get_xi(s2, (i, j), grid="coarse") + th, p, m = sio.get_xi(s2, (i, j), grid="reporting") assert np.array_equal(th, theta) assert np.array_equal(p, np.arange(6) * (k + 1) * 1e-5) assert np.array_equal(m, np.arange(6) * (k + 1) * 2e-5) @@ -390,7 +392,7 @@ def test_readers_on_mixed_file(tmp_path): theta = _theta() s = _base_sacc() sio.add_xi( - s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) ell = np.array([30.0, 120.0, 210.0]) W = np.random.default_rng(0).uniform(size=(20, 3)) @@ -424,7 +426,7 @@ def test_readers_on_mixed_file(tmp_path): ) s2 = _roundtrip(s, tmp_path, "mixed") - _, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + _, p, m = sio.get_xi(s2, (0, 0), grid="reporting") assert np.array_equal(p, np.arange(6) * 1e-5) and np.array_equal( m, np.arange(6) * 2e-5 ) @@ -454,26 +456,31 @@ def test_end_to_end_one_file_layout(tmp_path): # one file: analysis products first, fine-grid integration input last s = _base_sacc() sio.add_xi( - s, (0, 0), theta_c, np.arange(20) * 1e-5, np.arange(20) * 2e-5, grid="coarse" + s, (0, 0), theta_c, np.arange(20) * 1e-5, np.arange(20) * 2e-5, grid="reporting" ) sio.add_cosebis( s, (0, 0), np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7, (1.0, 100.0) ) sio.add_xi( - s, (0, 0), theta_f, np.arange(200) * 1e-5, np.arange(200) * 2e-5, grid="fine" + s, + (0, 0), + theta_f, + np.arange(200) * 1e-5, + np.arange(200) * 2e-5, + grid="integration", ) tr = ("source_0", "source_0") xi_c = np.concatenate( [ - s.indices(sio.XI_PLUS, tr, grid="coarse"), - s.indices(sio.XI_MINUS, tr, grid="coarse"), + s.indices(sio.XI_PLUS, tr, grid="reporting"), + s.indices(sio.XI_MINUS, tr, grid="reporting"), ] ) co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) xi_f = np.concatenate( [ - s.indices(sio.XI_PLUS, tr, grid="fine"), - s.indices(sio.XI_MINUS, tr, grid="fine"), + s.indices(sio.XI_PLUS, tr, grid="integration"), + s.indices(sio.XI_MINUS, tr, grid="integration"), ] ) # dense fine block (CosmoCov integration covariance in production) @@ -486,18 +493,18 @@ def test_end_to_end_one_file_layout(tmp_path): a = sio.load(str(tmp_path / f"{version}.sacc")) - th_c, p_c, _ = sio.get_xi(a, (0, 0), grid="coarse") + th_c, p_c, _ = sio.get_xi(a, (0, 0), grid="reporting") assert np.array_equal(th_c, theta_c) and np.array_equal(p_c, np.arange(20) * 1e-5) n, E, B = sio.get_cosebis(a, (0, 0)) assert np.array_equal(n, np.arange(1, 11)) assert a.covariance.dense.shape == (len(a.mean), len(a.mean)) - th_f, p_f, _ = sio.get_xi(a, (0, 0), grid="fine") + th_f, p_f, _ = sio.get_xi(a, (0, 0), grid="integration") assert np.array_equal(th_f, theta_f) and np.array_equal(p_f, np.arange(200) * 1e-5) # extract() of the fine selection pulls the aligned dense sub-covariance - fine = sio.extract(a, sio.XI_PLUS, tr, grid="fine") - idx_p = a.indices(sio.XI_PLUS, tr, grid="fine") + fine = sio.extract(a, sio.XI_PLUS, tr, grid="integration") + idx_p = a.indices(sio.XI_PLUS, tr, grid="integration") assert np.allclose(fine.covariance.dense, a.covariance.dense[np.ix_(idx_p, idx_p)]) # zero cross-blocks between analysis and fine points assert np.all(a.covariance.dense[np.ix_(xi_c, xi_f)] == 0) @@ -508,22 +515,27 @@ def test_one_file_layout_diagonal_fine_fallback(tmp_path): s = _base_sacc() theta_f = np.geomspace(0.1, 250.0, 50) sio.add_xi( - s, (0, 0), _theta(6), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), _theta(6), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) sio.add_xi( - s, (0, 0), theta_f, np.arange(50) * 1e-5, np.arange(50) * 2e-5, grid="fine" + s, + (0, 0), + theta_f, + np.arange(50) * 1e-5, + np.arange(50) * 2e-5, + grid="integration", ) tr = ("source_0", "source_0") xi_c = np.concatenate( [ - s.indices(sio.XI_PLUS, tr, grid="coarse"), - s.indices(sio.XI_MINUS, tr, grid="coarse"), + s.indices(sio.XI_PLUS, tr, grid="reporting"), + s.indices(sio.XI_MINUS, tr, grid="reporting"), ] ) xi_f = np.concatenate( [ - s.indices(sio.XI_PLUS, tr, grid="fine"), - s.indices(sio.XI_MINUS, tr, grid="fine"), + s.indices(sio.XI_PLUS, tr, grid="integration"), + s.indices(sio.XI_MINUS, tr, grid="integration"), ] ) variances = np.concatenate([np.arange(1, 51) * 1e-12, np.arange(1, 51) * 2e-12]) @@ -541,7 +553,7 @@ def test_add_xi_rejects_non_ascending_theta(): theta = _theta()[::-1] # descending with pytest.raises(ValueError, match="theta must be strictly ascending"): sio.add_xi( - s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) @@ -581,15 +593,15 @@ def test_bin_pair_normalisation(): theta = _theta() xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 s = _base_sacc(nbins=2) - sio.add_xi(s, (0, 1), theta, xip, xim, grid="coarse") - th01, p01, m01 = sio.get_xi(s, (0, 1), grid="coarse") - th10, p10, m10 = sio.get_xi(s, (1, 0), grid="coarse") # reversed order + sio.add_xi(s, (0, 1), theta, xip, xim, grid="reporting") + th01, p01, m01 = sio.get_xi(s, (0, 1), grid="reporting") + th10, p10, m10 = sio.get_xi(s, (1, 0), grid="reporting") # reversed order assert np.array_equal(th01, th10) assert np.array_equal(p01, p10) and np.array_equal(p10, xip) assert np.array_equal(m01, m10) and np.array_equal(m10, xim) # writing under (1, 0) lands in the same tracer pair, not a new one s2 = _base_sacc(nbins=2) - sio.add_xi(s2, (1, 0), theta, xip, xim, grid="coarse") + sio.add_xi(s2, (1, 0), theta, xip, xim, grid="reporting") assert len(s2.indices(sio.XI_PLUS, ("source_0", "source_1"))) == len(theta) @@ -611,7 +623,7 @@ def test_tomographic_xi_covariance_one_contiguous_block(): theta, np.arange(nth) * (k + 1) * 1e-5, np.arange(nth) * (k + 1) * 2e-5, - grid="coarse", + grid="reporting", ) # All ξ points as one contiguous block in insertion (pair-major) order. xi_idx = np.arange(len(s.mean)) @@ -631,7 +643,7 @@ def test_tomographic_xi_covariance_one_contiguous_block(): ) assert np.array_equal(sub.covariance.dense, cov[np.ix_(idx_p, idx_p)]) # readers stay covariance-aligned: get_xi returns in the same order - th, xip, _ = sio.get_xi(s, (i, j), grid="coarse") + th, xip, _ = sio.get_xi(s, (i, j), grid="reporting") assert np.array_equal(th, theta) assert np.array_equal(s.mean[idx_p], xip) @@ -643,7 +655,7 @@ def test_tomographic_xi_covariance_one_contiguous_block(): def _saved(tmp_path, name, *, type, concealed=None): s = _base_sacc() sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) if concealed is not None: s.metadata["concealed"] = concealed @@ -716,7 +728,7 @@ def test_load_requires_type_tag(tmp_path): def _xi_sacc(metadata=None): s = sio.new_sacc({0: _nz(0)}, metadata=metadata) sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) return s @@ -743,7 +755,7 @@ def test_merge_per_statistic_files(tmp_path): # readers work on the merged file after a round-trip sio.save(merged, str(tmp_path / "vM.sacc"), type="mock") merged_rt = sio.load(str(tmp_path / "vM.sacc")) - _, p, _ = sio.get_xi(merged_rt, (0, 0), grid="coarse") + _, p, _ = sio.get_xi(merged_rt, (0, 0), grid="reporting") assert np.array_equal(p, np.arange(6) * 1e-5) _, E, _ = sio.get_cosebis(merged_rt, (0, 0)) assert np.array_equal(E, np.arange(1, 6) * 1e-6) From 4489dd4c835ce25697838d75f2a21dd4f679458c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 02:27:53 +0200 Subject: [PATCH 007/160] Simplify sacc_io writers/readers and test builders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Factor the theta-tagged insertion loop into _add_theta_series (add_rho, add_tau, add_pure_eb); add_pure_eb zips PURE_TYPES.values() against its signature order, and PURE_KEYS is derived from PURE_TYPES instead of restating it. - Factor the (theta, plus, minus) read pattern into _get_pm (get_xi, get_rho, get_tau). - add_xi hoists the optional-tag None-filtering out of the point loop; extract and merge lose their throwaway mutable dicts; get_cosebis builds its scale-cut tags in one expression. - Tests: shared _add_xi default-ξ builder and _xi_block/_cl_block/ _cosebi_block canonical index-block helpers replace ~60 lines of copy-pasted setup; test_readers_on_mixed_file builds on _multi_statistic_sacc. Behaviour unchanged; 41/41 tests green in the container. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QWF72ofwJh6ekgnCt9Xx6C --- src/sp_validation/sacc_io.py | 103 ++++++++---------- src/sp_validation/tests/test_sacc_io.py | 138 ++++++++---------------- 2 files changed, 89 insertions(+), 152 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 4168e6c6..458c201d 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -74,10 +74,11 @@ "xip_amb": "galaxy_shear_xiPureAmb_plus", "xim_amb": "galaxy_shear_xiPureAmb_minus", } -# Insertion order of the six pure-EB blocks — matches b_modes._EB_KEYS, whose -# order is the [xip_E; xim_E; xip_B; xim_B; xip_amb; xim_amb] layout of the -# treecorr/MC pure-EB covariance (b_modes.calculate_eb_statistics, ~L392). -PURE_KEYS = ("xip_E", "xim_E", "xip_B", "xim_B", "xip_amb", "xim_amb") +# PURE_TYPES key order is the insertion order of the six pure-EB blocks — +# matches b_modes._EB_KEYS, whose order is the [xip_E; xim_E; xip_B; xim_B; +# xip_amb; xim_amb] layout of the treecorr/MC pure-EB covariance +# (b_modes.calculate_eb_statistics, ~L392). +PURE_KEYS = tuple(PURE_TYPES) RHO_PLUS = "psf_rho{k}_xi_plus" RHO_MINUS = "psf_rho{k}_xi_minus" @@ -155,6 +156,12 @@ def _check_ascending(name, values): ) +def _add_theta_series(s, dtype, tracers, theta, values): + """Insert one theta-tagged series, one point per (theta, value) pair.""" + for th, value in zip(theta, values): + s.add_data_point(dtype, tracers, float(value), theta=float(th)) + + def add_xi( s, bins, @@ -191,15 +198,15 @@ def add_xi( """ _check_ascending("theta", theta) tracers = _pair(bins) + optional = {"theta_nom": theta_nom, "npairs": npairs, "weight": weight} + extras = {key: arr for key, arr in optional.items() if arr is not None} for dtype, xi in ((XI_PLUS, xip), (XI_MINUS, xim)): for n, th in enumerate(theta): - tags = {"theta": float(th), "grid": grid} - if theta_nom is not None: - tags["theta_nom"] = float(theta_nom[n]) - if npairs is not None: - tags["npairs"] = float(npairs[n]) - if weight is not None: - tags["weight"] = float(weight[n]) + tags = { + "theta": float(th), + "grid": grid, + **{key: float(arr[n]) for key, arr in extras.items()}, + } s.add_data_point(dtype, tracers, float(xi[n]), **tags) @@ -293,18 +300,9 @@ def add_pure_eb(s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb): """ _check_ascending("theta", theta) tracers = _pair(bins) - values = { - "xip_E": xip_E, - "xim_E": xim_E, - "xip_B": xip_B, - "xim_B": xim_B, - "xip_amb": xip_amb, - "xim_amb": xim_amb, - } - for key in PURE_KEYS: - dtype, arr = PURE_TYPES[key], values[key] - for n, th in enumerate(theta): - s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + arrays = (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb) + for dtype, arr in zip(PURE_TYPES.values(), arrays): + _add_theta_series(s, dtype, tracers, theta, arr) def add_rho(s, k, theta, rho_p, rho_m): @@ -323,9 +321,8 @@ def add_rho(s, k, theta, rho_p, rho_m): """ _check_ascending("theta", theta) tracers = (PSF_TRACER, PSF_TRACER) - for dtype, arr in ((RHO_PLUS.format(k=k), rho_p), (RHO_MINUS.format(k=k), rho_m)): - for n, th in enumerate(theta): - s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + _add_theta_series(s, RHO_PLUS.format(k=k), tracers, theta, rho_p) + _add_theta_series(s, RHO_MINUS.format(k=k), tracers, theta, rho_m) def add_tau(s, bins, k, theta, tau_p, tau_m): @@ -347,9 +344,8 @@ def add_tau(s, bins, k, theta, tau_p, tau_m): """ _check_ascending("theta", theta) tracers = (source_name(bins[0]), PSF_TRACER) - for dtype, arr in ((TAU_PLUS.format(k=k), tau_p), (TAU_MINUS.format(k=k), tau_m)): - for n, th in enumerate(theta): - s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + _add_theta_series(s, TAU_PLUS.format(k=k), tracers, theta, tau_p) + _add_theta_series(s, TAU_MINUS.format(k=k), tracers, theta, tau_m) def assemble_covariance(s, blocks): @@ -453,16 +449,20 @@ def get_nz(s, i): return tracer.z, tracer.nz -def get_xi(s, bins, *, grid): - """Return ``(theta, xip, xim)`` for one tracer pair and grid.""" - tracers = _pair(bins) +def _get_pm(s, dtype_p, dtype_m, tracers, **tags): + """Return ``(theta, plus, minus)`` for a +/− data-type pair.""" return ( - _tag(s, XI_PLUS, tracers, "theta", grid=grid), - _mean(s, XI_PLUS, tracers, grid=grid), - _mean(s, XI_MINUS, tracers, grid=grid), + _tag(s, dtype_p, tracers, "theta", **tags), + _mean(s, dtype_p, tracers, **tags), + _mean(s, dtype_m, tracers, **tags), ) +def get_xi(s, bins, *, grid): + """Return ``(theta, xip, xim)`` for one tracer pair and grid.""" + return _get_pm(s, XI_PLUS, XI_MINUS, _pair(bins), grid=grid) + + def get_pseudo_cl(s, bins): """Return ``(ell_eff, cl_ee, cl_bb, cl_eb, window)`` for one tracer pair. @@ -491,11 +491,7 @@ def get_cosebis(s, bins, scale_cut=None): ``(theta_min, theta_max)`` to select when several cuts share the file. """ tracers = _pair(bins) - tags = ( - {"theta_min": float(scale_cut[0]), "theta_max": float(scale_cut[1])} - if scale_cut is not None - else {} - ) + tags = dict(zip(("theta_min", "theta_max"), map(float, scale_cut or ()))) modes = _tag(s, COSEBI_EE, tracers, "n", **tags) return ( modes.astype(int), @@ -518,23 +514,13 @@ def get_pure_eb(s, bins): def get_rho(s, k): """Return ``(theta, rho_p, rho_m)`` for ρ index ``k``.""" tracers = (PSF_TRACER, PSF_TRACER) - dt_p, dt_m = RHO_PLUS.format(k=k), RHO_MINUS.format(k=k) - return ( - _tag(s, dt_p, tracers, "theta"), - _mean(s, dt_p, tracers), - _mean(s, dt_m, tracers), - ) + return _get_pm(s, RHO_PLUS.format(k=k), RHO_MINUS.format(k=k), tracers) def get_tau(s, bins, k): """Return ``(theta, tau_p, tau_m)`` for τ index ``k`` and source bin.""" tracers = (source_name(bins[0]), PSF_TRACER) - dt_p, dt_m = TAU_PLUS.format(k=k), TAU_MINUS.format(k=k) - return ( - _tag(s, dt_p, tracers, "theta"), - _mean(s, dt_p, tracers), - _mean(s, dt_m, tracers), - ) + return _get_pm(s, TAU_PLUS.format(k=k), TAU_MINUS.format(k=k), tracers) def _mean(s, data_type, tracers, **tag_filters): @@ -583,11 +569,8 @@ def extract(s, data_type=None, tracers=None, **tag_filters): New Sacc holding only the selected points. """ sub = s.copy() - selection = {} - if tracers is not None: - selection["tracers"] = tuple(tracers) - selection.update(tag_filters) - sub.keep_selection(data_type, **selection) + tracer_filter = {"tracers": tuple(tracers)} if tracers is not None else {} + sub.keep_selection(data_type, **tracer_filter, **tag_filters) return sub @@ -631,11 +614,9 @@ def merge(saccs): metadata[key] = value # Strip metadata before concatenating (the library "resolves" clashing # keys by renaming them), then restore the validated union. - stripped = [] - for s in saccs: - s = s.copy() + stripped = [s.copy() for s in saccs] + for s in stripped: s.metadata.clear() - stripped.append(s) seen, shared = set(), set() # tracers appearing in more than one input for s in saccs: shared |= seen & set(s.tracers) diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index ec574f83..69713758 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -35,6 +35,30 @@ def _base_sacc(nbins=1): return sio.new_sacc({i: _nz(i) for i in range(nbins)}) +def _add_xi(s, bins=(0, 0), scale=1.0, grid="reporting"): + """Write the default synthetic ξ± block; returns ``(xip, xim)``.""" + xip, xim = np.arange(6) * scale * 1e-5, np.arange(6) * scale * 2e-5 + sio.add_xi(s, bins, _theta(), xip, xim, grid=grid) + return xip, xim + + +# Canonical per-statistic index blocks (concatenated in insertion order). +def _xi_block(s, tr, **tags): + return np.concatenate( + [s.indices(sio.XI_PLUS, tr, **tags), s.indices(sio.XI_MINUS, tr, **tags)] + ) + + +def _cl_block(s, tr): + return np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + + +def _cosebi_block(s, tr): + return np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + + # --------------------------------------------------------------------------- # # 1. Per-writer round-trip (arrays / tags / windows / NZ bitwise) # --------------------------------------------------------------------------- # @@ -180,11 +204,8 @@ def test_tau_roundtrip(tmp_path): # --------------------------------------------------------------------------- # def _multi_statistic_sacc(): """Sacc with ξ+/ξ−, Cℓ (ee/bb/eb) and COSEBIs, ready for a covariance.""" - theta = _theta() s = _base_sacc() - sio.add_xi( - s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + _add_xi(s) ell = np.array([30.0, 120.0, 210.0]) W = np.random.default_rng(0).uniform(size=(20, 3)) sio.add_pseudo_cl( @@ -207,11 +228,7 @@ def test_assemble_covariance_alignment(tmp_path): s = _multi_statistic_sacc() tr = ("source_0", "source_0") # Block selectors in canonical (insertion) order. - xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) - cl = np.concatenate( - [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] - ) - co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + xi, cl, co = _xi_block(s, tr), _cl_block(s, tr), _cosebi_block(s, tr) cov_xi, cov_cl, cov_co = _spd(len(xi), 1), _spd(len(cl), 2), _spd(len(co), 3) sio.assemble_covariance(s, [(xi, cov_xi), (cl, cov_cl), (co, cov_co)]) s2 = _roundtrip(s, tmp_path, "cov") @@ -231,10 +248,7 @@ def test_assemble_covariance_selector_tuples(): """Blocks addressed by (data_type, tracers) tuples, not raw indices.""" s = _multi_statistic_sacc() tr = ("source_0", "source_0") - xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) - cl = np.concatenate( - [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] - ) + xi, cl = _xi_block(s, tr), _cl_block(s, tr) sio.assemble_covariance( s, [ @@ -253,9 +267,7 @@ def test_assemble_covariance_selector_tuples(): # --------------------------------------------------------------------------- # def test_assemble_covariance_wrong_dimension(): s = _base_sacc() - sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + _add_xi(s) idx = np.arange(len(s.mean)) with pytest.raises(ValueError, match="span"): sio.assemble_covariance(s, [(idx, _spd(len(idx) - 1, 1))]) @@ -263,9 +275,7 @@ def test_assemble_covariance_wrong_dimension(): def test_assemble_covariance_non_contiguous(): s = _base_sacc() - sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + _add_xi(s) idx = np.array([0, 2, 4, 6, 8, 10, 1, 3]) # not contiguous/ascending with pytest.raises(ValueError, match="non-contiguous"): sio.assemble_covariance(s, [(idx, _spd(len(idx), 1))]) @@ -274,7 +284,7 @@ def test_assemble_covariance_non_contiguous(): def test_assemble_covariance_missing_coverage(): s = _multi_statistic_sacc() tr = ("source_0", "source_0") - xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + xi = _xi_block(s, tr) with pytest.raises(ValueError, match="tile"): sio.assemble_covariance( s, [(xi, _spd(len(xi), 1))] @@ -283,9 +293,7 @@ def test_assemble_covariance_missing_coverage(): def test_assemble_covariance_non_square(): s = _base_sacc() - sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + _add_xi(s) idx = np.arange(len(s.mean)) with pytest.raises(ValueError, match="square"): sio.assemble_covariance(s, [(idx, np.ones((len(idx), len(idx) - 1)))]) @@ -294,7 +302,7 @@ def test_assemble_covariance_non_square(): def test_assemble_covariance_overlap(): s = _multi_statistic_sacc() tr = ("source_0", "source_0") - xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + xi = _xi_block(s, tr) # second block starts before the first ended -> gap/overlap error with pytest.raises(ValueError, match="tile|gap|overlap"): sio.assemble_covariance(s, [(xi, _spd(len(xi), 1)), (xi, _spd(len(xi), 2))]) @@ -324,11 +332,7 @@ def test_diagonal_covariance_roundtrip(tmp_path): def test_extract_subblock_and_original_untouched(): s = _multi_statistic_sacc() tr = ("source_0", "source_0") - xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) - cl = np.concatenate( - [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] - ) - co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + xi, cl, co = _xi_block(s, tr), _cl_block(s, tr), _cosebi_block(s, tr) cov_co = _spd(len(co), 3) sio.assemble_covariance( s, [(xi, _spd(len(xi), 1)), (cl, _spd(len(cl), 2)), (co, cov_co)] @@ -345,12 +349,8 @@ def test_extract_subblock_and_original_untouched(): def test_extract_tag_filter(): theta = _theta() s = _base_sacc() - sio.add_xi( - s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) - sio.add_xi( - s, (0, 0), theta, np.arange(6) * 3e-5, np.arange(6) * 4e-5, grid="integration" - ) + _add_xi(s) + _add_xi(s, scale=3.0, grid="integration") sub = sio.extract( s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0"), grid="integration" ) @@ -390,25 +390,11 @@ def test_tomographic_per_pair_selection(tmp_path): # --------------------------------------------------------------------------- # def test_readers_on_mixed_file(tmp_path): theta = _theta() - s = _base_sacc() - sio.add_xi( - s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + # ξ+Cℓ+COSEBIs from the shared builder; the assertions below restate its + # values (ell, W match _multi_statistic_sacc). + s = _multi_statistic_sacc() ell = np.array([30.0, 120.0, 210.0]) W = np.random.default_rng(0).uniform(size=(20, 3)) - sio.add_pseudo_cl( - s, - (0, 0), - ell, - np.arange(3) * 1e-9, - np.arange(3) * 2e-9, - np.arange(3) * 3e-9, - window_ells=np.arange(2, 22).astype(float), - window_weights=W, - ) - sio.add_cosebis( - s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) - ) pure = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} sio.add_pure_eb(s, (0, 0), theta, **pure) for k in range(6): @@ -470,19 +456,9 @@ def test_end_to_end_one_file_layout(tmp_path): grid="integration", ) tr = ("source_0", "source_0") - xi_c = np.concatenate( - [ - s.indices(sio.XI_PLUS, tr, grid="reporting"), - s.indices(sio.XI_MINUS, tr, grid="reporting"), - ] - ) - co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) - xi_f = np.concatenate( - [ - s.indices(sio.XI_PLUS, tr, grid="integration"), - s.indices(sio.XI_MINUS, tr, grid="integration"), - ] - ) + xi_c = _xi_block(s, tr, grid="reporting") + co = _cosebi_block(s, tr) + xi_f = _xi_block(s, tr, grid="integration") # dense fine block (CosmoCov integration covariance in production) fine_block = _spd(len(xi_f), 3) sio.assemble_covariance( @@ -514,9 +490,7 @@ def test_one_file_layout_diagonal_fine_fallback(tmp_path): """No CosmoCov covariance: the fine block is np.diag(varxip/varxim).""" s = _base_sacc() theta_f = np.geomspace(0.1, 250.0, 50) - sio.add_xi( - s, (0, 0), _theta(6), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + _add_xi(s) sio.add_xi( s, (0, 0), @@ -526,18 +500,8 @@ def test_one_file_layout_diagonal_fine_fallback(tmp_path): grid="integration", ) tr = ("source_0", "source_0") - xi_c = np.concatenate( - [ - s.indices(sio.XI_PLUS, tr, grid="reporting"), - s.indices(sio.XI_MINUS, tr, grid="reporting"), - ] - ) - xi_f = np.concatenate( - [ - s.indices(sio.XI_PLUS, tr, grid="integration"), - s.indices(sio.XI_MINUS, tr, grid="integration"), - ] - ) + xi_c = _xi_block(s, tr, grid="reporting") + xi_f = _xi_block(s, tr, grid="integration") variances = np.concatenate([np.arange(1, 51) * 1e-12, np.arange(1, 51) * 2e-12]) sio.assemble_covariance(s, [(xi_c, _spd(len(xi_c), 1)), (xi_f, np.diag(variances))]) sio.save(s, str(tmp_path / "vDIAG.sacc"), type="mock") @@ -654,9 +618,7 @@ def test_tomographic_xi_covariance_one_contiguous_block(): # --------------------------------------------------------------------------- # def _saved(tmp_path, name, *, type, concealed=None): s = _base_sacc() - sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + _add_xi(s) if concealed is not None: s.metadata["concealed"] = concealed path = str(tmp_path / f"{name}.sacc") @@ -727,9 +689,7 @@ def test_load_requires_type_tag(tmp_path): # --------------------------------------------------------------------------- # def _xi_sacc(metadata=None): s = sio.new_sacc({0: _nz(0)}, metadata=metadata) - sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + _add_xi(s) return s @@ -791,11 +751,7 @@ def test_merge_conflicting_metadata_fails(): def test_update_statistic_values_only(): s = _multi_statistic_sacc() tr = ("source_0", "source_0") - xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) - cl = np.concatenate( - [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] - ) - co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + xi, cl, co = _xi_block(s, tr), _cl_block(s, tr), _cosebi_block(s, tr) cov = [(xi, _spd(len(xi), 1)), (cl, _spd(len(cl), 2)), (co, _spd(len(co), 3))] sio.assemble_covariance(s, cov) dense_before = s.covariance.dense.copy() From 8bd38171d6ad99743760aa51663d26a6b4ae7478 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 11:05:26 +0200 Subject: [PATCH 008/160] fix(sacc_io): fail loud on unmatched selections; merge/update guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial review (pass 2) findings: - Sacc.indices returns an EMPTY array (warning only) on an unmatched selection; every reader, extract(), and covariance-block selector now funnels through a shared _indices guard that raises instead — a typo'd tag or non-bitwise-identical float scale cut can no longer propagate empty arrays downstream (assemble_covariance previously died with an opaque IndexError on the same path). - get_cosebis(scale_cut=None) on a file carrying several scale cuts silently concatenated them; now raises and asks for an explicit cut. - update_statistic let two sub points claim the same target point (last-write-wins); now raises. - merge: the library keeps the FIRST input's tracer on a name clash with no equality check; shared tracers are now verified identical (z, nz) across inputs before concatenation. 7 regression tests; 48 total. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WzUt7VbtXwr2SCHUdiQTyt --- src/sp_validation/sacc_io.py | 58 +++++++++++++++++++++++-- src/sp_validation/tests/test_sacc_io.py | 58 +++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 3 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 458c201d..21872d0e 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -418,7 +418,7 @@ def _resolve_indices(s, selector): return np.asarray(selector, dtype=int) data_type, tracers = selector[0], selector[1] tags = selector[2] if len(selector) == 3 else {} - return np.asarray(s.indices(data_type, tuple(tracers), **tags), dtype=int) + return _indices(s, data_type, tuple(tracers), **tags) def add_diagonal_covariance(s, variances): @@ -491,6 +491,16 @@ def get_cosebis(s, bins, scale_cut=None): ``(theta_min, theta_max)`` to select when several cuts share the file. """ tracers = _pair(bins) + if scale_cut is None: + cuts = { + (s.data[i].tags["theta_min"], s.data[i].tags["theta_max"]) + for i in _indices(s, COSEBI_EE, tracers) + } + if len(cuts) > 1: + raise ValueError( + f"several COSEBIs scale cuts share the file ({sorted(cuts)}) " + "— pass scale_cut=(theta_min, theta_max) to pick one" + ) tags = dict(zip(("theta_min", "theta_max"), map(float, scale_cut or ()))) modes = _tag(s, COSEBI_EE, tracers, "n", **tags) return ( @@ -523,6 +533,24 @@ def get_tau(s, bins, k): return _get_pm(s, TAU_PLUS.format(k=k), TAU_MINUS.format(k=k), tracers) +def _indices(s, data_type, tracers, **tag_filters): + """``Sacc.indices`` that fails loud instead of selecting nothing. + + ``Sacc.indices`` returns an *empty array* (warning only) when a selection + matches no point — e.g. a typo'd tag value, or a float tag filter that is + not bitwise-identical to the stored one. Every reader here funnels through + this guard so an unmatched selection raises instead of propagating empty + arrays downstream. + """ + idx = np.asarray(s.indices(data_type, tracers, **tag_filters), dtype=int) + if len(idx) == 0: + raise ValueError( + f"selection matched no points: ({data_type}, {tracers}, " + f"{tag_filters}) — note float tags match by exact equality" + ) + return idx + + def _mean(s, data_type, tracers, **tag_filters): """Mean values for a selection, in ``s.indices`` (insertion) order. @@ -531,12 +559,12 @@ def _mean(s, data_type, tracers, **tag_filters): the covariance for any file (and ascending for canonically-written files, which the writers enforce). """ - return s.mean[s.indices(data_type, tracers, **tag_filters)] + return s.mean[_indices(s, data_type, tracers, **tag_filters)] def _tag(s, data_type, tracers, tag, **tag_filters): """Values of ``tag`` for a selection, in insertion order.""" - idx = s.indices(data_type, tracers, **tag_filters) + idx = _indices(s, data_type, tracers, **tag_filters) return np.array([s.data[i].tags[tag] for i in idx]) @@ -571,6 +599,10 @@ def extract(s, data_type=None, tracers=None, **tag_filters): sub = s.copy() tracer_filter = {"tracers": tuple(tracers)} if tracers is not None else {} sub.keep_selection(data_type, **tracer_filter, **tag_filters) + if len(sub.mean) == 0: + raise ValueError( + f"extract selected no points: ({data_type}, {tracers}, {tag_filters})" + ) return sub @@ -622,6 +654,19 @@ def merge(saccs): shared |= seen & set(s.tracers) seen |= set(s.tracers) same_tracers = sorted(shared) + # The library keeps the FIRST input's tracer on a name clash with no + # equality check — verify shared tracers really are the same object. + for name in same_tracers: + first, *rest = [s.tracers[name] for s in saccs if name in s.tracers] + for other in rest: + if type(other) is not type(first) or not all( + np.array_equal(getattr(first, a, None), getattr(other, a, None)) + for a in ("z", "nz") + ): + raise ValueError( + f"shared tracer {name!r} differs across merge inputs — " + "the merged file would silently keep the first" + ) merged = sacc.concatenate_data_sets(*stripped, same_tracers=same_tracers) for key, value in metadata.items(): merged.metadata[key] = value @@ -647,6 +692,7 @@ def update_statistic(s, sub): sub : sacc.Sacc The replacement block, e.g. ``extract(s, ...)`` after concealment. """ + claimed = set() for point in sub.data: idx = s.indices(point.data_type, point.tracers, **point.tags) if len(idx) != 1: @@ -655,6 +701,12 @@ def update_statistic(s, sub): f"({point.data_type}, {point.tracers}, {point.tags}) — need " "exactly one" ) + if idx[0] in claimed: + raise ValueError( + f"update_statistic: two sub points match the same target " + f"point ({point.data_type}, {point.tracers}, {point.tags})" + ) + claimed.add(idx[0]) s.data[idx[0]].value = point.value diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 69713758..11788fbb 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -780,6 +780,64 @@ def test_update_statistic_requires_unique_match(): sio.update_statistic(s, missing) +def test_update_statistic_rejects_duplicate_sub_points(): + s = _xi_sacc() + sub = sio.extract(s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0")) + sub.data[1].tags = dict(sub.data[0].tags) # two sub points -> one target + with pytest.raises(ValueError, match="same target"): + sio.update_statistic(s, sub) + + +def test_merge_rejects_divergent_shared_tracer(): + s_xi, s_co = _xi_sacc(), _cosebi_sacc() + s_co.tracers["source_0"].nz = s_co.tracers["source_0"].nz * 2.0 + with pytest.raises(ValueError, match="source_0.*differs"): + sio.merge([s_xi, s_co]) + + +# --------------------------------------------------------------------------- # +# 15. Unmatched selections fail loud — no silent-empty arrays anywhere. +# --------------------------------------------------------------------------- # +def test_readers_raise_on_unmatched_selection(): + s = _base_sacc() + _add_xi(s, grid="reporting") + with pytest.raises(ValueError, match="matched no points"): + sio.get_xi(s, (0, 0), grid="integration") # only 'reporting' exists + with pytest.raises(ValueError, match="matched no points"): + sio.get_xi(s, (0, 1), grid="reporting") # no such pair + + +def test_get_cosebis_raises_on_unmatched_scale_cut(): + s = _cosebi_sacc() # written with scale cut (1.0, 100.0) + with pytest.raises(ValueError, match="matched no points"): + sio.get_cosebis(s, (0, 0), scale_cut=(2.0, 50.0)) + + +def test_get_cosebis_rejects_ambiguous_multi_cut_file(): + s = _cosebi_sacc() + sio.add_cosebis( + s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (2.0, 50.0) + ) + with pytest.raises(ValueError, match="several COSEBIs scale cuts"): + sio.get_cosebis(s, (0, 0)) + n, En, _ = sio.get_cosebis(s, (0, 0), scale_cut=(2.0, 50.0)) + assert len(En) == 5 # explicit cut disambiguates + + +def test_extract_raises_on_empty_selection(): + s = _xi_sacc() + with pytest.raises(ValueError, match="no points"): + sio.extract(s, grid="integration") # only 'reporting' exists + + +def test_assemble_covariance_rejects_empty_selector(): + s = _xi_sacc() + with pytest.raises(ValueError, match="matched no points"): + sio.assemble_covariance( + s, [((sio.XI_PLUS, ("source_9", "source_9")), np.eye(6))] + ) + + # --------------------------------------------------------------------------- # # 14. get_pseudo_cl window/cl column correspondence: window column j maps to # the returned ell_eff[j] (verified through window_ind tags). From c9a4f025b4ca07b1fc2addcc68a1f11adb8b7776 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 11:40:12 +0200 Subject: [PATCH 009/160] refactor(blinding): stack on PR-2's canonical sacc_io; sweep grid vocabulary Rebuild the per-part-at-birth blinding stack on top of feat/sacc-2-sacc-io. sacc_io.py is now PR-2's canonical module verbatim + a single appended gather(): the terminal assembly delegates its tracer/point/covariance assembly to PR-2's merge() and adds only the blind-custody call (assert_consistent_blind) and shared-stamp write. The fail-closed load gate lives in sacc_io.load() per the PRD ("sacc_io fails closed on load"); the blinding tooling uses allow_unblinded=True explicitly where it must read a not-yet-blinded real vector (blind_part). save() now carries the required type= (inherited from each part's provenance). Grid vocabulary swept coarse->reporting, fine->integration across blinding.py, blinding_theory.py, b_modes.py, blind_data_vector.py, and the blinding tests. _extract_block/_set_values stay index-based (a code comment says why): Smokescreen's ConcealDataVector aligns theory_fn output to the sub-SACC mean by row position, so the block must be carved/written by contiguous integer index, not by PR-2's tag-matching extract()/update_statistic(). Escrow/custody/CAMB<->CCL machinery is preserved exactly. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WzUt7VbtXwr2SCHUdiQTyt --- scripts/blind_data_vector.py | 173 +++ src/sp_validation/b_modes.py | 73 ++ src/sp_validation/blinding.py | 819 ++++++++++++ src/sp_validation/blinding_theory.py | 424 +++++++ src/sp_validation/sacc_io.py | 47 + src/sp_validation/tests/test_blinding.py | 1110 +++++++++++++++++ .../tests/test_camb_ccl_crosscheck.py | 174 +++ 7 files changed, 2820 insertions(+) create mode 100644 scripts/blind_data_vector.py create mode 100644 src/sp_validation/blinding.py create mode 100644 src/sp_validation/blinding_theory.py create mode 100644 src/sp_validation/tests/test_blinding.py create mode 100644 src/sp_validation/tests/test_camb_ccl_crosscheck.py diff --git a/scripts/blind_data_vector.py b/scripts/blind_data_vector.py new file mode 100644 index 00000000..9b41109d --- /dev/null +++ b/scripts/blind_data_vector.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 + +"""Script blind_data_vector.py + +Per-part data-vector blinding with :mod:`sp_validation.blinding` +(Smokescreen-fork concealment, hash-commitment custody). + +``blind-init`` runs once per catalogue version: it draws an OS-entropy seed +(never printed, never written in plaintext), publishes a repo-committable +``commitment.json`` (``sha256(seed)`` + config digest), and encrypts the seed +into a Fernet bundle. ``blind-part`` blinds one intermediate part SACC +(reporting ξ±, integration ξ±, or pseudo-Cℓ) under that fixed state, escrows +the true vector into a per-part encrypted bundle beside the blinded output, +and deletes the plaintext part. ``unblind`` verifies both commitment hashes +and restores a true part (bit-for-bit when the part's escrow bundle is beside +it); it also works on the assembled ``{version}.sacc`` (integration rows +selected by the ``grid`` tag). ``verify`` is a cheap, seedless check that a +blinded file matches a commitment. + +:Authors: Cail Daley + +Examples +-------- +Once per catalogue version:: + + blind_data_vector.py blind-init blinded/ + +Per intermediate part, at birth:: + + blind_data_vector.py blind-part parts/xi_integration.fits --blind-dir blinded/ + +Unblind one part:: + + blind_data_vector.py unblind parts/xi_integration_blinded.fits \\ + --blind-dir blinded/ -o parts/xi_integration.fits + +Verify:: + + blind_data_vector.py verify parts/xi_integration_blinded.fits \\ + blinded/commitment.json +""" + +import argparse +import json +import pathlib +import sys + +from sp_validation import blinding, sacc_io + + +def _config_from_args(args): + """A :class:`blinding.BlindingConfig` from optional CLI overrides.""" + overrides = {} + if args.s8_half_width is not None: + overrides["s8_half_width"] = args.s8_half_width + if args.omega_m_half_width is not None: + overrides["omega_m_half_width"] = args.omega_m_half_width + return blinding.BlindingConfig.from_overrides(overrides) + + +def _blind_init(args): + config = _config_from_args(args) + blind_dir = pathlib.Path(args.blind_dir) + blind_dir.mkdir(parents=True, exist_ok=True) + # Refuse before drawing anything: a blind is a one-shot custody event and + # silently overwriting a previous blind's state would destroy the record + # tying that blind to its seed. + clashes = [ + p + for p in blinding.init_paths(str(blind_dir)).values() + if pathlib.Path(p).exists() + ] + if clashes: + raise SystemExit( + "refusing to overwrite existing blind state:\n " + + "\n ".join(clashes) + + "\nPick a fresh --blind-dir (never overwrite a blind)." + ) + blinding.blind_init(str(blind_dir), config=config, label=args.label) + print( + "Commit the commitment JSON to the repo; keep the bundle + key safe " + "and separated (colocation in the blind dir is not at-rest protection)." + ) + + +def _blind_part(args): + blinding.blind_part( + args.part, + args.blind_dir, + config=_config_from_args(args), + keep_input=args.keep_input, + ) + + +def _unblind(args): + blinding.unblind_part( + args.blinded, + args.blind_dir, + args.output, + config=_config_from_args(args), + ) + + +def _verify(args): + """Seedless check: blinded-file metadata ↔ commitment JSON.""" + s = sacc_io.load(args.blinded) + with open(args.commitment, encoding="utf-8") as f: + commitment = json.load(f) + problems = [] + if not s.metadata.get("concealed"): + problems.append("file is not marked concealed") + if s.metadata.get("blind_commitment") != commitment["seed_sha256"]: + problems.append("blind_commitment does not match the committed sha256(seed)") + if s.metadata.get("blind_config_digest") != commitment["config_digest"]: + problems.append("blind_config_digest does not match the committed digest") + if "seed_smokescreen" in s.metadata: + problems.append("PLAINTEXT SEED LEAKED into file metadata (seed_smokescreen)") + if problems: + raise SystemExit("verification FAILED:\n " + "\n ".join(problems)) + print( + f"OK: {args.blinded} matches {args.commitment} " + f"(blind {s.metadata.get('blind')!r})" + ) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[1]) + sub = parser.add_subparsers(dest="mode", required=True) + + for name in ("blind-init", "blind-part", "unblind"): + p = sub.add_parser(name) + p.add_argument("--s8-half-width", type=float, default=None) + p.add_argument("--omega-m-half-width", type=float, default=None) + if name == "blind-init": + p.add_argument( + "blind_dir", + help="directory for the blind's fixed state (commitment + " + "encrypted seed bundle)", + ) + p.add_argument("--label", default="A", help="blind label (default A)") + p.set_defaults(func=_blind_init) + elif name == "blind-part": + p.add_argument("part", help="intermediate part SACC file to blind") + p.add_argument( + "--blind-dir", required=True, help="blind-init state directory" + ) + p.add_argument( + "--keep-input", + action="store_true", + help="retain the plaintext input part (default: delete it " + "after blinding — the true vector is escrowed beside the " + "blinded output)", + ) + p.set_defaults(func=_blind_part) + else: + p.add_argument("blinded", help="blinded part (or assembled) SACC file") + p.add_argument( + "--blind-dir", required=True, help="blind-init state directory" + ) + p.add_argument("-o", "--output", required=True, help="output SACC path") + p.set_defaults(func=_unblind) + + p = sub.add_parser("verify") + p.add_argument("blinded", help="blinded SACC file") + p.add_argument("commitment", help="commitment JSON") + p.set_defaults(func=_verify) + + args = parser.parse_args(argv) + args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index 13f4c2e9..97251c3f 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -255,6 +255,79 @@ def pure_EB(corrs): return results +def cosebis_from_xi(theta, xip, xim, nmodes, scale_cut=None): + """COSEBIs (Eₙ, Bₙ) from ξ± arrays through the pipeline kernel (values only). + + The values-only seam of :func:`calculate_cosebis`, for callers holding + ξ± arrays rather than a TreeCorr ``GGCorrelation`` — e.g. deriving + born-blinded COSEBIs from a blinded integration-ξ± SACC part. Calls the same + ``cosmo_numba`` kernel (``COSEBIS.cosebis_from_xipm``) directly on the + values; the covariance/χ² machinery stays with :func:`calculate_cosebis`. + + ``scale_cut`` follows the :func:`sacc_io.add_cosebis` writer contract: + ``(theta_min, theta_max)`` are min/max of the *retained* bin centres + after the pipeline's ``scale_cut_to_bins``. The cut is contiguous in an + ascending grid, so selecting ``theta_min ≤ θ ≤ theta_max`` inclusively + reproduces exactly the retained set, and the kernel is built on that + set's min/max support and fed only the retained ξ± — bit-matching + :func:`calculate_cosebis`'s ``theta_cut``/``xip_cut``/``xim_cut`` path. + Identical inputs ⇒ identical numbers. + """ + from cosmo_numba.B_modes.cosebis import COSEBIS + + theta, xip, xim = (np.asarray(a) for a in (theta, xip, xim)) + tmin, tmax = scale_cut if scale_cut is not None else (theta.min(), theta.max()) + cut = (theta >= tmin) & (theta <= tmax) + theta_cut, xip_cut, xim_cut = theta[cut], xip[cut], xim[cut] + cosebis = COSEBIS( + theta_min=np.min(theta_cut), + theta_max=np.max(theta_cut), + N_max=nmodes, + precision=120, + ) + En, Bn = cosebis.cosebis_from_xipm(theta_cut, xip_cut, xim_cut, parallel=True) + return np.asarray(En), np.asarray(Bn) + + +def pure_eb_from_xi( + theta_report, xip_report, xim_report, theta_int, xip_int, xim_int, tmin, tmax +): + """Pure-E/B correlation functions from ξ± arrays through the pipeline kernel. + + The values-only seam of :func:`calculate_pure_eb_correlation`, for + callers holding ξ± arrays rather than TreeCorr correlations — e.g. + deriving born-blinded pure-E/B from blinded SACC parts. Calls the same + ``cosmo_numba`` kernel (``get_pure_EB_modes``) directly on the values. + The reporting grid must be a strict sub-range of the integration grid; + ``tmin``/``tmax`` are the reporting correlation's TreeCorr *bin edges* + (``gg.left_edges[0]`` / ``gg.right_edges[-1]``) — the pipeline's + convention, carried on SACC files by ``sacc_io.add_pure_eb``. A + reporting point coinciding with the integration boundary is degenerate + (no interior support) and comes back NaN, exactly as + :func:`calculate_pure_eb_correlation` returns it — never a spurious + finite value. + + Returns + ------- + dict + Keyed by ``_EB_KEYS`` (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb). + """ + from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes + + modes = get_pure_EB_modes( + theta=np.asarray(theta_report), + xip=np.asarray(xip_report), + xim=np.asarray(xim_report), + theta_int=np.asarray(theta_int), + xip_int=np.asarray(xip_int), + xim_int=np.asarray(xim_int), + tmin=tmin, + tmax=tmax, + parallel=True, + ) + return dict(zip(_EB_KEYS, (np.asarray(m) for m in modes))) + + def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): """ Calculate COSEBIs modes from a correlation function for multiple scale cuts. diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py new file mode 100644 index 00000000..972fb704 --- /dev/null +++ b/src/sp_validation/blinding.py @@ -0,0 +1,819 @@ +"""Blinding — conceal each intermediate data product behind a hidden cosmology. + +:Name: blinding.py + +:Description: Smokescreen blinding wiring, per part, at birth. Each blindable + intermediate SACC product — reporting ξ±, integration ξ±, pseudo-Cℓ — is shifted, + the moment the pipeline computes it, by a difference of theory vectors + between the fiducial cosmology and a *hidden* cosmology drawn inside a + fixed amplitude envelope, so no one can read S8 off the data until the + collaboration agrees to unblind (Muir et al. 2019: + ``d → d + t(hidden) − t(fiducial)``). Only blinded parts persist on disk. + + **The fork is the concealment engine.** The hidden cosmology is drawn by + the ``UNIONS-WL/Smokescreen`` fork's own CCL-native, per-key-independent + draw; each part goes through its ``ConcealDataVector(fiducial_params, + shifts_dict, sacc_data, *, seed, theory_fn)`` entry point. This module + supplies the two sp_validation-specific pieces: the three ``theory_fn`` + backends matching each part's row layout (reporting ξ±, integration ξ±, pseudo-Cℓ) + and the SACC handling around the concealed vectors. + + **Envelope calibration.** The blinding intent is an amplitude smear of a + chosen (S8, Ωm) box. The fork draws in CCL-native primitives, so + :meth:`BlindingConfig.shifts_dict` maps the intended box to + ``{sigma8, Omega_c}`` half-widths evaluated at the fiducial — + ``σ8 = S8/√(Ωm/0.3)``, ``Ω_c = Ωm − Ω_b − Ω_ν``. The same seed yields + the same hidden cosmology across all part passes (the fork's draw depends + only on ``(key, seed)``), so the blinded parts are mutually consistent by + construction. + + **Derived statistics are born blinded.** COSEBIs and pure-E/B are never + touched by this module: the pipeline's own estimators + (:mod:`sp_validation.b_modes`) run in their normal downstream place on + the already-blinded integration ξ± part, so their outputs are born blinded. + Covariance and ρ/τ PSF diagnostics are never blinded — blinding hides + the vector, not the uncertainty, and the shift is pure E-mode so the + B-mode null tests stay honest under the blind. + + **Custody: hash commitment, no keyholder.** :func:`blind_init` runs once + per catalogue version: it draws an OS-entropy seed, publishes + ``sha256(seed)`` plus a canonical config digest as a repo-committable + ``commitment.json``, and encrypts the seed into a Fernet bundle + (``smokescreen.encryption``) — the plaintext seed is never written. + Each :func:`blind_part` call reads that fixed state, conceals one part, + escrows the part's true vector into its own encrypted bundle beside the + blinded output, and deletes the plaintext part. Terminal assembly + (:func:`sp_validation.sacc_io.gather`) calls + :func:`assert_consistent_blind` to fail closed unless every blindable + part carries the identical ``blind_commitment``. :func:`unblind_part` + verifies both hashes against the commitment *before* subtracting + anything, then restores the true part. +""" + +import dataclasses +import hashlib +import json +import os +import secrets +import warnings + +import numpy as np + +from . import sacc_io +from .blinding_theory import TheoryConfig, cl_ee, xi_ccl, xi_ell_grid + + +# --------------------------------------------------------------------------- # +# Configuration surface — the blinding envelope +# --------------------------------------------------------------------------- # +@dataclasses.dataclass(frozen=True) +class BlindingConfig: + """Blinding envelope and fiducial. + + The hidden cosmology is drawn (by the fork) uniformly and independently + per key inside ``shifts_dict()``'s half-widths about the fiducial. The + half-widths are the deliberate, configurable size of the blind — config, + not code; the group may resize the envelope. ``theory`` carries the + fiducial :class:`TheoryConfig` whose defaults *are* the blinding + fiducial. + """ + + s8_half_width: float = 0.075 + omega_m_half_width: float = 0.1 + theory: TheoryConfig = dataclasses.field(default_factory=TheoryConfig) + + def shifts_dict(self): + """The (S8, Ωm) envelope as CCL-native ``{sigma8, Omega_c}`` half-widths. + + Evaluated at the fiducial: a ΔS8 half-width maps to + ``ΔS8/√(Ωm_fid/0.3)`` in σ8 (at fixed Ωm), and a ΔΩm half-width maps + one-to-one to Ω_c (Ω_b and Ω_ν are fixed). Exact enough for a + blinding smear — the target is a characteristic amplitude, not a + precise (S8, Ωm) posterior. The fork draws each key independently as + ``U(fid − h, fid + h)``. + """ + return { + "sigma8": self.s8_half_width / np.sqrt(self.theory.Omega_m / 0.3), + "Omega_c": self.omega_m_half_width, + } + + def config_digest(self): + """sha256 of a canonical serialization of the full blinding config. + + Binds the envelope half-widths, the complete fiducial + :class:`TheoryConfig` (cosmology + the two ``halofit_version`` + tokens + IA fields) into one digest: JSON with sorted keys over the + full ordered field set. Every ``float``-declared field is coerced + through :func:`float` before serialization, so the digest depends on + the numeric *value*, not on whether a config wrote ``-1`` or ``-1.0`` + — an int-vs-float literal difference (routine when configs come from + YAML/CLI/humans) can no longer split one physical cosmology into two + digests and deny a legitimate unblind. Python's ``json`` then emits + each float via its shortest round-trip ``repr``, so two runs of the + same config produce byte-identical digests. Checked (with + ``sha256(seed)``) at unblind, so a wrong envelope or a mismatched + P(k) recipe cannot silently subtract a wrong shift. + """ + payload = { + "s8_half_width": float(self.s8_half_width), + "omega_m_half_width": float(self.omega_m_half_width), + "theory": { + f.name: ( + float(getattr(self.theory, f.name)) + if f.type in (float, "float") + else getattr(self.theory, f.name) + ) + for f in dataclasses.fields(self.theory) + }, + } + return hashlib.sha256( + json.dumps(payload, sort_keys=True).encode("utf-8") + ).hexdigest() + + @classmethod + def from_overrides(cls, overrides): + """Build from a mapping of field overrides (fail loud on unknown keys). + + ``theory`` may be given as a :class:`TheoryConfig` or a mapping of + TheoryConfig overrides, mirroring :meth:`TheoryConfig.from_overrides`. + """ + by_name = {f.name: f for f in dataclasses.fields(cls)} + unknown = set(overrides) - set(by_name) + if unknown: + raise ValueError( + f"unknown BlindingConfig fields {sorted(unknown)}; " + f"valid fields are {sorted(by_name)}" + ) + overrides = { + name: (float(v) if by_name[name].type in (float, "float") else v) + for name, v in overrides.items() + } + theory = overrides.get("theory") + if theory is not None and not isinstance(theory, TheoryConfig): + overrides["theory"] = TheoryConfig.from_overrides(dict(theory)) + return cls(**overrides) + + +# --------------------------------------------------------------------------- # +# Custody primitives +# --------------------------------------------------------------------------- # +def seed_commitment(seed): + """Public commitment for a seed: its sha256 hex digest. + + Safe to publish and commit to the repo — it ties a blinded file to its + blind without revealing the seed, and lets unblind refuse a wrong seed. + """ + return hashlib.sha256(seed.encode("utf-8")).hexdigest() + + +def hidden_params(seed, config): + """The hidden CCL parameter point the fork realizes for ``(seed, config)``. + + Re-runs the fork's own draw (``smokescreen.param_shifts.draw_param_shifts`` + — per-key-independent, local RNG) on the calibrated envelope and overlays + the deltas on the fiducial, exactly as ``ConcealDataVector`` does + internally. Deterministic: same ``(seed, config)`` ⇒ same hidden point, + forever — the reproducibility contract unblinding relies on, and what + makes every part share one hidden cosmology under one seed. + """ + from smokescreen.param_shifts import draw_param_shifts + + deltas = draw_param_shifts(config.shifts_dict(), seed) + params = dict(config.theory.ccl_params()) + for key, delta in deltas.items(): + params[key] += delta + return params + + +# --------------------------------------------------------------------------- # +# Block discovery on a SACC (a standalone part, or the assembled file) +# --------------------------------------------------------------------------- # +def source_bins(s): + """Sorted source-bin indices present in ``s`` (from ``source_i`` tracers).""" + return sorted( + int(name.split("_", 1)[1]) for name in s.tracers if name.startswith("source_") + ) + + +def xi_pairs(s, grid): + """Unordered source-bin pairs ``(i ≤ j)`` carrying ξ+ on ``grid``.""" + bins = source_bins(s) + return [ + (i, j) + for a, i in enumerate(bins) + for j in bins[a:] + if len(s.indices(sacc_io.XI_PLUS, sacc_io._pair((i, j)), grid=grid)) + ] + + +def cl_pairs(s): + """Source-bin pairs ``(i ≤ j)`` carrying pseudo-Cℓ_EE.""" + bins = source_bins(s) + return [ + (i, j) + for a, i in enumerate(bins) + for j in bins[a:] + if len(s.indices(sacc_io.CL_EE, sacc_io._pair((i, j)))) + ] + + +def _xi_indices(s, grid): + """Row indices of the ξ± block on ``grid`` (ascending).""" + return np.sort( + np.concatenate( + [ + s.indices(sacc_io.XI_PLUS, grid=grid), + s.indices(sacc_io.XI_MINUS, grid=grid), + ] + ) + ).astype(int) + + +def _cl_ee_indices(s): + """Row indices of the pseudo-Cℓ_EE block (ascending). + + Only EE: a pure E-mode cosmology shift leaves BB and EB identically + zero, so those blocks are never extracted, never concealed. + """ + return np.sort(s.indices(sacc_io.CL_EE)).astype(int) + + +def _pair_nz(s, i, j): + """The two per-bin n(z) for pair ``(i, j)`` as ``((z_i, n_i), (z_j, n_j))``.""" + z_i, n_i = sacc_io.get_nz(s, i) + z_j, n_j = sacc_io.get_nz(s, j) + return (np.asarray(z_i), np.asarray(n_i)), (np.asarray(z_j), np.asarray(n_j)) + + +# --------------------------------------------------------------------------- # +# The three theory backends — callables aligned to a sub-SACC block's rows +# --------------------------------------------------------------------------- # +def xi_theory_fn(block, theory, grid): + """``theory_fn`` for a ξ± sub-SACC block (reporting or integration grid). + + Reads each bin's n(z) directly from the block's own tracers and lays the + output out to match the block's SACC rows element-for-element: for every + pair, ξ± is computed at that pair's stored θ (the ``theta`` tag, + arcmin) and scattered to the rows ``block.indices`` reports — the + block's own row order, never an assumed pairing. IA enters from + ``theory``'s NLA fields, identically at every parameter point. + """ + pairs = xi_pairs(block, grid) + layout = [] + for i, j in pairs: + tr = sacc_io._pair((i, j)) + idx_p = block.indices(sacc_io.XI_PLUS, tr, grid=grid) + idx_m = block.indices(sacc_io.XI_MINUS, tr, grid=grid) + theta = sacc_io._tag(block, sacc_io.XI_PLUS, tr, "theta", grid=grid) + layout.append(((i, j), idx_p, idx_m, np.asarray(theta, dtype=float))) + ell = xi_ell_grid() + + def theory_fn(params): + out = np.full(len(block.mean), np.nan) + for (i, j), idx_p, idx_m, theta in layout: + nz_i, nz_j = _pair_nz(block, i, j) + xip, xim = xi_ccl(params, theory, nz_i, nz_j, theta, ell) + out[idx_p] = xip + out[idx_m] = xim + return out + + return theory_fn + + +def cl_theory_fn(block, theory): + """``theory_fn`` for the pseudo-Cℓ_EE sub-SACC block. + + Per pair: theory Cℓ_EE on the stored ``BandpowerWindows`` support, + binned by the same window matrix the measurement used (``W @ Cℓ_EE``), + scattered to the block's own rows. The concealing factor the fork forms + from this is ``W @ ΔCℓ_EE`` — the shift lands in the measured + bandpowers; ΔBB = ΔEB ≡ 0 by construction (pure E-mode shift), so those + blocks are simply not part of this backend's rows. + """ + layout = [] + for i, j in cl_pairs(block): + tr = sacc_io._pair((i, j)) + idx = block.indices(sacc_io.CL_EE, tr) + window = block.get_bandpower_windows(idx) + layout.append( + ( + (i, j), + np.asarray(idx), + np.asarray(window.values, dtype=float), # (n_ell,) + np.asarray(window.weight, dtype=float), # (n_ell, n_bp) + ) + ) + + def theory_fn(params): + out = np.full(len(block.mean), np.nan) + for (i, j), idx, w_ell, w_mat in layout: + nz_i, nz_j = _pair_nz(block, i, j) + out[idx] = w_mat.T @ cl_ee(params, theory, nz_i, nz_j, w_ell) + return out + + return theory_fn + + +# --------------------------------------------------------------------------- # +# Extract → conceal → merge, per block +# --------------------------------------------------------------------------- # +def _blindable_blocks(s): + """The blindable blocks of a SACC as ``(name, indices, factory)``. + + Works identically on a standalone part (which carries exactly one block) + and on the assembled one-file product (whose integration rows are selected by + the ``grid`` tag — the layout contract's per-block tag selection). + ``indices`` are each block's recorded row indices (ascending, so the + extracted sub-SACC preserves row order); ``factory`` builds the matching + ``theory_fn`` from the extracted sub-SACC. Blocks absent from the file + are simply not listed. + """ + blocks = [] + for grid in ("reporting", "integration"): + idx = _xi_indices(s, grid) + if len(idx): + blocks.append( + ( + f"{grid} ξ±", + idx, + lambda sub, theory, grid=grid: xi_theory_fn(sub, theory, grid), + ) + ) + idx = _cl_ee_indices(s) + if len(idx): + blocks.append(("pseudo-Cℓ_EE", idx, cl_theory_fn)) + return blocks + + +def _extract_block(s, indices): + """Extract rows ``indices`` (ascending) into a sub-SACC, order preserved. + + Deliberately index-based rather than ``sacc_io.extract`` / + ``update_statistic`` (which select and merge by ``(data_type, tracers, + tags)``): Smokescreen's ``ConcealDataVector`` aligns its ``theory_fn`` + output to the sub-SACC's ``mean`` element-for-element by row position, so + the block must be carved and written back by contiguous integer index, not + by tag-matching. This is the one place the row-index path is load-bearing. + """ + sub = s.copy() + sub.keep_indices(np.asarray(indices, dtype=int)) + return sub + + +def _concealing_factor(s, indices, factory, config, seed): + """The fork-computed additive concealing factor for one block. + + Extracts the block into a sub-SACC containing exactly the rows the + ``theory_fn`` spans (the fork's length guard enforces the agreement), + drives ``ConcealDataVector`` with the fiducial point, the calibrated + envelope, and the block's ``theory_fn``, and returns the factor + ``t(hidden) − t(fiducial)`` aligned to ``indices``. + """ + from smokescreen import ConcealDataVector + + sub = _extract_block(s, indices) + smoke = ConcealDataVector( + config.theory.ccl_params(), + config.shifts_dict(), + sub, + seed=seed, + theory_fn=factory(sub, config.theory), + ) + smoke.calculate_concealing_factor(factor_type="add") + concealed = smoke.apply_concealing_to_likelihood_datavec() + return np.asarray(concealed, dtype=float) - np.asarray(sub.mean, dtype=float) + + +def _set_values(s, indices, values): + """Overwrite ``s.data[i].value`` for ``indices`` with ``values`` (aligned).""" + for i, v in zip(indices, values): + s.data[int(i)].value = float(v) + + +def _concealed(s): + """Whether ``s`` is already a blinded file (its ``concealed`` mark is set).""" + return bool(s.metadata.get("concealed")) + + +def blind_sacc(part, seed, config=None, label="A", log=print): + """Return a blinded copy of a part SACC (covariance and tags untouched). + + Per blindable block present (a standalone part carries exactly one — + reporting ξ±, integration ξ±, or pseudo-Cℓ_EE): extract the block into a sub-SACC, + conceal through the fork, write the shifted values back at their + recorded indices (row order preserved). Provenance is stamped and any + leaked seed key stripped. A file with no blindable block (e.g. a ρ/τ + diagnostic part) is refused loudly — it should never see a blind call. + """ + config = config or BlindingConfig() + if _concealed(part): + raise ValueError("already concealed — unblind first") + blocks = _blindable_blocks(part) + if not blocks: + raise ValueError( + "no blindable block (reporting/integration ξ± or pseudo-Cℓ_EE) in this SACC " + "— ρ/τ diagnostic parts are never blinded" + ) + + blinded = part.copy() + for name, indices, factory in blocks: + factor = _concealing_factor(part, indices, factory, config, seed) + _set_values(blinded, indices, np.asarray(blinded.mean)[indices] + factor) + log(f"[blind] {name}: shifted {len(indices)} points via Smokescreen fork") + + _stamp_provenance(blinded, seed_commitment(seed), label, config.config_digest()) + return blinded + + +def unblind_sacc(blinded, seed, config=None, log=print): + """Recover the true part SACC from a blinded one + the revealed ``seed``. + + Verifies ``sha256(seed)`` and the config digest against the stamped + metadata (loud failure on either mismatch — verification precedes + subtraction), recomputes each block's shift from the seed through the + same backends, and subtracts it. Works on a standalone part or on the + assembled file (integration rows selected by the ``grid`` tag). Derived + statistics, if present (assembled file), are *not* recomputed here — the + pipeline's own estimators re-derive them from the unblinded integration ξ±. + """ + config = config or BlindingConfig() + if not _concealed(blinded): + raise ValueError("file is not concealed — nothing to unblind") + if seed_commitment(seed) != blinded.metadata["blind_commitment"]: + raise ValueError( + "seed does not match blind_commitment — refusing to unblind " + "(a wrong seed would silently produce a wrong data vector)" + ) + if config.config_digest() != blinded.metadata["blind_config_digest"]: + raise ValueError( + "blinding config does not match blind_config_digest — refusing to " + "unblind (this config would subtract a different shift than was " + "added)" + ) + + hidden = hidden_params(seed, config) + fiducial = config.theory.ccl_params() + part = blinded.copy() + for name, indices, factory in _blindable_blocks(blinded): + theory = factory(_extract_block(blinded, indices), config.theory) + factor = theory(hidden) - theory(fiducial) + _set_values(part, indices, np.asarray(part.mean)[indices] - factor) + log(f"[unblind] {name}: subtracted {len(indices)} shifts") + + for key in ("concealed", "blind", "blind_commitment", "blind_config_digest"): + part.metadata.pop(key, None) + return part + + +def _stamp_provenance(s, commitment, label, config_digest): + """Stamp the blind's public provenance; strip any leaked seed. + + ``blind_commitment`` (sha256 of the seed) ties the file to its blind + without revealing it; ``blind_config_digest`` pins the envelope + + fiducial the shift was drawn against; ``concealed``/``blind`` mark the + file. ``seed_smokescreen`` — the raw seed Smokescreen's own writer would + stamp — is popped defensively: the seed must never ride a kept file. + """ + s.metadata.pop("seed_smokescreen", None) + s.metadata["concealed"] = True + s.metadata["blind"] = label + s.metadata["blind_commitment"] = commitment + s.metadata["blind_config_digest"] = config_digest + + +# --------------------------------------------------------------------------- # +# Assembly-time custody: one blind across all parts +# --------------------------------------------------------------------------- # +def assert_consistent_blind(parts): + """Assert every blindable part shares one blind; return the shared stamp. + + Custody logic for the terminal :func:`sp_validation.sacc_io.gather`: a + part is *blindable* if it carries a blindable block (ξ± or pseudo-Cℓ_EE); + ρ/τ diagnostic and covariance-only parts are exempt. Fails closed — + ``ValueError`` — if blinded and plaintext blindable parts are mixed, or + if two parts carry different ``blind_commitment``/``blind_config_digest`` + (they were blinded under different seeds or configs and must never be + combined). The consistency key is ``(commitment, digest)`` only — the + ``blind`` *label* is informational provenance, not custody state, so + parts blinded under one seed+config but tagged with different ``--label`` + values assemble cleanly (a distinct warning is logged, not a failure). + + **Unconcealed parts must be declared mocks** (PRD §4, "Mocks vs data"): + when no blindable part is concealed, every blindable part's metadata must + carry ``type == "mock"`` — the tag :mod:`sp_validation.sacc_io` stamps + when the data vector is computed. An unconcealed ``type == "data"`` part, + or one missing the tag, fails assembly closed: skipping the blind can + never silently expose real data. A concealed+plaintext mix fails closed + regardless of ``type`` (see above). A fully-mock plaintext assembly + returns ``None``. + + Returns + ------- + dict or None + The shared blind metadata (``concealed``, ``blind``, + ``blind_commitment``, ``blind_config_digest``) for the gather to + stamp on the assembled file, or ``None`` when nothing is blinded. + """ + blindable = [p for p in parts if _blindable_blocks(p)] + concealed = [p for p in blindable if _concealed(p)] + if not concealed: + # `.get` is deliberate here: a missing `type` tag must count as + # not-a-mock and fail closed, not KeyError with less context. + exposed = sorted( + {str(p.metadata.get("type", "")) for p in blindable} - {"mock"} + ) + if exposed: + raise ValueError( + f"unconcealed blindable parts with type {exposed} in assembly " + "— only parts declared `type: mock` may assemble without a " + "blind (an unconcealed data part exposes the real vector)" + ) + return None + if len(concealed) != len(blindable): + raise ValueError( + f"blinded and plaintext blindable parts mixed in one assembly " + f"({len(concealed)} of {len(blindable)} blinded) — refusing to " + "combine (a plaintext part beside blinded ones leaks the shift)" + ) + # Custody state is (commitment, digest) only — the label is provenance. + stamps = { + (p.metadata["blind_commitment"], p.metadata["blind_config_digest"]) + for p in concealed + } + if len(stamps) != 1: + raise ValueError( + "parts carry different blind commitments — they were blinded " + "under different seeds or configs and must never be combined: " + + "; ".join(f"({c[:12]}…, {d[:12]}…)" for c, d in stamps) + ) + ((commitment, digest),) = stamps + labels = sorted({p.metadata["blind"] for p in concealed}) + if len(labels) != 1: + warnings.warn( + f"blindable parts share one blind (commitment {commitment[:12]}…, " + f"config {digest[:12]}…) but carry different labels {labels} — " + "assembling anyway; the label is provenance, not custody state. " + f"Stamping the assembled file with label {labels[0]!r}." + ) + return { + "concealed": True, + "blind": labels[0], + "blind_commitment": commitment, + "blind_config_digest": digest, + } + + +# --------------------------------------------------------------------------- # +# File-level custody: blind-init / blind-part / unblind +# --------------------------------------------------------------------------- # +def init_paths(blind_dir): + """The fixed custody state written by :func:`blind_init` in ``blind_dir``.""" + return { + "commitment": os.path.join(blind_dir, "commitment.json"), + "bundle": os.path.join(blind_dir, "blind_seed.encrpt"), + "key": os.path.join(blind_dir, "blind_seed.key"), + } + + +def part_paths(part_path): + """Blinded-output and escrow-bundle paths beside a part file.""" + stem, ext = os.path.splitext(part_path) + return { + "blinded": f"{stem}_blinded{ext or '.fits'}", + "escrow": f"{stem}_escrow.encrpt", + "escrow_key": f"{stem}_escrow.key", + } + + +def blind_init(blind_dir, config=None, label="A", log=print): + """Fix the blind for one catalogue version: seed, commitment, seed bundle. + + Runs once per catalogue version: + + 1. Draw an OS-entropy seed (never written in plaintext, never returned). + 2. Write ``commitment.json`` (repo-committable): ``sha256(seed)`` + the + canonical config digest + the blind label. + 3. Encrypt the seed into a Fernet bundle (``smokescreen.encryption``); + the temporary plaintext is deleted by the encryptor. + + These outputs are the blind's fixed state; every :func:`blind_part` and + :func:`unblind_part` call reads them. + + Custody caveat: the bundle and its Fernet key land in the *same* + ``blind_dir``. Fernet is only as protective as the key's separation — + anyone with both files can decrypt the seed. Keep the key out-of-band; + colocation is convenience, not at-rest protection. + + Returns + ------- + dict + Paths written: ``commitment``, ``bundle``, ``key``. + """ + config = config or BlindingConfig() + paths = init_paths(blind_dir) + for path in paths.values(): + if os.path.exists(path): + raise FileExistsError( + f"refusing to overwrite existing blind state {path} — a blind " + "is a one-shot custody event; choose another directory" + ) + + seed = secrets.token_hex(16) + commitment = { + "label": label, + "seed_sha256": seed_commitment(seed), + "config_digest": config.config_digest(), + } + with open(paths["commitment"], "w", encoding="utf-8") as f: + json.dump(commitment, f, indent=2, sort_keys=True) + _write_encrypted_json(paths["bundle"], {"label": label, "seed": seed}) + + log(f"[blind-init] commitment (repo-committable): {paths['commitment']}") + log(f"[blind-init] encrypted seed bundle + key: {paths['bundle']}, {paths['key']}") + log( + "[blind-init] custody: keep the bundle key out-of-band from the bundle " + "(colocation in the blind dir is not at-rest protection)" + ) + return paths + + +def _read_seed(blind_dir, config): + """Decrypt the seed bundle and verify it against the commitment. + + Both ``sha256(seed)`` and the config digest are checked before the seed + is handed to any caller — a tampered bundle or a drifted config fails + loud here, whether the caller is about to blind or to unblind. + + Returns + ------- + tuple + ``(seed, commitment_dict)``. + """ + paths = init_paths(blind_dir) + bundle = _read_encrypted_json(paths["bundle"], paths["key"]) + with open(paths["commitment"], encoding="utf-8") as f: + commitment = json.load(f) + if seed_commitment(bundle["seed"]) != commitment["seed_sha256"]: + raise ValueError( + "bundle seed does not match the committed sha256(seed) — refusing " + "to proceed" + ) + if config.config_digest() != commitment["config_digest"]: + raise ValueError( + "blinding config does not match the committed config digest — " + "refusing to proceed (a wrong envelope or P(k) recipe would " + "silently produce a wrong shift)" + ) + return bundle["seed"], commitment + + +def blind_part(part_path, blind_dir, config=None, keep_input=False, log=print): + """Blind one intermediate part SACC at birth, under the fixed blind state. + + Reads the encrypted seed and ``commitment.json`` written by + :func:`blind_init` (verifying both hashes), conceals the part through its + matching backend (:func:`blind_sacc`), writes the blinded part beside the + input with the part's true vector escrowed into a per-part Fernet bundle, + and deletes the plaintext part — only the blinded part persists on disk. + Each part's escrow is self-contained: restoring it needs only that + part's bundle plus the seed bundle; corruption of one bundle loses one + part, not all. Pass ``keep_input=True`` to retain the plaintext part + (and own the custody implication). + + Returns + ------- + dict + Paths written: ``blinded``, ``escrow``, ``escrow_key``. + """ + config = config or BlindingConfig() + seed, commitment = _read_seed(blind_dir, config) + paths = part_paths(part_path) + for path in paths.values(): + if os.path.exists(path): + raise FileExistsError( + f"refusing to overwrite existing blind output {path} — a blind " + "is a one-shot custody event" + ) + + # The plaintext part is real, unblinded data — the blinding step is the one + # legitimate reader of the true vector, so it passes the load escape hatch. + part = sacc_io.load(part_path, allow_unblinded=True) + blinded = blind_sacc(part, seed, config=config, label=commitment["label"], log=log) + + _write_encrypted_json( + paths["escrow"], + { + "label": commitment["label"], + "seed_sha256": commitment["seed_sha256"], + "true_mean": np.asarray(part.mean, dtype=float).tolist(), + }, + ) + # The blinded file inherits the part's provenance (data vs mock); it also + # carries concealed=True (stamped by blind_sacc), so it loads without the + # escape hatch. + sacc_io.save(blinded, paths["blinded"], type=part.metadata["type"]) + if not keep_input: + os.remove(part_path) + log(f"[blind-part] deleted plaintext part {part_path}") + else: + log(f"[blind-part] plaintext part RETAINED at {part_path} (keep_input=True)") + log(f"[blind-part] wrote {paths['blinded']} (escrow beside it)") + return paths + + +def unblind_part(blinded_path, blind_dir, out_path, config=None, log=print): + """Unblind one blinded part (or the assembled file), verifying first. + + Decrypts the seed bundle and verifies both ``sha256(seed)`` and the + config digest against ``commitment.json`` (fail closed on either — + verification precedes subtraction), recomputes the part's shift from the + seed and subtracts it (:func:`unblind_sacc`). **The seed-subtracted + vector is the authority** — the seed plus its commitment is the custody + root of trust, and the returned vector is what the seed math produced. + + When the part's escrow bundle exists beside the blinded file it serves + two subordinate roles, and only after its stored ``seed_sha256`` is + verified to match the same commitment (so an escrow from a different + blind can never be trusted): (1) a *tighter equality check* — the seed + subtraction and the escrowed truth must agree to ``1e-6`` relative or the + unblind fails closed; (2) *ulp-residue removal* — float add-then-subtract + leaves ~ulp residue against the pre-blind truth, and once the escrow is + bound to this commitment its exact value clears that residue so the + restore is bit-for-bit. The escrow is never the source of correctness: + if it disagrees materially the guard raises, and a mismatched or + unbound escrow never determines the output. On the assembled file — no + escrow — the subtraction stands alone, with integration rows selected by the + ``grid`` tag. + """ + config = config or BlindingConfig() + seed, commitment = _read_seed(blind_dir, config) + blinded = sacc_io.load(blinded_path) + part = unblind_sacc(blinded, seed, config=config, log=log) + + stem, ext = os.path.splitext(blinded_path) + unblinded_stem = os.path.join( + os.path.dirname(stem), os.path.basename(stem).replace("_blinded", "") + ) + escrow = part_paths(unblinded_stem + ext) + if os.path.exists(escrow["escrow"]): + bundle = _read_encrypted_json(escrow["escrow"], escrow["escrow_key"]) + if bundle.get("seed_sha256") != commitment["seed_sha256"]: + raise ValueError( + "escrow bundle beside the blinded file was written under a " + "different seed than the commitment — refusing to trust it " + "(the seed subtraction is authoritative; this escrow is not " + "bound to this blind)" + ) + true_mean = np.asarray(bundle["true_mean"], dtype=float) + recovered = np.asarray(part.mean, dtype=float) + residual = np.nanmax( + np.abs(recovered - true_mean) / (np.abs(true_mean) + 1e-30) + ) + if residual > 1e-6: + raise ValueError( + f"unblinded vector disagrees with the escrowed true vector " + f"(max rel {residual:.2e}) — wrong escrow for this part?" + ) + # Seed-bound escrow: clear the add-then-subtract ulp residue so the + # restore is bit-for-bit. Correctness already came from the seed + # subtraction above; the guard proved the escrow agrees with it. + _set_values(part, np.arange(len(true_mean)), true_mean) + log(f"[unblind] escrow verified (subtraction residual {residual:.2e})") + # unblind_sacc stripped the concealed/blind stamps, so this is the true + # revealed vector; it inherits the blinded file's provenance (data vs mock). + sacc_io.save(part, out_path, type=blinded.metadata["type"]) + log(f"[unblind] wrote {out_path}") + return out_path + + +def _write_encrypted_json(encrpt_path, payload): + """Encrypt ``payload`` (JSON) to ``encrpt_path`` + sibling ``.key``; no + plaintext survives. + + We drive ``smokescreen.encryption.encrypt_file`` (Fernet) for the crypto + but write the ciphertext and key at the exact names we control. Its + ``save_file`` mode names outputs from ``basename.split('.')[0]`` — it + truncates at the first dot — so for a dotted stem (the canonical + catalogue-version case, e.g. ``v1.4.6.3_xi_integration_escrow``) it would land at + ``v1.encrpt``/``v1.key``, diverging from what :func:`part_paths` declares + and colliding across parts that share a first-dot prefix. Instead we take + the returned ``(ciphertext, key)`` and write them ourselves. + """ + from smokescreen.encryption import encrypt_file + + key_path = encrpt_path.replace(".encrpt", ".key") + plaintext = encrpt_path.replace(".encrpt", ".json") + with open(plaintext, "w", encoding="utf-8") as f: + json.dump(payload, f) + ciphertext, key = encrypt_file(plaintext, save_file=False, keep_original=False) + with open(encrpt_path, "wb") as f: + f.write(ciphertext) + with open(key_path, "wb") as f: + f.write(key) + + +def _read_encrypted_json(encrpt_path, key_path): + """Decrypt and parse a Fernet-encrypted JSON bundle.""" + from smokescreen.encryption import decrypt_file + + return json.loads(decrypt_file(encrpt_path, key_path).decode("utf-8")) diff --git a/src/sp_validation/blinding_theory.py b/src/sp_validation/blinding_theory.py new file mode 100644 index 00000000..b44f7e73 --- /dev/null +++ b/src/sp_validation/blinding_theory.py @@ -0,0 +1,424 @@ +"""Blinding theory: fiducial configuration and the two ξ± theory paths. + +:Name: blinding_theory.py + +:Description: The blinding backend's theory surface — the fiducial + configuration (:class:`TheoryConfig`) and two independent routes to the + tomographic shear two-point prediction. + + **Division of responsibility** (per the UNIONS layering: ``cs_util`` is + the cosmology *library* — generic machinery; ``sp_validation`` holds + *configuration* and *survey-specific implementations*). This module lives + in the blinding namespace because :class:`TheoryConfig` is configuration + and the master-layout theory backends in :mod:`sp_validation.blinding` + (reporting ξ±, integration ξ±, pseudo-Cℓ) are survey-specific. The **generic** + theory machinery below — the CCL-native ξ± path (:func:`xi_ccl`, + :func:`cl_ee`), the independent CAMB P(k)→``Pk2D`` path (:func:`xi_camb`), + and the σ8/A_s rescale (:func:`camb_As_for_sigma8`) — lives here **for + now** but is destined for ``cs_util.cosmo`` (tracked in cs_util#80): it is + cosmology-library code, not blinding-specific. There is deliberately **no** + ``sp_validation/cosmology.py`` — ``develop`` removed the local cosmology + module (#223) and moved cosmology to ``cs_util.cosmo``; this module does + not resurrect it. + + Two independent routes to the shear two-point prediction: + + - **CCL-native path** (:func:`xi_ccl`, :func:`cl_ee`): CCL builds the + nonlinear P(k) through its Boltzmann-CAMB HMCode2020 route + (``matter_power_spectrum='camb'`` + ``extra_parameters``) and projects + to Cℓ/ξ± via its own Limber (``angular_cl``) + FFTLog + (``correlation``). This is the recipe the blinding theory backends use. + - **Independent-CAMB path** (:func:`xi_camb`): a direct ``pycamb`` run + produces the HMCode2020 ``P(k, z)`` (σ8-matched via the closed-form + A_s rescale of :func:`camb_As_for_sigma8`), wrapped in a ``ccl.Pk2D`` + and projected through the same CCL Limber + FFTLog machinery. + + Because both paths route their nonlinear P(k) through CAMB's HMCode2020 + and both project through CCL, a common Limber+FFTLog bug cancels between + them: the CAMB↔CCL cross-check test built on these two paths validates + the **P(k) recipe** and the **σ8/A_s amplitude convention**, not the + projection machinery. + + This module imports only ``numpy`` at module level; CCL and CAMB are + imported inside the functions that need them, so importing + :class:`TheoryConfig` never drags in a theory backend. +""" + +import dataclasses + +import numpy as np + +# Fixed constants of the fiducial — load-bearing for the CAMB↔CCL amplitude +# match, so they are emitted explicitly to both stacks rather than left to +# either stack's default. Not user-facing TheoryConfig fields. +NEFF = 3.046 +T_CMB = 2.7255 + + +# --------------------------------------------------------------------------- # +# Configuration surface — the ONE place fiducial cosmology + model choices live +# --------------------------------------------------------------------------- # +@dataclasses.dataclass(frozen=True) +class TheoryConfig: + """Fiducial cosmology and model configuration for the theory paths. + + Every field is a deliberate, configurable choice. The defaults mirror the + ``cosmo_inference`` CosmoSIS fiducial (the ``SP_v1.4.6.3_A_cell`` pipeline + + ``values_ia.ini`` central values), so the CCL theory computed here and + the CAMB theory CosmoSIS computes agree to the level the CAMB↔CCL + cross-check test asserts. Adopting a different named group fiducial is a + change to these *values*, not to any code. + + Cosmology is parametrised by the blind axes ``S8`` and ``Omega_m`` and + converted to CCL's native ``sigma8``/``Omega_c`` by :meth:`sigma8` / + :meth:`omega_c`. + + **Nonlinear-model tokens (load-bearing).** The HMCode2020+feedback recipe + is named by a token in each stack's API: CCL takes + ``extra_parameters['camb']['halofit_version']``, CAMB takes + ``NonLinearModel.set_params(halofit_version=…)``. :class:`TheoryConfig` + carries **two** tokens (``ccl_halofit_version``, ``camb_halofit_version``) + denoting one recipe, and each stack is fed its own — a single shared + string invites a silent stack disagreement the moment the two APIs name + the recipe differently (``mead2020`` vs ``mead2020_feedback`` differ by + several % at k ≳ 1/Mpc). Today both stacks accept the same string for the + feedback recipe, so the two defaults coincide; the cross-check test pins + the CCL token against the inference config independently. + """ + + # Cosmological parameters (blind axes S8, Omega_m + the rest). + S8: float = 0.80 # values_ia.ini S_8_input central + Omega_m: float = 0.30 + Omega_b: float = 0.0469 # ombh2=0.023 at h=0.7 -> 0.023/0.7^2 + h: float = 0.70 + n_s: float = 0.96 + m_nu: float = 0.06 # Σm_ν in eV, distributed under `mass_split` + w0: float = -1.0 + wa: float = 0.0 + + # Neutrino mass split: normal hierarchy (CosmoSIS `neutrino_hierarchy=normal`). + mass_split: str = "normal" + + # Boltzmann/transfer-function backend for the CCL path (#280). The default + # `boltzmann_camb` makes CCL call CAMB for the linear P(k), so blinding + # theory and the CosmoSIS+CAMB inference stack share one power-spectrum + # path. The halofit route follows the choice (see `ccl_cosmology`): only + # under `boltzmann_camb` can the nonlinear P(k) run through CAMB's HMCode + # (`matter_power_spectrum="camb"` + the tokens below); any other backend + # falls back to CCL's own halofit — consistent, but a different recipe, + # so a non-default backend is a deliberate cross-check tool, not a + # production setting. + transfer_function: str = "boltzmann_camb" + + # One nonlinear recipe (CAMB HMCode2020 + baryonic feedback), two + # stack-specific tokens — see the class docstring. + ccl_halofit_version: str = "mead2020_feedback" + camb_halofit_version: str = "mead2020_feedback" + hmcode_logT_AGN: float = 7.5 # values_ia.ini logT_AGN central + + # Intrinsic alignments: NLA. The fiducial defaults IA OFF (ia_bias=0) — + # the blinding shift is a difference of two theory vectors at the same IA, + # so IA nearly cancels there, and IA-off keeps the CAMB↔CCL cross-check a + # clean test of the shear calculation. Set `ia_bias` nonzero (CosmoSIS + # central A=1.0) to include NLA. + ia_bias: float = 0.0 + ia_z_piv: float = 0.62 + ia_alphaz: float = 0.0 + + def sigma8(self): + """CCL ``sigma8`` implied by ``S8`` and ``Omega_m``. + + ``S8 ≡ σ8 √(Ωm / 0.3)`` — the standard weak-lensing definition — so + ``σ8 = S8 / √(Ωm / 0.3)``. At the fiducial (S8=0.80, Ωm=0.30), + σ8 = 0.80. + """ + return self.S8 / np.sqrt(self.Omega_m / 0.3) + + def omega_c(self): + """CCL cold-dark-matter density ``Omega_c = Omega_m − Omega_b − Ω_ν``. + + The neutrino density ``Ω_ν h² = Σm_ν / 93.14 eV`` is subtracted so + the *total* matter density is exactly ``Omega_m`` (CCL treats massive + neutrinos as a separate species, not part of ``Omega_c``). + """ + omega_nu = self.m_nu / (93.14 * self.h**2) + return self.Omega_m - self.Omega_b - omega_nu + + def ccl_params(self): + """The fiducial point as a plain CCL-native parameter mapping. + + Exactly the keys ``Omega_c, Omega_b, h, n_s, sigma8, m_nu, + mass_split, w0, wa, Neff, T_CMB`` and no others — no CCL default + rides along. ``Neff``/``T_CMB`` are the fixed module constants. This + mapping is what the Smokescreen fork receives as ``fiducial_params`` + and what every ``theory_fn`` receives back (possibly with + ``sigma8``/``Omega_c`` overlaid by the hidden draw). + """ + return { + "Omega_c": self.omega_c(), + "Omega_b": self.Omega_b, + "h": self.h, + "n_s": self.n_s, + "sigma8": self.sigma8(), + "m_nu": self.m_nu, + "mass_split": self.mass_split, + "w0": self.w0, + "wa": self.wa, + "Neff": NEFF, + "T_CMB": T_CMB, + } + + @classmethod + def from_overrides(cls, overrides): + """Build from a mapping of field overrides (fail loud on unknown keys). + + Numeric overrides are coerced to ``float`` per the field's declared + type, so a YAML/CLI ``w0: -1`` (int) yields the same value — and the + same :meth:`config_digest` — as the float default ``-1.0``. + """ + by_name = {f.name: f for f in dataclasses.fields(cls)} + unknown = set(overrides) - set(by_name) + if unknown: + raise ValueError( + f"unknown TheoryConfig fields {sorted(unknown)}; " + f"valid fields are {sorted(by_name)}" + ) + coerced = { + name: (float(v) if by_name[name].type in (float, "float") else v) + for name, v in overrides.items() + } + return cls(**coerced) + + +# --------------------------------------------------------------------------- # +# CCL-native path: cosmology construction, Cℓ_EE, ξ± +# --------------------------------------------------------------------------- # +# The two cosmologies of a blind (fiducial + hidden) are evaluated by three +# theory backends over multiple blocks; caching the ccl.Cosmology per parameter +# point avoids re-running the CAMB P(k) computation for every block. +_COSMO_CACHE = {} + + +def ccl_cosmology(params, config): + """A ``pyccl.Cosmology`` at ``params`` with ``config``'s nonlinear recipe. + + ``params`` is a plain CCL-native mapping (:meth:`TheoryConfig.ccl_params`, + possibly with keys overlaid by the hidden draw); ``config`` supplies only + the non-sampled recipe tokens (``transfer_function``, + ``ccl_halofit_version``, ``hmcode_logT_AGN``). The Boltzmann backend is + ``config.transfer_function`` (#280); the halofit route stays consistent + with that choice: under ``boltzmann_camb`` the nonlinear P(k) runs + through CAMB's HMCode2020 (``matter_power_spectrum="camb"`` + the CAMB + tokens), while any other backend has no CAMB run to hand tokens to, so it + takes CCL's own halofit (``matter_power_spectrum="halofit"``). Either + way, the same recipe sits on both sides of any theory difference. + Cosmology objects are cached per parameter point (CCL memoises its P(k) + on the object, so the cache saves repeated Boltzmann runs across blocks). + """ + import pyccl as ccl + + key = ( + tuple(sorted(params.items())), + config.transfer_function, + config.ccl_halofit_version, + config.hmcode_logT_AGN, + ) + if key not in _COSMO_CACHE: + nonlinear = ( + { + "matter_power_spectrum": "camb", + "extra_parameters": { + "camb": { + "halofit_version": config.ccl_halofit_version, + "HMCode_logT_AGN": config.hmcode_logT_AGN, + } + }, + } + if config.transfer_function == "boltzmann_camb" + else {"matter_power_spectrum": "halofit"} + ) + _COSMO_CACHE[key] = ccl.Cosmology( + **params, + transfer_function=config.transfer_function, + **nonlinear, + ) + return _COSMO_CACHE[key] + + +def xi_ell_grid(): + """The ℓ grid the ξ± Hankel projection integrates over. + + Integers 2…49, then 200 log-spaced multipoles up to 6·10⁴ — dense enough + at low ℓ (where ξ± at large θ lives) and wide enough for the small-θ + tail. ``ccl.correlation`` interpolates C(ℓ) internally, so this fixes the + resolution of every ξ± this module produces. + """ + return np.unique( + np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)]).astype(float) + ) + + +def _tracer(cosmo, z, nz, config): + """A ``WeakLensingTracer`` for one bin's n(z), NLA from ``config``. + + With the fiducial ``ia_bias = 0`` the tracer is built bare — no IA term. + A nonzero ``ia_bias`` enters as the NLA amplitude + ``A(z) = ia_bias · ((1+z)/(1+z_piv))^alphaz``. + """ + import pyccl as ccl + + z = np.asarray(z) + if config.ia_bias == 0.0: + return ccl.WeakLensingTracer(cosmo, dndz=(z, np.asarray(nz))) + a_ia = config.ia_bias * ((1 + z) / (1 + config.ia_z_piv)) ** config.ia_alphaz + return ccl.WeakLensingTracer( + cosmo, dndz=(z, np.asarray(nz)), ia_bias=(z, a_ia), use_A_ia=True + ) + + +def cl_ee(params, config, nz_i, nz_j, ell): + """Cross Cℓ_EE at ``ell`` for the bin pair with n(z) ``nz_i``, ``nz_j``. + + Two-tracer: one :class:`~pyccl.WeakLensingTracer` per bin from that bin's + own ``(z, nz)``, then ``angular_cl(cosmo, tracer_i, tracer_j, ell)`` — + the cross-spectrum for i ≠ j, the auto-spectrum when the two n(z) are the + same bin. The shear ``angular_cl`` is the E-mode spectrum; B and EB are + zero in theory, which is why only Cℓ_EE ever receives a blinding shift. + """ + import pyccl as ccl + + cosmo = ccl_cosmology(params, config) + tracer_i = _tracer(cosmo, *nz_i, config) + tracer_j = _tracer(cosmo, *nz_j, config) + return ccl.angular_cl(cosmo, tracer_i, tracer_j, np.asarray(ell, dtype=float)) + + +def xi_ccl(params, config, nz_i, nz_j, theta_arcmin, ell=None): + """CCL-native ξ± at ``theta_arcmin`` for one bin pair (Path A). + + Cross Cℓ_EE on :func:`xi_ell_grid` (or ``ell``), then ``ccl.correlation`` + (FFTLog Hankel transform) at θ in degrees, ``type="GG+"`` / ``"GG-"``. + + Returns + ------- + (np.ndarray, np.ndarray) + ``(xip, xim)`` aligned to ``theta_arcmin``. + """ + import pyccl as ccl + + ell = xi_ell_grid() if ell is None else np.asarray(ell, dtype=float) + cosmo = ccl_cosmology(params, config) + cl = cl_ee(params, config, nz_i, nz_j, ell) + theta_deg = np.asarray(theta_arcmin) / 60.0 + xip = ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta_deg, type="GG+") + xim = ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta_deg, type="GG-") + return xip, xim + + +# --------------------------------------------------------------------------- # +# Independent-CAMB path: A_s reconciliation + P(k) → Pk2D → CCL projection +# --------------------------------------------------------------------------- # +def make_camb_params(config, As, *, nonlinear, zmax=3.0, n_z=48, kmax=20.0): + """A ``CAMBparams`` at ``config``'s background with amplitude ``As``. + + Every :class:`TheoryConfig` field CCL sees is fed to CAMB from the same + source — ``w0``/``wa`` via ``set_dark_energy``, ``Neff``/``T_CMB`` as the + module constants, ``m_nu``/``mass_split`` through ``set_cosmology`` — so + the independent path differs from the CCL path only in who computes P(k), + never in an unmatched background parameter. + """ + import camb + + p = camb.CAMBparams() + p.set_cosmology( + H0=config.h * 100, + ombh2=config.Omega_b * config.h**2, + omch2=config.omega_c() * config.h**2, + mnu=config.m_nu, + num_massive_neutrinos=1, + neutrino_hierarchy=config.mass_split, + nnu=NEFF, + TCMB=T_CMB, + ) + p.set_dark_energy(w=config.w0, wa=config.wa, dark_energy_model="ppf") + p.InitPower.set_params(As=As, ns=config.n_s) + p.set_matter_power(redshifts=list(np.linspace(0.0, zmax, n_z)), kmax=kmax) + if nonlinear: + p.NonLinear = camb.model.NonLinear_both + p.NonLinearModel.set_params( + halofit_version=config.camb_halofit_version, + HMCode_logT_AGN=config.hmcode_logT_AGN, + ) + else: + p.NonLinear = camb.model.NonLinear_none + return p + + +def camb_linear_sigma8(config, As, **kwargs): + """CAMB's linear σ8(z=0) at amplitude ``As``.""" + import camb + + results = camb.get_results(make_camb_params(config, As, nonlinear=False, **kwargs)) + return float(results.get_sigma8_0()) + + +def camb_As_for_sigma8(config, sigma8_target, As_seed=2.1e-9, **kwargs): + """The CAMB ``A_s`` whose linear σ8 equals ``sigma8_target``. + + Closed-form: linear σ8² ∝ A_s exactly, so one CAMB linear-σ8 evaluation + at ``As_seed`` and one rescale ``As_seed · (σ8_target/σ8_seed)²`` land on + the target — no iteration. This settles the convention subtlety that our + fiducial fixes σ8 for CCL but A_s for CAMB: a nominal ``A_s = 2.1e-9`` + leaves CAMB's σ8 ≈3% off target, enough to blow a ξ± comparison to + ~9–10%. + """ + sigma8_seed = camb_linear_sigma8(config, As_seed, **kwargs) + return As_seed * (sigma8_target / sigma8_seed) ** 2 + + +def xi_camb(config, nz, theta_arcmin, *, n_ell=300, ell_max=60000, kmax=20.0, n_k=400): + """Independent-CAMB ξ± for one bin (Path B): CAMB P(k) → Pk2D → CCL. + + A direct pycamb run produces the HMCode2020 nonlinear ``P(k, z)`` at a + σ8-matched ``A_s`` (:func:`camb_As_for_sigma8`), extracted through + ``get_matter_power_interpolator(hubble_units=False, k_hunit=False)`` so + it comes out in CCL's native units (k in 1/Mpc, P in Mpc³) — **no** + ``·h`` / ``/h³`` conversion is applied (applying one would double-count + an h³ amplitude error). Both ``Pk2D`` axes are arranged ascending + (log-k ascending; scale factor ascending, i.e. CAMB's z-ascending grid + reversed). Projection is CCL's own Limber + FFTLog with a bare tracer + (IA off — this path exists for the cross-check). + + Returns + ------- + (np.ndarray, np.ndarray, float) + ``(xip, xim, As)`` — the σ8-matched amplitude is returned for + assertion by the cross-check test. + """ + import camb + import pyccl as ccl + + sigma8 = config.sigma8() + As = camb_As_for_sigma8(config, sigma8, kmax=kmax) + results = camb.get_results(make_camb_params(config, As, nonlinear=True, kmax=kmax)) + interp = results.get_matter_power_interpolator( + nonlinear=True, hubble_units=False, k_hunit=False + ) + k = np.geomspace(1e-4, kmax * config.h, n_k) # 1/Mpc + z = np.linspace(0.0, 3.0, 48) + pk = interp.P(z, k) # (n_z, n_k), Mpc^3 + a = 1.0 / (1.0 + z) + order = np.argsort(a) # Pk2D wants ascending scale factor + pk2d = ccl.Pk2D( + a_arr=a[order], lk_arr=np.log(k), pk_arr=np.log(pk[order]), is_logp=True + ) + + cosmo = ccl_cosmology(config.ccl_params(), config) + z_nz, nz_vals = nz + lens = ccl.WeakLensingTracer(cosmo, dndz=(np.asarray(z_nz), np.asarray(nz_vals))) + ells = np.unique(np.geomspace(2, ell_max, n_ell).astype(int)).astype(float) + cl = ccl.angular_cl(cosmo, lens, lens, ells, p_of_k_a=pk2d) + theta_deg = np.asarray(theta_arcmin) / 60.0 + xip = ccl.correlation(cosmo, ell=ells, C_ell=cl, theta=theta_deg, type="GG+") + xim = ccl.correlation(cosmo, ell=ells, C_ell=cl, theta=theta_deg, type="GG-") + return xip, xim, As diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 21872d0e..fd2e8df8 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -773,3 +773,50 @@ def load(path, *, allow_unblinded=False): "allow_unblinded=True." ) return s + + +# --------------------------------------------------------------------------- # +# Terminal assembly — added on top of PR-2's canonical module (PR-6 blinding). +# Everything above this banner is byte-identical to feat/sacc-2-sacc-io; only +# gather() and its blind-custody call site live here. +# --------------------------------------------------------------------------- # +def gather(parts, metadata=None): + """Assemble standalone part SACCs into the one-file ``{version}.sacc``. + + Each part is an intermediate product as it came off its producing rule + (reporting ξ±, integration ξ±, pseudo-Cℓ, ρ/τ, …). Assembly itself — the + first-wins tracer union, in-order point concatenation with all tags + (bandpower windows included), and the block-diagonal covariance — is + exactly :func:`merge`, so gather delegates to it and adds only the one + thing merge cannot know about: **blind custody.** + + **Blind custody (the module's one blind-aware call site):** + :func:`sp_validation.blinding.assert_consistent_blind` runs before the + merge — it fails closed unless every blindable part carries the identical + ``blind_commitment``/``blind_config_digest`` (or, when nothing is blinded, + every blindable part is declared ``type='mock'``). Its returned shared + stamp is written onto the assembled file so the one-file product carries + the blind it was built from; the blinded parts already carry those keys, + so merge preserves them and this stamp is a consistent (idempotent) + re-affirmation. + + Parameters + ---------- + parts : sequence of sacc.Sacc + The part SACCs, in the assembly (covariance) order. + metadata : dict, optional + Extra key/value pairs to store on the assembled file's metadata. + + Returns + ------- + sacc.Sacc + The assembled file. + """ + from . import blinding + + parts = list(parts) + stamp = blinding.assert_consistent_blind(parts) + s = merge(parts) + for key, value in {**(metadata or {}), **(stamp or {})}.items(): + s.metadata[key] = value + return s diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py new file mode 100644 index 00000000..7013a81a --- /dev/null +++ b/src/sp_validation/tests/test_blinding.py @@ -0,0 +1,1110 @@ +"""Tests for :mod:`sp_validation.blinding` — per-part Smokescreen blinding. + +Acceptance criteria AC1–AC9 of the blinding PRD, plus fast unit coverage of +the config/custody surface. Fast tests (no CCL import — the envelope +calibration, digest, commitment, fork-draw determinism, blind-init custody, +the merge alignment against a monkeypatched concealing factor, and the +assembly hash assertion) run in the default suite; the theory tests (fork + +CCL) are marked ``slow``; the derived-statistics tests additionally +``importorskip`` ``cosmo_numba``. + +All fixtures are synthetic and deterministic. Each blindable intermediate is +its own standalone part SACC (reporting ξ±, integration ξ±, pseudo-Cℓ), as in the +per-part-at-birth architecture; derived statistics (COSEBIs, pure-E/B) are +never stored in parts — they are computed downstream from the (blinded) integration +ξ± through the pipeline seams ``b_modes.cosebis_from_xi`` / +``b_modes.pure_eb_from_xi``, exactly as the pipeline does. The fork's +``ConcealDataVector`` carries no data-vector consistency check (only the +length guard), so fixture ξ± values are smooth synthetic templates — no +theory fill is needed to blind. +""" + +import json +import pathlib + +import numpy as np +import pytest + +from sp_validation import blinding as bd +from sp_validation import sacc_io as sio +from sp_validation.blinding_theory import TheoryConfig + +_NOLOG = lambda *a, **k: None # noqa: E731 + + +# --------------------------------------------------------------------------- # +# Synthetic part fixtures +# --------------------------------------------------------------------------- # +def _gauss_nz(z0, sigma, n=200): + z = np.linspace(0.0, 3.0, n) + nz = np.exp(-0.5 * ((z - z0) / sigma) ** 2) + return z, nz / np.trapezoid(nz, z) + + +def _reporting_theta(n=8): + return np.geomspace(5.0, 250.0, n) + + +def _integration_theta(n=80): + # The integration grid is the pure-E/B INTEGRATION grid, so it spans wider than + # the reporting range on both ends (production: ~0.08–300 arcmin). + return np.geomspace(0.1, 300.0, n) + + +def _xi_template(theta, k=0): + """Smooth synthetic ξ± for pair index ``k`` (no CCL needed).""" + theta = np.asarray(theta) + xip = 1e-4 * (1 + 0.1 * k) * (theta / 10.0) ** -0.6 + xim = 0.5e-4 * (1 + 0.1 * k) * (theta / 10.0) ** -0.9 + return xip, xim + + +def _b_mode_template(theta, amplitude): + """A smooth ξ_B(θ) template. B contributes +ξ_B to ξ+, −ξ_B to ξ−.""" + return amplitude * np.exp(-((np.log(np.asarray(theta) / 30.0)) ** 2) / 2.0) + + +def _nz_dict(nbins): + return {i: _gauss_nz(0.5 + 0.3 * i, 0.15 + 0.02 * i) for i in range(nbins)} + + +def _pairs(nbins): + return [(i, j) for i in range(nbins) for j in range(i, nbins)] + + +def make_xi_part(grid, nbins=1, b_amplitude=0.0): + """A standalone ξ± part SACC (one grid), synthetic values, eye covariance. + + ``b_amplitude`` injects a pure B-mode (+ξ_B to ξ+, −ξ_B to ξ−; + b_modes.py sign convention) — used on the integration part for AC4/AC9. + """ + theta = _reporting_theta() if grid == "reporting" else _integration_theta() + s = sio.new_sacc( + _nz_dict(nbins), metadata={"catalogue_version": "vTEST", "type": "mock"} + ) + blocks = [] + for k, (i, j) in enumerate(_pairs(nbins)): + xip, xim = _xi_template(theta, k) + xi_b = _b_mode_template(theta, b_amplitude) + sio.add_xi(s, (i, j), theta, xip + xi_b, xim - xi_b, grid=grid) + tr = sio._pair((i, j)) + idx = np.concatenate( + [ + s.indices(sio.XI_PLUS, tr, grid=grid), + s.indices(sio.XI_MINUS, tr, grid=grid), + ] + ) + blocks.append((idx, np.eye(len(idx)) * 1e-12)) + sio.assemble_covariance(s, blocks) + return s + + +def make_cl_part(nbins=1): + """A standalone pseudo-Cℓ part SACC (EE/BB/EB + bandpower windows).""" + s = sio.new_sacc( + _nz_dict(nbins), metadata={"catalogue_version": "vTEST", "type": "mock"} + ) + ell_eff = np.array([30.0, 80.0, 150.0, 280.0, 450.0]) + w_ell = np.arange(2, 501).astype(float) + w_mat = np.zeros((len(w_ell), len(ell_eff))) + for b, le in enumerate(ell_eff): + w_mat[:, b] = np.exp(-0.5 * ((w_ell - le) / 40.0) ** 2) + w_mat[:, b] /= w_mat[:, b].sum() + blocks = [] + for k, (i, j) in enumerate(_pairs(nbins)): + cl_ee = 1e-8 * (1 + 0.1 * k) * (ell_eff / 100.0) ** -1.2 + sio.add_pseudo_cl( + s, + (i, j), + ell_eff, + cl_ee, + np.zeros(5), + np.zeros(5), + window_ells=w_ell, + window_weights=w_mat, + ) + tr = sio._pair((i, j)) + idx = np.concatenate( + [s.indices(dt, tr) for dt in (sio.CL_EE, sio.CL_BB, sio.CL_EB)] + ) + blocks.append((idx, np.eye(len(idx)) * 1e-16)) + sio.assemble_covariance(s, blocks) + return s + + +def make_rho_part(): + """A standalone ρ/τ PSF-diagnostics part SACC — never blindable.""" + ctheta = _reporting_theta() + s = sio.new_sacc( + _nz_dict(1), metadata={"catalogue_version": "vTEST", "type": "mock"} + ) + blocks = [] + for k in range(2): + sio.add_rho( + s, k, ctheta, np.arange(len(ctheta)) * 1e-7, np.arange(len(ctheta)) * 2e-7 + ) + idx = np.concatenate( + [s.indices(sio.RHO_PLUS.format(k=k)), s.indices(sio.RHO_MINUS.format(k=k))] + ) + blocks.append((idx, np.eye(len(idx)) * 1e-18)) + sio.add_tau( + s, (0,), 0, ctheta, np.arange(len(ctheta)) * 3e-7, np.arange(len(ctheta)) * 4e-7 + ) + idx = np.concatenate( + [s.indices(sio.TAU_PLUS.format(k=0)), s.indices(sio.TAU_MINUS.format(k=0))] + ) + blocks.append((idx, np.eye(len(idx)) * 1e-18)) + sio.assemble_covariance(s, blocks) + return s + + +def make_parts(nbins=1, b_amplitude=0.0, with_rho=True): + """All intermediate parts of one catalogue version, keyed by name.""" + parts = { + "xi_reporting": make_xi_part("reporting", nbins), + "xi_integration": make_xi_part("integration", nbins, b_amplitude=b_amplitude), + "cl": make_cl_part(nbins), + } + if with_rho: + parts["rho_tau"] = make_rho_part() + return parts + + +def _derive_downstream(reporting_part, integration_part, nmodes=6): + """COSEBIs + pure-E/B the way the pipeline derives them downstream. + + COSEBIs from the integration ξ± (full-range scale cut); pure-E/B from the + measured reporting reporting ξ± + the integration integration ξ±, with the + edge-based bounds set to the reporting grid's span — the outermost + reporting point sits at tmax with no interior support and comes back + NaN (the AC9 boundary case). + """ + from sp_validation import b_modes + + theta_f, xip_f, xim_f = sio.get_xi(integration_part, (0, 0), grid="integration") + theta_c, xip_c, xim_c = sio.get_xi(reporting_part, (0, 0), grid="reporting") + En, Bn = b_modes.cosebis_from_xi( + theta_f, xip_f, xim_f, nmodes, scale_cut=(theta_f.min(), theta_f.max()) + ) + modes = b_modes.pure_eb_from_xi( + theta_c, + xip_c, + xim_c, + theta_f, + xip_f, + xim_f, + float(theta_c[0]), + float(theta_c[-1]), + ) + return En, Bn, modes + + +# --------------------------------------------------------------------------- # +# BlindingConfig: envelope calibration + digest (fast) +# --------------------------------------------------------------------------- # +def test_blinding_config_defaults(): + c = bd.BlindingConfig() + assert c.s8_half_width == 0.075 + assert c.omega_m_half_width == 0.1 + assert c.theory.S8 == 0.80 # fiducial TheoryConfig defaults + + +def test_blinding_config_overrides_fail_loud(): + c = bd.BlindingConfig.from_overrides({"s8_half_width": 0.05}) + assert c.s8_half_width == 0.05 + with pytest.raises(ValueError, match="unknown BlindingConfig fields"): + bd.BlindingConfig.from_overrides({"s8_half_width": 0.05, "bogus": 1}) + with pytest.raises(ValueError, match="unknown TheoryConfig fields"): + bd.BlindingConfig.from_overrides({"theory": {"nope": 1}}) + + +def test_blinding_config_is_frozen(): + with pytest.raises(Exception): + bd.BlindingConfig().s8_half_width = 0.2 + + +def test_envelope_calibration_maps_s8_box_to_ccl_halfwidths(): + """(S8, Ωm) half-widths → {sigma8, Omega_c} at the fiducial (exact forms).""" + c = bd.BlindingConfig() + shifts = c.shifts_dict() + assert set(shifts) == {"sigma8", "Omega_c"} + assert shifts["sigma8"] == pytest.approx( + c.s8_half_width / np.sqrt(c.theory.Omega_m / 0.3) + ) + assert shifts["Omega_c"] == c.omega_m_half_width + # every shift key must exist in the fiducial point (fork contract) + assert set(shifts) <= set(c.theory.ccl_params()) + + +def test_config_digest_stable_and_sensitive(): + """Canonical digest: byte-stable across runs, moves with every bound field.""" + c = bd.BlindingConfig() + assert c.config_digest() == bd.BlindingConfig().config_digest() + assert len(c.config_digest()) == 64 + assert bd.BlindingConfig(s8_half_width=0.05).config_digest() != c.config_digest() + assert ( + bd.BlindingConfig.from_overrides({"theory": {"S8": 0.79}}).config_digest() + != c.config_digest() + ) + # the P(k) recipe tokens are bound: a different halofit token = new digest + assert ( + bd.BlindingConfig.from_overrides( + {"theory": {"ccl_halofit_version": "takahashi"}} + ).config_digest() + != c.config_digest() + ) + # the Boltzmann backend (#280) is bound too — a different transfer + # function is a different P(k) path, so a different blind + assert ( + bd.BlindingConfig.from_overrides( + {"theory": {"transfer_function": "eisenstein_hu"}} + ).config_digest() + != c.config_digest() + ) + + +def test_theory_config_transfer_function_default_and_override(): + """#280: the Boltzmann backend is one config knob, CAMB by default (the + inference pipeline's Boltzmann code), overridable like any other field.""" + assert TheoryConfig().transfer_function == "boltzmann_camb" + cfg = TheoryConfig.from_overrides({"transfer_function": "eisenstein_hu"}) + assert cfg.transfer_function == "eisenstein_hu" + + +def test_config_digest_int_float_canonical(): + """One physical cosmology has one digest, regardless of int-vs-float literals. + + Configs come from YAML/CLI/humans, so a field can arrive as ``-1`` (int) or + ``-1.0`` (float). The digest must depend on the numeric *value*, not the + literal's Python type — otherwise an int-vs-float mismatch between the blind + and a later unblind would raise "config digest mismatch" and deny a + legitimate unblind. Every declared-float field must be canonical this way. + """ + for field, int_val, float_val in [ + ("w0", -1, -1.0), + ("wa", 0, 0.0), + ("Omega_m", 1, 1.0), + ("m_nu", 0, 0.0), + ("S8", 1, 1.0), + ]: + assert ( + bd.BlindingConfig.from_overrides( + {"theory": {field: int_val}} + ).config_digest() + == bd.BlindingConfig.from_overrides( + {"theory": {field: float_val}} + ).config_digest() + ), f"int-vs-float digest split on theory.{field}" + # the envelope half-widths (BlindingConfig's own float fields) too + assert ( + bd.BlindingConfig.from_overrides({"s8_half_width": 1}).config_digest() + == bd.BlindingConfig.from_overrides({"s8_half_width": 1.0}).config_digest() + ) + # and a full round-trip: an all-int override matches the float default digest + assert ( + bd.BlindingConfig.from_overrides( + {"theory": {"w0": -1, "Omega_m": 0, "wa": 0}} + ).config_digest() + == bd.BlindingConfig.from_overrides( + {"theory": {"w0": -1.0, "Omega_m": 0.0, "wa": 0.0}} + ).config_digest() + ) + + +def test_theory_config_ccl_params_exact_keyset(): + """ccl_params() carries exactly the contracted keys — nothing rides along.""" + params = TheoryConfig().ccl_params() + assert set(params) == { + "Omega_c", + "Omega_b", + "h", + "n_s", + "sigma8", + "m_nu", + "mass_split", + "w0", + "wa", + "Neff", + "T_CMB", + } + assert params["sigma8"] == pytest.approx(0.80) # S8=0.80 at Ωm=0.30 + assert params["Neff"] == 3.046 and params["T_CMB"] == 2.7255 + + +# --------------------------------------------------------------------------- # +# Commitment + the fork's draw (fast; smokescreen import is light) +# --------------------------------------------------------------------------- # +def test_commitment_is_sha256_of_seed(): + import hashlib + + seed = "the-secret" + assert bd.seed_commitment(seed) == hashlib.sha256(seed.encode()).hexdigest() + assert bd.seed_commitment("right") != bd.seed_commitment("wrong") + + +def test_hidden_params_deterministic_and_in_envelope(): + """Same (seed, config) ⇒ same hidden point; draws respect the envelope.""" + import secrets + + c = bd.BlindingConfig() + fid = c.theory.ccl_params() + h1, h2 = bd.hidden_params("a-seed", c), bd.hidden_params("a-seed", c) + assert h1 == h2 + assert bd.hidden_params("другой", c) != h1 + shifts = c.shifts_dict() + for _ in range(50): + h = bd.hidden_params(secrets.token_hex(8), c) + for key, half in shifts.items(): + assert abs(h[key] - fid[key]) <= half + # only the enveloped keys move + assert all(h[k] == fid[k] for k in fid if k not in shifts) + + +def test_hidden_params_no_global_rng_state(): + """The fork's draw uses a local RNG — global numpy state is untouched.""" + np.random.seed(0) + before = np.random.get_state()[1].copy() + bd.hidden_params("whatever", bd.BlindingConfig()) + assert np.array_equal(before, np.random.get_state()[1]) + + +# --------------------------------------------------------------------------- # +# Per-part merge + provenance with a monkeypatched factor (fast — no CCL) +# --------------------------------------------------------------------------- # +def _patch_constant_factor(monkeypatch, value=1e-6): + def fake(part, indices, factory, config, seed): + return np.arange(len(indices), dtype=float) * value + value + + monkeypatch.setattr(bd, "_concealing_factor", fake) + return fake + + +def test_merge_places_shift_at_recorded_indices_only(monkeypatch): + """AC5 (merge half), per part: the shift lands exactly on the blindable + rows, in stored order; covariance, n(z), and every tag are untouched.""" + _patch_constant_factor(monkeypatch) + for name, part in make_parts(nbins=2, with_rho=False).items(): + orig = np.array(part.mean) + orig_cov = part.covariance.dense.copy() + orig_nz = part.tracers["source_0"].nz.copy() + + blinded = bd.blind_sacc(part, "seed", log=_NOLOG) + + blocks = bd._blindable_blocks(part) + assert len(blocks) == 1, f"{name}: a part carries exactly one block" + shifted = np.zeros(len(orig), dtype=bool) + for _, indices, _ in blocks: + expected = orig[indices] + (np.arange(len(indices)) * 1e-6 + 1e-6) + assert np.array_equal(np.array(blinded.mean)[indices], expected) + shifted[indices] = True + assert np.array_equal(np.array(blinded.mean)[~shifted], orig[~shifted]) + assert np.array_equal(blinded.covariance.dense, orig_cov) + assert np.array_equal(blinded.tracers["source_0"].nz, orig_nz) + # row-order preservation: type/tracers/tags sequence is bitwise unchanged + for a, b in zip(part.data, blinded.data): + assert a.data_type == b.data_type + assert a.tracers == b.tracers + assert a.tags == b.tags + + +def test_provenance_metadata_contract(monkeypatch): + """Blinded parts carry concealed/blind/commitment/digest; no seed key.""" + _patch_constant_factor(monkeypatch) + s = make_xi_part("reporting") + s.metadata["seed_smokescreen"] = "leaked!" # must be stripped + c = bd.BlindingConfig() + blinded = bd.blind_sacc(s, "seed", config=c, label="B", log=_NOLOG) + assert blinded.metadata["concealed"] is True + assert blinded.metadata["blind"] == "B" + assert blinded.metadata["blind_commitment"] == bd.seed_commitment("seed") + assert blinded.metadata["blind_config_digest"] == c.config_digest() + assert "seed_smokescreen" not in blinded.metadata + assert blinded.metadata["catalogue_version"] == "vTEST" + + +def test_blind_refuses_double_blind(monkeypatch): + _patch_constant_factor(monkeypatch) + s = make_xi_part("reporting") + blinded = bd.blind_sacc(s, "seed", log=_NOLOG) + with pytest.raises(ValueError, match="already concealed"): + bd.blind_sacc(blinded, "seed2", log=_NOLOG) + + +def test_blind_refuses_non_blindable_part(): + """A ρ/τ diagnostic part must never see a blind call — loud refusal.""" + with pytest.raises(ValueError, match="no blindable block"): + bd.blind_sacc(make_rho_part(), "seed", log=_NOLOG) + + +def test_unblind_fails_closed_on_wrong_seed_or_config(monkeypatch): + """AC6 (in-memory half): wrong seed and wrong config both refuse loudly.""" + _patch_constant_factor(monkeypatch) + s = make_xi_part("reporting") + blinded = bd.blind_sacc(s, "right-seed", log=_NOLOG) + with pytest.raises(ValueError, match="blind_commitment"): + bd.unblind_sacc(blinded, "wrong-seed", log=_NOLOG) + with pytest.raises(ValueError, match="blind_config_digest"): + bd.unblind_sacc( + blinded, + "right-seed", + config=bd.BlindingConfig(s8_half_width=0.01), + log=_NOLOG, + ) + with pytest.raises(ValueError, match="not concealed"): + bd.unblind_sacc(s, "right-seed", log=_NOLOG) + + +# --------------------------------------------------------------------------- # +# blind-init custody + assembly hash assertion (fast — encryption only) +# --------------------------------------------------------------------------- # +def test_blind_init_writes_commitment_and_encrypted_bundle_only(tmp_path): + """AC6 (init): commitment.json + encrypted bundle; never a plaintext seed.""" + paths = bd.blind_init(str(tmp_path), log=_NOLOG) + with open(paths["commitment"], encoding="utf-8") as f: + commitment = json.load(f) + assert set(commitment) == {"label", "seed_sha256", "config_digest"} + assert len(commitment["seed_sha256"]) == 64 + assert commitment["config_digest"] == bd.BlindingConfig().config_digest() + # exactly the three custody outputs, no plaintext bundle + assert {p.name for p in tmp_path.iterdir()} == { + "commitment.json", + "blind_seed.encrpt", + "blind_seed.key", + } + # the decrypted seed matches the public commitment + bundle = bd._read_encrypted_json(paths["bundle"], paths["key"]) + assert bd.seed_commitment(bundle["seed"]) == commitment["seed_sha256"] + # one-shot custody: a second init in the same dir refuses + with pytest.raises(FileExistsError, match="refusing to overwrite"): + bd.blind_init(str(tmp_path), log=_NOLOG) + + +def test_read_seed_fails_closed_on_drifted_config(tmp_path): + bd.blind_init(str(tmp_path), log=_NOLOG) + with pytest.raises(ValueError, match="config digest"): + bd._read_seed(str(tmp_path), bd.BlindingConfig(s8_half_width=0.01)) + + +def _stamp(s, seed="s", label="A", config=None): + bd._stamp_provenance( + s, + bd.seed_commitment(seed), + label, + (config or bd.BlindingConfig()).config_digest(), + ) + return s + + +def test_assert_consistent_blind_shared_stamp(): + """One commitment across all blindable parts ⇒ the shared stamp returns; + ρ/τ parts are exempt from the assertion.""" + parts = make_parts(nbins=1) + for name in ("xi_reporting", "xi_integration", "cl"): + _stamp(parts[name]) + stamp = bd.assert_consistent_blind(list(parts.values())) + assert stamp == { + "concealed": True, + "blind": "A", + "blind_commitment": bd.seed_commitment("s"), + "blind_config_digest": bd.BlindingConfig().config_digest(), + } + + +def test_assert_consistent_blind_fails_closed(): + """AC6 (assembly): mismatched commitments and mixed states both refuse.""" + parts = make_parts(nbins=1, with_rho=False) + _stamp(parts["xi_reporting"], seed="one") + _stamp(parts["xi_integration"], seed="one") + _stamp(parts["cl"], seed="two") # different seed ⇒ different commitment + with pytest.raises(ValueError, match="different blind commitments"): + bd.assert_consistent_blind(list(parts.values())) + + parts = make_parts(nbins=1, with_rho=False) + _stamp(parts["xi_reporting"]) # blinded beside plaintext blindable parts + with pytest.raises(ValueError, match="mixed"): + bd.assert_consistent_blind(list(parts.values())) + + +def test_assert_consistent_blind_all_plaintext_is_none(): + """A declared-mock plaintext assembly (nothing blinded) asserts nothing.""" + assert bd.assert_consistent_blind(list(make_parts().values())) is None + + +def test_assert_consistent_blind_unconcealed_data_fails_closed(): + """PRD §4 "Mocks vs data": an unconcealed blindable part may assemble only + if its metadata declares ``type == "mock"`` — an unconcealed ``data`` part, + or one missing the tag, fails closed (skipping the blind can never + silently expose real data). Mixed concealed/plaintext still fails on the + mixed guard regardless of type.""" + parts = make_parts(nbins=1, with_rho=False) + parts["xi_integration"].metadata["type"] = "data" + with pytest.raises(ValueError, match="type"): + bd.assert_consistent_blind(list(parts.values())) + + parts = make_parts(nbins=1, with_rho=False) + del parts["cl"].metadata["type"] # missing tag counts as not-a-mock + with pytest.raises(ValueError, match=""): + bd.assert_consistent_blind(list(parts.values())) + + # concealed data parts assemble integration (that is the whole point of the blind) + parts = make_parts(nbins=1, with_rho=False) + for p in parts.values(): + p.metadata["type"] = "data" + _stamp(p) + assert bd.assert_consistent_blind(list(parts.values()))["concealed"] is True + + # mixed concealed/plaintext fails on the mixed guard even for mocks + parts = make_parts(nbins=1, with_rho=False) + _stamp(parts["xi_reporting"]) + with pytest.raises(ValueError, match="mixed"): + bd.assert_consistent_blind(list(parts.values())) + + +def test_gather_fails_closed_on_unconcealed_data_part(): + """The type guard reaches the terminal gather surface too.""" + parts = make_parts(nbins=1, with_rho=False) + parts["xi_reporting"].metadata["type"] = "data" + with pytest.raises(ValueError, match="type"): + sio.gather(list(parts.values())) + + +def test_assert_consistent_blind_differing_labels_warn_not_fail(): + """Same seed+config, different --label ⇒ assemble cleanly with a warning. + + The label is provenance, not custody state: parts blinded under one blind + but tagged with different labels must not be misread as different blinds. + The assembly succeeds (keyed on commitment+digest); a distinct warning + surfaces the label divergence rather than a false "different commitments". + """ + parts = make_parts(nbins=1, with_rho=False) + _stamp(parts["xi_reporting"], seed="one", label="A") + _stamp(parts["xi_integration"], seed="one", label="A") + _stamp(parts["cl"], seed="one", label="B") # same blind, different label + with pytest.warns(UserWarning, match="different labels"): + stamp = bd.assert_consistent_blind(list(parts.values())) + assert stamp["blind_commitment"] == bd.seed_commitment("one") + assert stamp["blind"] == "A" # deterministic: sorted-first label + + +def test_gather_assembles_parts_and_stamps_blind(): + """The sacc_io gather combines parts (points, tags, covariance blocks), + calls the assembly assertion, and stamps the shared blind.""" + parts = make_parts(nbins=1) + for name in ("xi_reporting", "xi_integration", "cl"): + _stamp(parts[name]) + ordered = [parts[k] for k in ("xi_reporting", "cl", "rho_tau", "xi_integration")] + s = sio.gather(ordered, metadata={"catalogue_version": "vTEST", "type": "mock"}) + + assert len(s.mean) == sum(len(p.mean) for p in ordered) + assert np.array_equal(np.array(s.mean), np.concatenate([p.mean for p in ordered])) + # covariance: block-diagonal of the parts, in order + cursor = 0 + for p in ordered: + n = len(p.mean) + assert np.array_equal( + s.covariance.dense[cursor : cursor + n, cursor : cursor + n], + p.covariance.dense, + ) + cursor += n + # the integration rows are addressable by the grid tag in the assembled file + assert len(s.indices(sio.XI_PLUS, grid="integration")) == len( + parts["xi_integration"].indices(sio.XI_PLUS, grid="integration") + ) + # bandpower windows survive assembly + assert s.get_bandpower_windows(s.indices(sio.CL_EE)) is not None + # blind stamp on the assembled file + assert s.metadata["concealed"] is True + assert s.metadata["blind_commitment"] == bd.seed_commitment("s") + assert s.metadata["catalogue_version"] == "vTEST" + + +def test_gather_fails_closed_on_mismatched_blinds(): + parts = make_parts(nbins=1, with_rho=False) + _stamp(parts["xi_reporting"], seed="one") + _stamp(parts["xi_integration"], seed="two") + _stamp(parts["cl"], seed="one") + with pytest.raises(ValueError, match="different blind commitments"): + sio.gather(list(parts.values())) + + +def test_cli_blind_init_refuses_existing_state(tmp_path): + """The blind-init CLI refuses to overwrite a previous blind's state.""" + import importlib.util + + script = ( + pathlib.Path(__file__).resolve().parents[3] / "scripts" / "blind_data_vector.py" + ) + spec = importlib.util.spec_from_file_location("_blind_cli", script) + cli = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cli) + + (tmp_path / "commitment.json").write_text("{}") + with pytest.raises(SystemExit, match="refusing to overwrite"): + cli.main(["blind-init", str(tmp_path)]) + + +# --------------------------------------------------------------------------- # +# AC2 + AC3 + AC7: the shift itself (slow — fork + CCL) +# --------------------------------------------------------------------------- # +@pytest.mark.slow +@pytest.mark.parametrize( + "transfer_function", + ["boltzmann_camb", "eisenstein_hu"], + ids=["default-camb", "non-default-eh"], +) +def test_ac2_on_file_shift_equals_theory_difference_per_part(transfer_function): + """AC2: per-row shift on each blinded part == theory_fn(hidden) − + theory_fn(fiducial), hidden recovered by re-running the fork's draw — + for all three parts, on a two-bin fixture; and the recovered hidden + cosmology is identical across the three parts (one seed → one hidden). + + Scope: this verifies fork-draw recovery + placement (the shift on the + file is exactly what re-running the same backend at the recovered + hidden/fiducial points predicts, at the recorded rows). It is NOT a + backend-correctness test: both sides run the identical ``theory_fn``, so + any wrong-cosmology dependence cancels and a wrong backend would still + pass here. Backend correctness is carried by AC3. + + Parametrized over the Boltzmann backend (#280): the default CAMB route + and one non-default (Eisenstein–Hu — cheap, no CAMB run) both thread the + same ``transfer_function`` knob through all three theory backends.""" + cfg = bd.BlindingConfig.from_overrides( + {"theory": {"transfer_function": transfer_function}} + ) + seed = "ac2-seed" + parts = make_parts(nbins=2, with_rho=False) + + hiddens, worst = [], 0.0 + for name, part in parts.items(): + blinded = bd.blind_sacc(part, seed, config=cfg, log=_NOLOG) + hidden = bd.hidden_params(seed, cfg) # recovered per part + hiddens.append(hidden) + fiducial = cfg.theory.ccl_params() + ((block_name, indices, factory),) = bd._blindable_blocks(part) + theory = factory(bd._extract_block(part, indices), cfg.theory) + expected = theory(hidden) - theory(fiducial) + actual = np.array(blinded.mean)[indices] - np.array(part.mean)[indices] + gap = np.max(np.abs(actual - expected)) + scale = np.max(np.abs(expected)) + worst = max(worst, gap / scale) + assert gap <= 1e-10 * max(scale, 1e-30), f"{name}/{block_name}: |Δ|={gap:.3e}" + # one seed → one hidden cosmology across all parts + assert hiddens[0] == hiddens[1] == hiddens[2] + print(f"\nAC2 max relative shift mismatch across parts: {worst:.3e}") + + +@pytest.mark.slow +def test_ac3_cross_backend_against_independent_ccl_reference(): + """AC3: the realized shift matches an independently written direct-CCL + reference (self-contained here; does not touch the blinding backends).""" + import pyccl as ccl + + cfg = bd.BlindingConfig() + seed = "ac3-seed" + reporting = make_xi_part("reporting", nbins=2) + cl_part = make_cl_part(nbins=2) + blinded_xi = bd.blind_sacc(reporting, seed, config=cfg, log=_NOLOG) + blinded_cl = bd.blind_sacc(cl_part, seed, config=cfg, log=_NOLOG) + hidden = bd.hidden_params(seed, cfg) + fiducial = cfg.theory.ccl_params() + + # ----- independent reference (from scratch; same fixture n(z), θ, ℓ) ---- + def ref_cosmo(params): + return ccl.Cosmology( + **params, + matter_power_spectrum="camb", + extra_parameters={ + "camb": {"halofit_version": "mead2020_feedback", "HMCode_logT_AGN": 7.5} + }, + ) + + ell = np.unique( + np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)]).astype(float) + ) + + def ref_xi(params, nz_i, nz_j, theta): + cosmo = ref_cosmo(params) + ti = ccl.WeakLensingTracer(cosmo, dndz=nz_i) + tj = ccl.WeakLensingTracer(cosmo, dndz=nz_j) + cl = ccl.angular_cl(cosmo, ti, tj, ell) + xip = ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta / 60.0, type="GG+") + xim = ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta / 60.0, type="GG-") + return xip, xim + + worst = 0.0 + for i, j in bd.xi_pairs(reporting, "reporting"): + tr = sio._pair((i, j)) + theta = sio._tag(reporting, sio.XI_PLUS, tr, "theta", grid="reporting") + nz_i, nz_j = sio.get_nz(reporting, i), sio.get_nz(reporting, j) + xip_h, xim_h = ref_xi(hidden, nz_i, nz_j, theta) + xip_f, xim_f = ref_xi(fiducial, nz_i, nz_j, theta) + for dt, ref_shift in ( + (sio.XI_PLUS, xip_h - xip_f), + (sio.XI_MINUS, xim_h - xim_f), + ): + idx = reporting.indices(dt, tr, grid="reporting") + realized = np.array(blinded_xi.mean)[idx] - np.array(reporting.mean)[idx] + worst = max(worst, np.max(np.abs(realized - ref_shift))) + print(f"\nAC3 max |realized − independent reference| (reporting ξ±): {worst:.3e}") + assert worst < 1e-8 # observed ~1e-10; factor magnitudes ~1e-6 + + # pseudo-Cℓ: W @ ΔCℓ_EE against the same independent reference + for i, j in bd.cl_pairs(cl_part): + tr = sio._pair((i, j)) + idx = cl_part.indices(sio.CL_EE, tr) + window = cl_part.get_bandpower_windows(idx) + w_ell = np.asarray(window.values, dtype=float) + w_mat = np.asarray(window.weight, dtype=float) + nz_i, nz_j = sio.get_nz(cl_part, i), sio.get_nz(cl_part, j) + + def ref_cl(params): + cosmo = ref_cosmo(params) + ti = ccl.WeakLensingTracer(cosmo, dndz=nz_i) + tj = ccl.WeakLensingTracer(cosmo, dndz=nz_j) + return ccl.angular_cl(cosmo, ti, tj, w_ell) + + ref_shift = w_mat.T @ (ref_cl(hidden) - ref_cl(fiducial)) + realized = np.array(blinded_cl.mean)[idx] - np.array(cl_part.mean)[idx] + assert np.max(np.abs(realized - ref_shift)) < 1e-12 # bandpowers ~1e-9 + + +@pytest.mark.slow +def test_ac7_reproducibility_same_seed_same_shift(): + """AC7: two blind runs of the same part with the same (seed, config) + produce identical shifts; a different seed produces a different one.""" + part = make_xi_part("reporting") + b1 = bd.blind_sacc(part, "repro-seed", log=_NOLOG) + b2 = bd.blind_sacc(part, "repro-seed", log=_NOLOG) + assert np.array_equal(np.array(b1.mean), np.array(b2.mean)) + b3 = bd.blind_sacc(part, "other-seed", log=_NOLOG) + assert not np.array_equal(np.array(b1.mean), np.array(b3.mean)) + + +# --------------------------------------------------------------------------- # +# AC1, AC4, AC5, AC9: born-blinded derived statistics (slow + cosmo_numba) +# --------------------------------------------------------------------------- # +@pytest.mark.slow +def test_ac1_zero_shift_is_identity(): + """AC1: a zero envelope reproduces every part exactly, and the integration part + run downstream through the b_modes seams yields COSEBIs and pure-E/B + identical to the unblinded run — the per-part plumbing and the + born-blinded derivation path are the identity at zero shift.""" + pytest.importorskip("cosmo_numba") + zero = bd.BlindingConfig(s8_half_width=0.0, omega_m_half_width=0.0) + parts = make_parts(nbins=1, with_rho=False) + blinded = { + name: bd.blind_sacc(p, "any-seed", config=zero, log=_NOLOG) + for name, p in parts.items() + } + for name, part in parts.items(): + assert np.array_equal(np.array(part.mean), np.array(blinded[name].mean)), ( + f"zero-shift blind changed {name}" + ) + # derived statistics downstream: identical inputs ⇒ identical numbers + En_t, Bn_t, modes_t = _derive_downstream( + parts["xi_reporting"], parts["xi_integration"] + ) + En_b, Bn_b, modes_b = _derive_downstream( + blinded["xi_reporting"], blinded["xi_integration"] + ) + assert np.array_equal(En_t, En_b) and np.array_equal(Bn_t, Bn_b) + for key in modes_t: + t, b = modes_t[key], modes_b[key] + both_nan = np.isnan(t) & np.isnan(b) + assert np.array_equal(t[~both_nan], b[~both_nan]), key + assert np.array_equal(np.isnan(t), np.isnan(b)), key + + +@pytest.mark.slow +def test_ac4_b_mode_invariance_and_leakage_floor(): + """AC4: the ΔBₙ induced by deriving B-modes from the blinded integration ξ± is + independent of the injected B amplitude (a fixed absolute E→B leakage + offset, not fractional). The magnitude is measured and reported, never + asserted against a constant.""" + pytest.importorskip("cosmo_numba") + seed = "ac4-seed" + deltas, reports = [], [] + for amp in (2e-6, 2e-5): + reporting = make_xi_part("reporting") + integration = make_xi_part("integration", b_amplitude=amp) + blinded_reporting = bd.blind_sacc(reporting, seed, log=_NOLOG) + blinded_integration = bd.blind_sacc(integration, seed, log=_NOLOG) + _, Bn_t, modes_t = _derive_downstream(reporting, integration) + _, Bn_b, modes_b = _derive_downstream(blinded_reporting, blinded_integration) + d_bn = Bn_b - Bn_t + d_xib = modes_b["xip_B"] - modes_t["xip_B"] + finite = np.isfinite(d_xib) + deltas.append((d_bn, d_xib[finite])) + reports.append( + f"B={amp:.0e}: max|ΔBₙ|={np.max(np.abs(d_bn)):.3e} " + f"(ΔBₙ/Bₙ={np.max(np.abs(d_bn)) / np.max(np.abs(Bn_t)):.2e}), " + f"max|Δξ+_B|={np.max(np.abs(d_xib[finite])):.3e} " + f"({np.max(np.abs(d_xib[finite])) / amp:.2%} of injected B)" + ) + print("\nAC4 " + "\nAC4 ".join(reports)) + + (d_bn_1, d_xib_1), (d_bn_2, d_xib_2) = deltas + scale = max(np.max(np.abs(d_bn_1)), 1e-30) + gap = np.max(np.abs(d_bn_1 - d_bn_2)) + print( + f"AC4 ΔBₙ amplitude-independence: max|ΔBₙ(2e-6) − ΔBₙ(2e-5)| = " + f"{gap:.3e} ({gap / scale:.2e} of |ΔBₙ|)" + ) + assert gap <= 1e-9 * scale + 1e-24, ( + "ΔBₙ depends on the injected B amplitude — the shift is not pure E" + ) + # The pure-ξ_B leakage is also amplitude-independent, but only to the + # adaptive-quadrature floor: cosmo_numba's Schneider integrals subdivide + # adaptively, so the estimator is not bit-linear in its inputs and the + # two runs differ at a small fraction of the (tiny) leakage itself. The + # COSEBIs assertion above carries the exact-identity criterion; this one + # bounds the quadrature wobble. + scale_x = max(np.max(np.abs(d_xib_1)), 1e-30) + gap_x = np.max(np.abs(d_xib_1 - d_xib_2)) + print( + f"AC4 Δξ+_B amplitude-independence: {gap_x:.3e} " + f"({gap_x / scale_x:.2e} of the leakage)" + ) + assert gap_x <= 0.05 * scale_x + + +@pytest.mark.slow +def test_ac5_untouched_blocks_and_row_order(): + """AC5: each part's covariance byte-identical; the Cℓ BB/EB rows and the + ρ/τ part never blinded; every part's shifted rows land at their original + within-part indices (order-preservation).""" + parts = make_parts(nbins=1) + seed = "ac5-seed" + blinded = { + name: bd.blind_sacc(parts[name], seed, log=_NOLOG) + for name in ("xi_reporting", "xi_integration", "cl") + } + for name, b in blinded.items(): + part = parts[name] + assert np.array_equal(b.covariance.dense, part.covariance.dense), name + # row order: the identity of every row (type/tracers/tags) unchanged + for a, c in zip(part.data, b.data): + assert (a.data_type, a.tracers, a.tags) == (c.data_type, c.tracers, c.tags) + # BB/EB rows of the Cℓ part untouched (pure E-mode shift) + for dt in (sio.CL_BB, sio.CL_EB): + idx = parts["cl"].indices(dt) + assert np.array_equal( + np.array(blinded["cl"].mean)[idx], np.array(parts["cl"].mean)[idx] + ), f"{dt} was touched by the blind" + # the blindable rows did move (the blind actually blinded) + for name, grid in ( + ("xi_reporting", "reporting"), + ("xi_integration", "integration"), + ): + idx = bd._xi_indices(parts[name], grid) + assert not np.allclose( + np.array(blinded[name].mean)[idx], np.array(parts[name].mean)[idx], atol=0 + ) + # ρ/τ: refused by blind_sacc (test_blind_refuses_non_blindable_part) and + # exempt in assembly — pass through gather untouched + s = sio.gather( + [ + blinded["xi_reporting"], + parts["rho_tau"], + blinded["cl"], + blinded["xi_integration"], + ] + ) + idx = s.indices(sio.RHO_PLUS.format(k=0)) + assert np.array_equal( + np.array(s.mean)[idx], + np.array(parts["rho_tau"].mean)[ + parts["rho_tau"].indices(sio.RHO_PLUS.format(k=0)) + ], + ) + + +@pytest.mark.slow +def test_ac9_pure_eb_nan_parity_under_blind(): + """AC9: the pure-E/B NaN pattern born from the blinded parts is identical + to the true parts' — blinding never moves a NaN. + + The Schneider estimator returns NaN wherever a reporting point lacks + interior support against the edge-based integration bounds. Whatever + that pattern is on the true parts (the fixture puts the outermost + reporting point at the boundary, so it is non-empty here; on production + files it is empty), the blinded derivation must reproduce it bit-for-bit: + the blind is a pure shift of the estimator's inputs, not a change of + estimator support. The finite values move (the ξ± shifted); the NaN mask + does not.""" + pytest.importorskip("cosmo_numba") + seed = "ac9-seed" + reporting, integration = make_xi_part("reporting"), make_xi_part("integration") + _, _, modes_t = _derive_downstream(reporting, integration) + _, _, modes_b = _derive_downstream( + bd.blind_sacc(reporting, seed, log=_NOLOG), + bd.blind_sacc(integration, seed, log=_NOLOG), + ) + for key in modes_t: + t, b = modes_t[key], modes_b[key] + assert np.array_equal(np.isnan(t), np.isnan(b)), ( + f"blinding moved the pure-E/B NaN pattern for {key}" + ) + # the finite values did move (the blind actually shifted the ξ±) + t, b = modes_t["xip_E"], modes_b["xip_E"] + finite = np.isfinite(t) + assert finite.any() and not np.allclose(t[finite], b[finite], atol=0) + + +# --------------------------------------------------------------------------- # +# AC6 + AC8: end-to-end custody through the file surface (slow + cosmo_numba) +# --------------------------------------------------------------------------- # +@pytest.mark.slow +def test_ac6_ac8_end_to_end_init_parts_gather_unblind(tmp_path): + """AC8: blind-init → blind-part on each intermediate part → terminal + gather (hash assertion passes, stamp lands) → unblind restores each part + bit-for-bit, and the derived statistics re-derived from the unblinded + integration part reproduce the truth. AC6: no plaintext part or seed survives + on disk, no ``seed_smokescreen`` key; unblind fails closed on a tampered + commitment.""" + pytest.importorskip("cosmo_numba") + parts = make_parts(nbins=1) + En_true, Bn_true, modes_true = _derive_downstream( + parts["xi_reporting"], parts["xi_integration"] + ) + + blind_dir = tmp_path / "blind" + blind_dir.mkdir() + init = bd.blind_init(str(blind_dir), log=_NOLOG) + + part_files, out_paths = {}, {} + for name in ("xi_reporting", "xi_integration", "cl"): + path = tmp_path / f"{name}.fits" + sio.save(parts[name], str(path)) + out_paths[name] = bd.blind_part(str(path), str(blind_dir), log=_NOLOG) + part_files[name] = path + + # -- custody hygiene (AC6) --------------------------------------------- -- + for name, path in part_files.items(): + assert not path.exists(), f"plaintext part {name} was not deleted" + blinded = sio.load(out_paths[name]["blinded"]) + assert blinded.metadata["concealed"] is True + assert "seed_smokescreen" not in blinded.metadata + assert pathlib.Path(out_paths[name]["escrow"]).exists() + assert not np.array_equal(np.array(blinded.mean), np.array(parts[name].mean)) + with open(init["commitment"], encoding="utf-8") as f: + commitment = json.load(f) + assert set(commitment) == {"label", "seed_sha256", "config_digest"} + blinded_parts = {n: sio.load(p["blinded"]) for n, p in out_paths.items()} + for b in blinded_parts.values(): + assert b.metadata["blind_commitment"] == commitment["seed_sha256"] + # no plaintext json anywhere beside the blind outputs + assert not list(tmp_path.rglob("*escrow.json")) + assert not (blind_dir / "blind_seed.json").exists() + + # -- terminal assembly: hash assertion + stamp (AC8) -------------------- -- + assembled = sio.gather( + [ + blinded_parts["xi_reporting"], + blinded_parts["cl"], + parts["rho_tau"], + blinded_parts["xi_integration"], + ], + metadata={"catalogue_version": "vTEST", "type": "mock"}, + ) + assert assembled.metadata["blind_commitment"] == commitment["seed_sha256"] + # born-blinded derived statistics from the blinded parts differ from truth + En_b, Bn_b, _ = _derive_downstream( + blinded_parts["xi_reporting"], blinded_parts["xi_integration"] + ) + assert not np.allclose(En_b, En_true, atol=0) + + # -- fail-closed on tampered commitment (AC6) --------------------------- -- + tampered = dict(commitment, seed_sha256="0" * 64) + with open(init["commitment"], "w", encoding="utf-8") as f: + json.dump(tampered, f) + with pytest.raises(ValueError, match="sha256"): + bd.unblind_part( + out_paths["xi_integration"]["blinded"], + str(blind_dir), + str(tmp_path / "never.fits"), + log=_NOLOG, + ) + with open(init["commitment"], "w", encoding="utf-8") as f: + json.dump(commitment, f) + with pytest.raises(ValueError, match="config digest"): + bd.unblind_part( + out_paths["xi_integration"]["blinded"], + str(blind_dir), + str(tmp_path / "never.fits"), + config=bd.BlindingConfig(s8_half_width=0.01), + log=_NOLOG, + ) + + # -- bit-for-bit restoration per part (AC8) ----------------------------- -- + restored = {} + for name in ("xi_reporting", "xi_integration", "cl"): + out = tmp_path / f"{name}_restored.fits" + bd.unblind_part( + out_paths[name]["blinded"], str(blind_dir), str(out), log=_NOLOG + ) + restored[name] = sio.load(str(out)) + assert np.array_equal( + np.array(restored[name].mean), np.array(parts[name].mean) + ), f"{name} not restored bit-for-bit" + assert not restored[name].metadata.get("concealed", False) + assert "blind_commitment" not in restored[name].metadata + + # unblinding then re-deriving reproduces the true derived statistics + En_r, Bn_r, modes_r = _derive_downstream( + restored["xi_reporting"], restored["xi_integration"] + ) + assert np.array_equal(En_r, En_true) and np.array_equal(Bn_r, Bn_true) + for key in modes_true: + t, r = modes_true[key], modes_r[key] + both_nan = np.isnan(t) & np.isnan(r) + assert np.array_equal(t[~both_nan], r[~both_nan]), key + assert np.array_equal(np.isnan(t), np.isnan(r)), key + + +def test_ac8_dotted_versioned_part_names_escrow_and_restore(tmp_path): + """AC8 under the canonical catalogue-version naming (dotted stems). + + Production part files carry the versioned name ``v1.4.6.3_xi_reporting.fits`` + etc. ``smokescreen.encryption.encrypt_file`` names its outputs from + ``basename.split('.')[0]``, so both these parts would misfile onto + ``v1.encrpt``/``v1.key`` and the second would silently overwrite the + first's escrowed truth. Guard: the escrow lands at the exact + :func:`part_paths` name, two dot-prefix-sharing parts do not collide, and + each restores bit-for-bit.""" + parts = make_parts(nbins=1) + blind_dir = tmp_path / "blind" + blind_dir.mkdir() + bd.blind_init(str(blind_dir), log=_NOLOG) + + version = "v1.4.6.3" + out_paths, part_files = {}, {} + for name in ("xi_reporting", "xi_integration"): + path = tmp_path / f"{version}_{name}.fits" + sio.save(parts[name], str(path)) + out_paths[name] = bd.blind_part(str(path), str(blind_dir), log=_NOLOG) + part_files[name] = path + + # escrow bundles landed at the declared names (no split('.') truncation), + # and the two dot-prefix-sharing parts did not collide onto one bundle. + escrow_files = {n: p["escrow"] for n, p in out_paths.items()} + assert escrow_files["xi_reporting"] != escrow_files["xi_integration"] + for name, path in part_files.items(): + assert not path.exists(), f"plaintext part {name} was not deleted" + assert pathlib.Path(out_paths[name]["escrow"]).exists(), name + assert pathlib.Path(out_paths[name]["escrow_key"]).exists(), name + # the truncated-name collision target must not exist + assert not (tmp_path / "v1.encrpt").exists() + assert not (tmp_path / "v1.key").exists() + assert not list(tmp_path.rglob("*escrow.json")) + + # each part restores bit-for-bit via its own escrow (not subtraction-only) + for name in ("xi_reporting", "xi_integration"): + out = tmp_path / f"{version}_{name}_restored.fits" + bd.unblind_part( + out_paths[name]["blinded"], str(blind_dir), str(out), log=_NOLOG + ) + restored = sio.load(str(out)) + assert np.array_equal(np.array(restored.mean), np.array(parts[name].mean)), ( + f"{name} not restored bit-for-bit" + ) diff --git a/src/sp_validation/tests/test_camb_ccl_crosscheck.py b/src/sp_validation/tests/test_camb_ccl_crosscheck.py new file mode 100644 index 00000000..b4cb3b3f --- /dev/null +++ b/src/sp_validation/tests/test_camb_ccl_crosscheck.py @@ -0,0 +1,174 @@ +"""CAMB↔CCL theory cross-check (blinding PRD AC10–14). + +The blinding shift is a difference of CCL theory vectors; downstream +inference runs CAMB (CosmoSIS). The shift only means what it is intended to +mean if CCL and CAMB predict the same ξ± at a fixed cosmology on our θ grid. +This module asserts that agreement between the two independent ξ± paths in +:mod:`sp_validation.blinding_theory`: + +- **Path A** (:func:`~sp_validation.blinding_theory.xi_ccl`): CCL-native — CCL's + Boltzmann-CAMB HMCode2020 P(k) route, projected by CCL Limber + FFTLog. +- **Path B** (:func:`~sp_validation.blinding_theory.xi_camb`): an independent + pycamb run produces the HMCode2020 ``P(k, z)`` (σ8-matched ``A_s``), + wrapped in a ``ccl.Pk2D`` and projected through the same CCL machinery. + +Because both paths route their nonlinear P(k) through CAMB's HMCode2020 and +both project through CCL, a common Limber+FFTLog bug cancels: this test +validates the **P(k) recipe** and the **σ8/A_s amplitude convention**, not +the projection. The one convention subtlety it settles: the fiducial fixes +σ8 for CCL but A_s for CAMB; a nominal ``A_s = 2.1e-9`` leaves CAMB's σ8 +≈3% off target — enough to blow a ξ± comparison to ~9–10%. +""" + +import pathlib +import re + +import numpy as np +import pytest + +from sp_validation import blinding_theory as cm + +# Tolerances (AC11/AC12). Observed floor on this fixture: see the printed +# numbers in the slow tests — the tolerances sit above the floor with +# headroom; version bumps move the floor and that is not a regression. +XIP_RTOL = 0.005 # 0.5 % +XIM_RTOL = 0.010 # 1.0 % +# ξ− crosses zero on this grid: the relative assertion applies only where +# |ξ−| exceeds an absolute floor set from the fixture's peak |ξ−|. +XIM_FLOOR_FRAC = 0.05 + + +# --------------------------------------------------------------------------- # +# Deterministic fixture: one Gaussian source bin, 12-point θ grid +# --------------------------------------------------------------------------- # +def _gauss_nz(n=400): + z = np.linspace(0.01, 3.0, n) + nz = np.exp(-0.5 * ((z - 0.7) / 0.2) ** 2) + return z, nz / np.trapezoid(nz, z) + + +THETA_ARCMIN = np.geomspace(5.0, 250.0, 12) + + +def _both_paths(config, **camb_kwargs): + z, nz = _gauss_nz() + xip_a, xim_a = cm.xi_ccl( + config.ccl_params(), config, (z, nz), (z, nz), THETA_ARCMIN + ) + xip_b, xim_b, As = cm.xi_camb(config, (z, nz), THETA_ARCMIN, **camb_kwargs) + return (xip_a, xim_a), (xip_b, xim_b), As + + +def _assert_xi_agreement(a, b, label): + (xip_a, xim_a), (xip_b, xim_b) = a, b + assert np.all(xip_a > 0) and np.all(xip_b > 0) # sensible cosmic shear + rel_p = np.abs(xip_b - xip_a) / np.abs(xip_a) + assert rel_p.max() < XIP_RTOL, ( + f"{label}: ξ+ max rel diff {rel_p.max():.3%} ≥ {XIP_RTOL:.1%}" + ) + floor = XIM_FLOOR_FRAC * np.max(np.abs(xim_a)) + above = np.abs(xim_a) > floor + rel_m = np.abs(xim_b - xim_a)[above] / np.abs(xim_a)[above] + assert rel_m.max() < XIM_RTOL, ( + f"{label}: ξ− max rel diff {rel_m.max():.3%} ≥ {XIM_RTOL:.1%} (on |ξ−| > floor)" + ) + # near the zero crossing: absolute agreement at the floor scale + abs_m = np.abs(xim_b - xim_a)[~above] + if len(abs_m): + assert abs_m.max() < XIM_RTOL * floor, ( + f"{label}: ξ− absolute diff {abs_m.max():.3e} near zero crossing" + ) + print( + f"\n{label}: ξ+ max rel {rel_p.max():.3%}; " + f"ξ− max rel {rel_m.max():.3%} (above floor, " + f"{above.sum()}/{len(above)} points)" + ) + + +# --------------------------------------------------------------------------- # +# AC10: σ8/A_s reconciliation +# --------------------------------------------------------------------------- # +@pytest.mark.slow +def test_ac10_sigma8_As_reconciliation(): + """(a) nominal A_s leaves CAMB's σ8 >2% off target — the convention + offset is real; (b) the closed-form rescale lands on target to <1e-4.""" + cfg = cm.TheoryConfig() + target = cfg.sigma8() + + nominal = cm.camb_linear_sigma8(cfg, 2.1e-9) + offset = abs(nominal / target - 1) + print(f"\nAC10 nominal-A_s σ8 offset: {offset:.4f}") + assert offset > 0.02 + + As = cm.camb_As_for_sigma8(cfg, target) + matched = cm.camb_linear_sigma8(cfg, As) + print(f"AC10 σ8-matched residual: {abs(matched - target):.2e} (A_s={As:.4e})") + assert abs(matched - target) < 1e-4 + + +# --------------------------------------------------------------------------- # +# AC11 + AC12: ξ± agreement at and off the fiducial +# --------------------------------------------------------------------------- # +@pytest.mark.slow +def test_ac11_xi_agreement_at_fiducial(): + cfg = cm.TheoryConfig() + a, b, _ = _both_paths(cfg) + _assert_xi_agreement(a, b, "AC11 fiducial") + + +@pytest.mark.slow +def test_ac12_xi_agreement_off_fiducial(): + """A representative in-envelope offset — the *shift* (a difference of two + theory vectors) must not inherit a stack-disagreement bias.""" + cfg = cm.TheoryConfig.from_overrides({"S8": 0.80 + 0.075, "Omega_m": 0.30 - 0.05}) + a, b, _ = _both_paths(cfg) + _assert_xi_agreement(a, b, "AC12 off-fiducial") + + +# --------------------------------------------------------------------------- # +# AC13: halofit token pinned to the inference config (fast) +# --------------------------------------------------------------------------- # +def test_ac13_halofit_token_matches_inference_config(): + """The blinding fiducial's CCL halofit token equals the CosmoSIS + inference config's ``halofit_version`` — asserted against the config + file itself. All three blinding backends share one recipe by + construction and would agree with each other while jointly diverging + from the inference stack, so this cannot be caught by the cross-backend + test and is asserted independently here.""" + ini = ( + pathlib.Path(__file__).resolve().parents[3] + / "cosmo_inference" + / "cosmosis_config" + / "cosmosis_pipeline_A_ia_cell.ini" + ) + match = re.search(r"^halofit_version\s*=\s*(\S+)", ini.read_text(), re.MULTILINE) + assert match, f"no halofit_version in {ini}" + inference_token = match.group(1) + cfg = cm.TheoryConfig() + assert cfg.ccl_halofit_version == inference_token + # the two stack tokens denote ONE recipe; a divergence is a config bug + assert cfg.camb_halofit_version == cfg.ccl_halofit_version + # #280: the shipped Boltzmann backend is CAMB-through-CCL, matching the + # CosmoSIS+CAMB inference stack — one power-spectrum path. The cross-check + # tests above (AC10–12, 14) all run at this default configuration. + assert cfg.transfer_function == "boltzmann_camb" + + +# --------------------------------------------------------------------------- # +# AC14: fast smoke — broken wiring caught in the fast suite +# --------------------------------------------------------------------------- # +def test_ac14_crosscheck_smoke(): + """Both paths run at coarse resolution: finite, positive, + few-percent-agreeing ξ+, and a σ8-matched A_s in a sane range.""" + cfg = cm.TheoryConfig() + z, nz = _gauss_nz(n=150) + theta = np.geomspace(10.0, 100.0, 4) + xip_a, _ = cm.xi_ccl(cfg.ccl_params(), cfg, (z, nz), (z, nz), theta) + xip_b, _, As = cm.xi_camb( + cfg, (z, nz), theta, n_ell=120, ell_max=30000, kmax=10.0, n_k=200 + ) + assert np.all(np.isfinite(xip_a)) and np.all(np.isfinite(xip_b)) + assert np.all(xip_a > 0) and np.all(xip_b > 0) + assert 1e-9 < As < 3e-9 + rel = np.abs(xip_b - xip_a) / np.abs(xip_a) + assert rel.max() < 0.05, f"smoke ξ+ rel diff {rel.max():.3%} unexpectedly large" From 90d53afdcf1a9feb7d41b9bad7996459cf5bb666 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 11:50:07 +0200 Subject: [PATCH 010/160] Rebuild PR4 on feat/sacc-2-sacc-io: drop vendored sacc_io, keep migration The branch previously carried a stale vendored copy of sacc_io.py / test_sacc_io.py from before PR2's review rounds. Rebuilt directly on feat/sacc-2-sacc-io (8bd38171) so the canonical module is inherited, bringing over the cosmo_val + Snakemake born-as-SACC migration work from the old tip unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WzUt7VbtXwr2SCHUdiQTyt --- .github/workflows/deploy-image.yml | 6 + .github/workflows/lint.yml | 209 +- .gitignore | 13 +- CONTRIBUTING.md | 25 +- Dockerfile | 57 +- README.md | 20 + config/calibration/mask_v1.X.4_im_sim.yaml | 70 - .../mask_v1.X.9_im_sim.overlay.yaml | 97 - config/calibration/mask_v1.X.9_im_sim.yaml | 77 - cosmo_inference/.gitignore | 1 - cosmo_inference/README.md | 4 +- cosmo_inference/cfis_pipeline.sh | 66 + cosmo_inference/cosmocov_config/cosmocov.ini | 80 + .../cosmosis_config/cosmosis_pipeline.ini | 89 + .../cosmosis_pipeline_A_ia.ini | 108 + .../cosmosis_pipeline_A_ia_cell.ini | 105 + .../cosmosis_pipeline_A_psf.ini | 114 + ..._minsep=1_maxsep=250_nbins=20_npatch=1.ini | 124 + ..._minsep=1_maxsep=250_nbins=20_npatch=1.ini | 124 + ...sep=1.0_maxsep=250.0_nbins=20_npatch=1.ini | 124 + ....0_maxsep=250.0_nbins=20_npatch=1_cell.ini | 113 + .../cosmosis_pipeline_SP_v1.4.6.3_A_cell.ini | 114 + ..._pipeline_SP_v1.4.6.3_leak_corr_A_cell.ini | 111 + ...v1.4.6.3_leak_corr_HMCode_nobar_A_cell.ini | 114 + ...ne_SP_v1.4.6.3_leak_corr_OneCov_A_cell.ini | 114 + ...e_SP_v1.4.6.3_leak_corr_halofit_A_cell.ini | 114 + ..._leak_corr_include_large_scales_A_cell.ini | 114 + ...SP_v1.4.6.3_leak_corr_kmax=1Mpc_A_cell.ini | 114 + ...SP_v1.4.6.3_leak_corr_kmax=3Mpc_A_cell.ini | 114 + ...SP_v1.4.6.3_leak_corr_kmax=5Mpc_A_cell.ini | 114 + ...v1.4.6.3_leak_corr_large_scales_A_cell.ini | 114 + ...v1.4.6.3_leak_corr_small_scales_A_cell.ini | 114 + .../cosmosis_pipeline_SP_v1.4.6.3_B_cell.ini | 111 + ..._pipeline_SP_v1.4.6.3_leak_corr_B_cell.ini | 111 + ...v1.4.6.3_leak_corr_HMCode_nobar_B_cell.ini | 111 + ...ne_SP_v1.4.6.3_leak_corr_OneCov_B_cell.ini | 111 + ...e_SP_v1.4.6.3_leak_corr_halofit_B_cell.ini | 111 + ..._leak_corr_include_large_scales_B_cell.ini | 111 + ...SP_v1.4.6.3_leak_corr_kmax=1Mpc_B_cell.ini | 111 + ...SP_v1.4.6.3_leak_corr_kmax=3Mpc_B_cell.ini | 111 + ...SP_v1.4.6.3_leak_corr_kmax=5Mpc_B_cell.ini | 111 + ...v1.4.6.3_leak_corr_large_scales_B_cell.ini | 111 + ...v1.4.6.3_leak_corr_small_scales_B_cell.ini | 111 + .../cosmosis_pipeline_SP_v1.4.6.3_C_cell.ini | 114 + ..._pipeline_SP_v1.4.6.3_leak_corr_C_cell.ini | 111 + ...v1.4.6.3_leak_corr_HMCode_nobar_C_cell.ini | 114 + ...ne_SP_v1.4.6.3_leak_corr_OneCov_C_cell.ini | 114 + ...e_SP_v1.4.6.3_leak_corr_halofit_C_cell.ini | 114 + ..._leak_corr_include_large_scales_C_cell.ini | 114 + ...SP_v1.4.6.3_leak_corr_kmax=1Mpc_C_cell.ini | 114 + ...SP_v1.4.6.3_leak_corr_kmax=3Mpc_C_cell.ini | 114 + ...SP_v1.4.6.3_leak_corr_kmax=5Mpc_C_cell.ini | 114 + ...v1.4.6.3_leak_corr_large_scales_C_cell.ini | 114 + ...v1.4.6.3_leak_corr_small_scales_C_cell.ini | 114 + cosmo_inference/cosmosis_config/priors.ini | 11 + .../cosmosis_config/priors_mock.ini | 15 + .../cosmosis_config/priors_mock_cell.ini | 11 + .../priors_mock_cell_no_sys.ini | 5 + .../cosmosis_config/priors_psf.ini | 15 + cosmo_inference/cosmosis_config/values.ini | 27 + .../cosmosis_config/values_empty.ini | 27 + cosmo_inference/cosmosis_config/values_ia.ini | 26 + .../cosmosis_config/values_ia_no_sys.ini | 26 + .../cosmosis_config/values_ia_test.ini | 27 + .../cosmosis_config/values_psf.ini | 30 + .../cosmosis_config/values_template.ini | 23 + cosmo_inference/get_chi2.ipynb | 1391 ++++++ cosmo_inference/get_chi2_cell.ipynb | 1527 ++++++ .../S8_om_sigma8_whisker.ipynb | 645 +++ .../best_fit_xipm.ipynb | 607 +++ .../contours.ipynb | 950 ++++ .../get_chi2.ipynb | 690 +++ .../get_chi2_glass_mock.ipynb | 565 +++ .../get_prior_psf_leakage.ipynb | 261 + .../glass_mock_hist.ipynb | 586 +++ .../masking.ipynb | 132 + .../nonlin_k_analysis.ipynb | 174 + .../unblinding_party_plots.py | 894 ++++ .../2D_cosmic_shear_unblinding/utils.py | 442 ++ cosmo_inference/notebooks/cfis_analysis.ipynb | 1065 ++++ cosmo_inference/notebooks/cfis_mcmc.ipynb | 1546 ++++++ .../notebooks/get_prior_psf_leakage.ipynb | 269 + cosmo_inference/pipeline.sh | 131 + cosmo_inference/scripts/2pt_like_xi_sys.py | 614 +++ .../scripts/chain_postprocessing.py | 24 +- cosmo_inference/scripts/cosmocov_process.py | 81 + cosmo_inference/scripts/cosmosis_fitting.py | 12 +- cosmo_inference/scripts/k_analysis.py | 308 -- cosmo_inference/scripts/masking.py | 319 ++ cosmo_inference/scripts/matching.py | 40 + cosmo_inference/scripts/nz_writeout.py | 26 + cosmo_inference/scripts/slurm.sh | 22 + cosmo_inference/scripts/treecorr_calc.py | 107 + cosmo_inference/scripts/xi_sys_psf.py | 53 + papers/bmodes/scripts/run_xi_sweep.py | 10 +- papers/realspace/S8_om_sigma8_whisker.py | 549 -- papers/realspace/best_fit_xipm.py | 497 -- papers/realspace/contours.py | 745 --- papers/realspace/cov_masking.py | 82 - papers/realspace/get_chi2.py | 564 --- papers/realspace/get_chi2_glass_mock.py | 468 -- papers/realspace/get_prior_psf_leakage.py | 163 - papers/realspace/glass_mock_hist.py | 458 -- papers/realspace/nonlin_k_analysis.py | 104 - pyproject.toml | 96 +- .../calibrate_comprehensive_cat.py | 7 +- scripts/calibration/extract_info.py | 266 +- scripts/calibration/params.py | 3 - scripts/compute_m_bias_image_sims.py | 361 -- scripts/diagnostics_image_sims.py | 227 - scripts/patch_firecrown.py | 225 + src/sp_validation/calibration.py | 16 +- src/sp_validation/catalog.py | 144 +- src/sp_validation/catalog_builders.py | 16 - src/sp_validation/cosmo_val/core.py | 18 + src/sp_validation/cosmo_val/cosebis.py | 42 + src/sp_validation/cosmo_val/pseudo_cl.py | 94 +- .../cosmo_val/psf_systematics.py | 36 + src/sp_validation/cosmo_val/pure_eb.py | 21 + src/sp_validation/cosmo_val/real_space.py | 81 +- src/sp_validation/cosmo_val/sacc_writers.py | 254 + src/sp_validation/image_sims.py | 324 -- src/sp_validation/masks.py | 82 +- src/sp_validation/pseudo_cl.py | 29 + src/sp_validation/tests/test_assemble_sacc.py | 269 + .../tests/test_bmodes_workflow_dry_run.py | 68 +- src/sp_validation/tests/test_cli_seams.py | 65 + .../tests/test_config_paths_exist.py | 9 +- src/sp_validation/tests/test_image_sims.py | 253 - src/sp_validation/tests/test_mask_overlay.py | 85 - src/sp_validation/tests/test_masks.py | 63 - src/sp_validation/tests/test_pseudo_cl.py | 52 +- src/sp_validation/tests/test_sacc_writers.py | 313 ++ uv-overrides.txt | 21 + uv.lock | 4451 ----------------- workflow/README.md | 33 - workflow/Snakefile | 5 - workflow/common.py | 10 + workflow/image_sims/Snakefile | 46 - workflow/image_sims/config.yaml | 89 - workflow/image_sims/params_im_sim.py | 228 - workflow/profiles/candide/config.yaml | 85 - workflow/rules/cosmo_val.smk | 160 +- workflow/rules/image_sims.smk | 568 --- workflow/rules/inference.smk | 22 +- workflow/rules/twopoint.smk | 39 +- workflow/scripts/assemble_sacc.py | 254 + workflow/scripts/cv_cosebis.py | 18 +- workflow/scripts/cv_pseudo_cl.py | 8 +- workflow/scripts/cv_pure_eb.py | 12 +- workflow/scripts/cv_summarize_bmodes.py | 4 +- workflow/scripts/generate_cosmocov_ini.py | 149 + workflow/scripts/generate_pseudo_cl.py | 66 +- workflow/scripts/im_build_manifest.py | 244 - workflow/scripts/im_compose_mask.py | 109 - workflow/scripts/run_2pcf.py | 54 +- workflow/scripts/run_2pcf_highres.py | 74 +- workflow/scripts/run_cosmocov_chain.sh | 90 + workflow/scripts/run_rho_tau.py | 6 +- 159 files changed, 20852 insertions(+), 12225 deletions(-) delete mode 100644 config/calibration/mask_v1.X.4_im_sim.yaml delete mode 100644 config/calibration/mask_v1.X.9_im_sim.overlay.yaml delete mode 100644 config/calibration/mask_v1.X.9_im_sim.yaml create mode 100644 cosmo_inference/cfis_pipeline.sh create mode 100644 cosmo_inference/cosmocov_config/cosmocov.ini create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline.ini create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_cell.ini create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_A_psf.ini create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_A_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_A_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_A_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_A_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_A_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_A_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_A_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_A_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_A_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_A_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_A_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_B_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_B_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_B_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_B_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_B_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_B_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_B_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_B_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_B_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_B_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_B_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_C_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_C_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_C_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_C_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_C_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_C_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_C_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_C_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_C_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_C_cell.ini create mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_C_cell.ini create mode 100644 cosmo_inference/cosmosis_config/priors.ini create mode 100644 cosmo_inference/cosmosis_config/priors_mock.ini create mode 100644 cosmo_inference/cosmosis_config/priors_mock_cell.ini create mode 100644 cosmo_inference/cosmosis_config/priors_mock_cell_no_sys.ini create mode 100644 cosmo_inference/cosmosis_config/priors_psf.ini create mode 100644 cosmo_inference/cosmosis_config/values.ini create mode 100644 cosmo_inference/cosmosis_config/values_empty.ini create mode 100644 cosmo_inference/cosmosis_config/values_ia.ini create mode 100644 cosmo_inference/cosmosis_config/values_ia_no_sys.ini create mode 100644 cosmo_inference/cosmosis_config/values_ia_test.ini create mode 100644 cosmo_inference/cosmosis_config/values_psf.ini create mode 100644 cosmo_inference/cosmosis_config/values_template.ini create mode 100644 cosmo_inference/get_chi2.ipynb create mode 100644 cosmo_inference/get_chi2_cell.ipynb create mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/S8_om_sigma8_whisker.ipynb create mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/best_fit_xipm.ipynb create mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/contours.ipynb create mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2.ipynb create mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2_glass_mock.ipynb create mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_prior_psf_leakage.ipynb create mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/glass_mock_hist.ipynb create mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/masking.ipynb create mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/nonlin_k_analysis.ipynb create mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_unblinding/unblinding_party_plots.py create mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_unblinding/utils.py create mode 100644 cosmo_inference/notebooks/cfis_analysis.ipynb create mode 100644 cosmo_inference/notebooks/cfis_mcmc.ipynb create mode 100644 cosmo_inference/notebooks/get_prior_psf_leakage.ipynb create mode 100755 cosmo_inference/pipeline.sh create mode 100644 cosmo_inference/scripts/2pt_like_xi_sys.py create mode 100644 cosmo_inference/scripts/cosmocov_process.py delete mode 100644 cosmo_inference/scripts/k_analysis.py create mode 100644 cosmo_inference/scripts/masking.py create mode 100644 cosmo_inference/scripts/matching.py create mode 100644 cosmo_inference/scripts/nz_writeout.py create mode 100644 cosmo_inference/scripts/slurm.sh create mode 100644 cosmo_inference/scripts/treecorr_calc.py create mode 100644 cosmo_inference/scripts/xi_sys_psf.py delete mode 100644 papers/realspace/S8_om_sigma8_whisker.py delete mode 100644 papers/realspace/best_fit_xipm.py delete mode 100644 papers/realspace/contours.py delete mode 100644 papers/realspace/cov_masking.py delete mode 100644 papers/realspace/get_chi2.py delete mode 100644 papers/realspace/get_chi2_glass_mock.py delete mode 100644 papers/realspace/get_prior_psf_leakage.py delete mode 100644 papers/realspace/glass_mock_hist.py delete mode 100644 papers/realspace/nonlin_k_analysis.py delete mode 100644 scripts/compute_m_bias_image_sims.py delete mode 100644 scripts/diagnostics_image_sims.py create mode 100644 scripts/patch_firecrown.py create mode 100644 src/sp_validation/cosmo_val/sacc_writers.py delete mode 100644 src/sp_validation/image_sims.py create mode 100644 src/sp_validation/tests/test_assemble_sacc.py create mode 100644 src/sp_validation/tests/test_cli_seams.py delete mode 100644 src/sp_validation/tests/test_image_sims.py delete mode 100644 src/sp_validation/tests/test_mask_overlay.py delete mode 100644 src/sp_validation/tests/test_masks.py create mode 100644 src/sp_validation/tests/test_sacc_writers.py create mode 100644 uv-overrides.txt delete mode 100644 uv.lock delete mode 100644 workflow/image_sims/Snakefile delete mode 100644 workflow/image_sims/config.yaml delete mode 100644 workflow/image_sims/params_im_sim.py delete mode 100644 workflow/profiles/candide/config.yaml delete mode 100644 workflow/rules/image_sims.smk create mode 100644 workflow/scripts/assemble_sacc.py create mode 100644 workflow/scripts/generate_cosmocov_ini.py delete mode 100644 workflow/scripts/im_build_manifest.py delete mode 100644 workflow/scripts/im_compose_mask.py create mode 100644 workflow/scripts/run_cosmocov_chain.sh diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 192e87a6..13f0fafe 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -44,6 +44,12 @@ jobs: - name: Import smoke test run: docker run --rm ${{ steps.meta.outputs.tags }} python -c "import sp_validation" + # The fast suite doesn't import the blinding stack, so a broken + # firecrown/smokescreen install would otherwise ship green. Prove the + # image can actually load it (sacc + patched firecrown + smokescreen). + - name: Blinding-stack import smoke test + run: docker run --rm ${{ steps.meta.outputs.tags }} python -c "import sacc; import firecrown.likelihood; import smokescreen" + # Run the fast test suite against the freshly-built image *before* # pushing, so a failing suite blocks publication. The image carries the # full stack and the test files (COPY . + editable install), so this diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a0f36765..c2be73bd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,34 +1,21 @@ name: Lint -# Lint gate for `develop` — warn locally, fix-and-account here. +# Lint gate for `develop` — warn locally, account here. # -# The model (settled with Cail + Sacha, 2026-06-23/30; autofix added 2026-07-10): +# The model (settled with Cail + Sacha, 2026-06-23/30): # * Locally, ruff auto-applies safe fixes and only WARNS on the rest (see # .pre-commit-config.yaml) — you commit freely. # * Getting into `develop` is gated. This job runs the FULL ruff policy -# (`ruff check .` + `ruff format --check .`, region-aware per pyproject.toml). -# What happens next depends on the event: -# -# - On a same-repo PR (head branch lives in THIS repo, not a fork) → the gate -# doesn't just report; it FIXES. If the first pass isn't clean it runs -# `ruff check --fix-only` + `ruff format`, commits the diff as the -# github-actions bot, and pushes it back to the PR branch. Then it re-runs -# ruff on the fixed tree IN THE SAME RUN and gates on THAT: if formatting + -# safe fixes cleaned everything → job GREEN, comment says autofix was -# pushed; if anything survives (unsafe/judgement lint — undefined names, -# unused vars) → job RED, comment lists ONLY the residual (the mechanical -# stuff is already fixed). Contributors mostly never touch ruff by hand. -# -# - On a fork PR (read-only token, can't push to the fork's branch) → the old -# behaviour: run the checks, and on failure (1) fail RED so the check blocks -# the merge, (2) post/update a COMMENT on the PR with the full violation -# list. The comment turns green when ruff passes. -# -# - On a direct push to develop → there's no PR to comment on, so on failure -# it opens (or updates) ONE lint-debt issue for the committer, @-mentioning -# and assigning them. Auto-closes when their next push is clean. (No autofix -# here — pushing a bot commit onto develop would be its own event.) -# +# (`ruff check .` + `ruff format --check .`, region-aware per pyproject.toml) +# and on any failure it (1) fails the job RED so the check blocks the merge, +# and (2) tells the author what to fix — in the surface that fits the event: +# - On a PR → posts/updates a COMMENT on the PR itself, with the full +# violation list (and ruff annotations in the run's Checks view). The +# author sees it where they already are; no disconnected issue. The +# comment turns green when ruff passes. +# - On a direct push to develop → there's no PR to comment on, so it opens +# (or updates) ONE lint-debt issue for the committer, @-mentioning and +# assigning them. Auto-closes when their next push is clean. # * If ruff itself can't run (network, bad version), the job goes red but says # nothing — that's infra, not the committer's lint debt. # @@ -36,25 +23,17 @@ name: Lint # branches stay quiet (too noisy otherwise). `workflow_dispatch` is for manual # testing of the gate itself. # -# Why `pull_request_target` for PRs: commenting on (and pushing to) a PR needs a -# write token, and PRs from forks (e.g. sachaguer/) get a read-only token under -# the plain `pull_request` event. `pull_request_target` runs this workflow from -# the BASE branch (so the workflow definition is trusted) with a write token, -# while we check out the PR head ONLY to lint it. Two hardening measures make -# running tooling over untrusted PR code safe here: ruff is a static analyzer (it -# parses files, never imports/executes them), and `uvx --no-config` makes uv -# ignore any `uv.toml`/`[tool.uv]` in the PR tree, so a malicious PR can't -# redirect ruff's download to a trojaned index. The checkout also drops its git -# credentials. (A PR can still edit `[tool.ruff]` to weaken its own policy, but -# that's visible in the diff and reviewed like any other change.) -# -# Why the autofix push is safe under pull_request_target: we run ONLY ruff over -# the untrusted tree (static, never executes PR code), and we push back ONLY the -# diff ruff itself produced — no PR-authored script runs with our write token. -# The push uses the workflow token explicitly (the checkout keeps -# persist-credentials: false), and a GITHUB_TOKEN push does NOT trigger a new -# workflow run — so no recursion, but also no fresh CI on the bot commit, which -# is exactly why we re-lint and gate in THIS run rather than waiting for a rerun. +# Why `pull_request_target` for PRs: commenting on a PR needs a write token, and +# PRs from forks (e.g. sachaguer/) get a read-only token under the plain +# `pull_request` event. `pull_request_target` runs this workflow from the BASE +# branch (so the workflow definition is trusted) with a write token, while we +# check out the PR head ONLY to lint it. Two hardening measures make running +# tooling over untrusted PR code safe here: ruff is a static analyzer (it parses +# files, never imports/executes them), and `uvx --no-config` makes uv ignore any +# `uv.toml`/`[tool.uv]` in the PR tree, so a malicious PR can't redirect ruff's +# download to a trojaned index. The checkout also drops its git credentials. +# (A PR can still edit `[tool.ruff]` to weaken its own policy, but that's visible +# in the diff and reviewed like any other change.) on: push: @@ -69,7 +48,7 @@ concurrency: cancel-in-progress: true permissions: - contents: write # push ruff autofix commit back to a same-repo PR branch + contents: read issues: write # develop-push lint-debt issue pull-requests: write # PR lint comment @@ -88,19 +67,6 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v3 - # Is this a PR whose head branch lives in THIS repo (not a fork)? Only then - # can we push an autofix commit back to it with the workflow token. - - name: Decide whether autofix can push - id: mode - shell: bash - run: | - if [ "${{ github.event_name }}" = "pull_request_target" ] && \ - [ "${{ github.event.pull_request.head.repo.full_name }}" = "${{ github.repository }}" ]; then - echo "autofix=true" >> "$GITHUB_OUTPUT" - else - echo "autofix=false" >> "$GITHUB_OUTPUT" - fi - # Run the checks WITHOUT failing the step — we post feedback before turning # the job red. `ruff@` matches the pre-commit version; `--no-config` # neutralizes any uv config in the (untrusted) PR tree. @@ -136,109 +102,20 @@ jobs: echo "passed=false" >> "$GITHUB_OUTPUT" fi - # ── Autofix (same-repo PRs only) ────────────────────────────────────────── - # The first pass found something on a branch we can push to: apply ruff's - # own fixes (safe lint fixes + formatting), commit the diff as the bot, and - # push it back. SAFE under pull_request_target: only ruff runs over the PR - # tree (static, never executes it), and only ruff's own diff is pushed — no - # PR-authored code touches our write token. Then re-lint the FIXED tree in - # this same run: a GITHUB_TOKEN push doesn't trigger a new workflow, so the - # residual pass/fail we compute here is what the gate reports. - - name: Ruff autofix + push (same-repo PR) - id: autofix - if: >- - steps.mode.outputs.autofix == 'true' && - steps.ruff.outputs.tool_error == 'false' && - steps.ruff.outputs.passed == 'false' - shell: bash - run: | - # `check --fix-only` exits 1 when unfixable violations remain even - # after applying every safe fix, so don't let that abort the step. - uvx --no-config ruff@0.15.18 check --fix-only . || true - uvx --no-config ruff@0.15.18 format . || true - - set -e - if git diff --quiet; then - # ruff couldn't fix anything (all issues are unsafe/judgement calls): - # nothing to push. The gate falls back to the first-pass result and - # the original report already lists these — no autofix comment. - echo "pushed=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git add -A - git commit -m "ruff autofix (format + safe lint fixes)" \ - -m "Pushed by the lint gate." - - # Push back to the PR HEAD branch. The checkout kept - # persist-credentials: false, so authenticate the push explicitly with - # the workflow token via the remote URL. - BRANCH='${{ github.event.pull_request.head.ref }}' - REPO='${{ github.event.pull_request.head.repo.full_name }}' - git push "https://x-access-token:${{ github.token }}@github.com/${REPO}.git" "HEAD:${BRANCH}" - echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - echo "pushed=true" >> "$GITHUB_OUTPUT" - - # Re-lint the FIXED tree — this is what the gate reports (no rerun comes). - set +e - uvx --no-config ruff@0.15.18 check . --output-format=concise > check2.txt 2>&1; check_rc=$? - uvx --no-config ruff@0.15.18 format --check . > format2.txt 2>&1; fmt_rc=$? - set -e - { - echo "### Residual \`ruff check .\` (after autofix)" - if [ "$check_rc" -eq 0 ]; then echo; echo "✅ clean"; else echo; echo '```'; cat check2.txt; echo '```'; fi - echo - echo "### Residual \`ruff format --check .\` (after autofix)" - if [ "$fmt_rc" -eq 0 ]; then echo; echo "✅ clean"; else echo; echo '```'; cat format2.txt; echo '```'; fi - } > residual.md - - if [ "$check_rc" -eq 0 ] && [ "$fmt_rc" -eq 0 ]; then - echo "residual_passed=true" >> "$GITHUB_OUTPUT" - else - echo "residual_passed=false" >> "$GITHUB_OUTPUT" - fi - - # Resolve the outcome the gate reports. For a same-repo PR that got an - # autofix push, the residual pass/fail on the FIXED tree supersedes the - # first pass (that's what's now on the branch); otherwise the first pass - # stands. `pushed`/`residual` are surfaced so the comment can say so. - - name: Resolve gate outcome - id: gate - if: steps.ruff.outputs.tool_error == 'false' - shell: bash - run: | - if [ "${{ steps.autofix.outputs.pushed }}" = "true" ]; then - echo "passed=${{ steps.autofix.outputs.residual_passed }}" >> "$GITHUB_OUTPUT" - echo "autofixed=true" >> "$GITHUB_OUTPUT" - echo "sha=${{ steps.autofix.outputs.sha }}" >> "$GITHUB_OUTPUT" - echo "report_file=residual.md" >> "$GITHUB_OUTPUT" - else - echo "passed=${{ steps.ruff.outputs.passed }}" >> "$GITHUB_OUTPUT" - echo "autofixed=false" >> "$GITHUB_OUTPUT" - echo "report_file=report.md" >> "$GITHUB_OUTPUT" - fi - # Feedback is a side effect — never let it red a clean run. - name: Tell the author (PR comment) or record it (develop-push issue) if: steps.ruff.outputs.tool_error == 'false' continue-on-error: true uses: actions/github-script@v7 env: - PASSED: ${{ steps.gate.outputs.passed }} - AUTOFIXED: ${{ steps.gate.outputs.autofixed }} - AUTOFIX_SHA: ${{ steps.gate.outputs.sha }} - REPORT_FILE: ${{ steps.gate.outputs.report_file }} + PASSED: ${{ steps.ruff.outputs.passed }} with: script: | const fs = require('fs'); const passed = process.env.PASSED === 'true'; - const autofixed = process.env.AUTOFIXED === 'true'; - const autofixSha = (process.env.AUTOFIX_SHA || '').slice(0, 7); const { owner, repo } = context.repo; const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; - const report = passed ? '' : fs.readFileSync(process.env.REPORT_FILE, 'utf8'); + const report = passed ? '' : fs.readFileSync('report.md', 'utf8'); // ---- PR: speak on the PR itself (comment, auto-updating) ---------- if (context.eventName === 'pull_request_target') { @@ -251,35 +128,22 @@ jobs: const mine = comments.find(c => c.body && c.body.includes(MARKER)); if (passed) { - // Clean now. If we got here by pushing an autofix, say so (the - // push is why the branch changed under the author). Otherwise - // only update an existing comment to green — don't post on a PR + // Only update an existing comment to green; don't post on a PR // that was never dirty. - const body = autofixed - ? `🤖 **autofix pushed \`${autofixSha}\`, ruff is clean** — formatting and safe lint fixes were applied for you; nothing else to do. ${MARKER}` - : `✅ **ruff is clean** — nothing to fix here. ${MARKER}`; if (mine) { - await github.rest.issues.updateComment({ owner, repo, comment_id: mine.id, body }); - } else if (autofixed) { - await github.rest.issues.createComment({ owner, repo, issue_number: pr.number, body }); + await github.rest.issues.updateComment({ + owner, repo, comment_id: mine.id, + body: `✅ **ruff is clean** — nothing to fix here. ${MARKER}`, + }); } core.info('PR clean.'); return; } - const intro = autofixed - ? [ - `### 🔴 ruff — residual issues after autofix`, - ``, - `@${author} — I pushed \`${autofixSha}\` with the formatting and safe lint fixes, but these need a human and still block the merge into \`develop\`:`, - ] - : [ - `### 🔴 ruff found lint / format issues`, - ``, - `@${author} — these block the merge into \`develop\`. Full list below (also surfaced as annotations in the CI run):`, - ]; const body = [ - ...intro, + `### 🔴 ruff found lint / format issues`, + ``, + `@${author} — these block the merge into \`develop\`. Full list below (also surfaced as annotations in the CI run):`, ``, report, ``, @@ -369,11 +233,8 @@ jobs: # Red → blocks the merge. `always()` so a hiccup in the feedback step above # can't suppress the red on genuine lint debt; a tooling error also reds. - # A tooling error means `gate` was skipped (its outputs are empty), so test - # it first; otherwise the resolved gate outcome (post-autofix on same-repo - # PRs, first-pass elsewhere) decides. - name: Fail the job if the gate didn't pass - if: always() && (steps.ruff.outputs.tool_error == 'true' || steps.gate.outputs.passed == 'false') + if: always() && steps.ruff.outputs.passed == 'false' run: | if [ "${{ steps.ruff.outputs.tool_error }}" = "true" ]; then echo "::error::ruff could not run (network / version) — gate inconclusive, blocking." diff --git a/.gitignore b/.gitignore index 82fb2697..f8eec899 100644 --- a/.gitignore +++ b/.gitignore @@ -122,7 +122,7 @@ venv.bak/ # and the lockfile has never been tracked. Ignore it rather than commit a # pinned-dep reproducibility promise the project hasn't made. Flip to tracked # if we decide to pin deps via uv. -# uv.lock is committed — it is the reproducible pin the container installs from. +uv.lock # Spyder project settings .spyderproject @@ -177,10 +177,13 @@ cosmo_inference/cosmosis_config/glass_mocks_v* # repo, so these are script/notebook outputs, not LaTeX-tracked figures. papers/catalog/plots/*.pdf -# felt fiber store: canonical copy lives in ~/loom (git-synced privately); -# .felt here is a machine-local symlink into it. Never track it in this repo. -/.felt/ -/.felt +# felt — track the fiber records (engineering decisions & findings); skip only +# the regenerable index and runtime locks. +.felt/*.db +.felt/*.db-shm +.felt/*.db-wal +.felt/*.lock +.felt/index-sync.* # Claude Code agent worktrees — transient isolated checkouts for background # agents; never tracked. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 00a8d907..35d5e659 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,26 +70,17 @@ undefined names, unused variables, other judgement calls — is printed as a **warning** and never blocks the commit. Judgement-call lint stays out of your way locally; the gate below is where it's enforced. -**`develop` is the gate — and on a PR it fixes for you.** On every push to -`develop` and every PR into it, CI runs the full ruff policy. - -- **On a PR from a branch in this repo** → the gate doesn't just report, it - **fixes**: it runs `ruff format` + `ruff check --fix` and **pushes the result - back to your branch as the `github-actions` bot**, then re-checks. If that - cleaned everything, the PR comment goes green (`🤖 autofix pushed …, ruff is - clean`) and there's nothing to do — just `git pull` to pick up the commit. If - anything ruff *won't* safely fix survives (undefined names, unused variables, - other judgement calls), the check stays **red** and the comment lists **only - the residual** — the mechanical stuff is already handled. So you rarely touch - ruff by hand; when you do, it's the real judgement calls. -- **On a PR from a fork** (where CI can't push to your branch) → it posts (and - keeps updating) a **comment on the PR** with the full violation list. Push a - fix and the comment turns green. +**`develop` is the gate.** On every push to `develop` and every PR into it, CI +runs the full ruff policy. If it fails, the check goes **red and blocks the +merge**, and the bot tells you what to fix where you already are: + +- **On a PR** → it posts (and keeps updating) a **comment on the PR** listing the + violations (also surfaced as annotations in the CI run). Push a fix and the + comment turns green. - **On a direct push to `develop`** (no PR) → it opens (or updates) a single **lint-debt issue assigned to you**, which auto-closes when CI is green. -So: warn while you work, and for same-repo PRs the gate mostly cleans up after -you before it lands. +So: warn while you work, clean before it lands. ## Commit hygiene (notebooks & large files) diff --git a/Dockerfile b/Dockerfile index ccc930bf..818db67d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Development image with more bells and whistles -FROM ghcr.io/cosmostat/shapepipe:im_sims +FROM ghcr.io/cosmostat/shapepipe:develop RUN apt-get update -y --quiet --fix-missing && \ apt-get dist-upgrade -y --quiet --fix-missing && \ @@ -12,28 +12,41 @@ RUN apt-get update -y --quiet --fix-missing && \ npm \ tmux -# The base shapepipe image provides a uv-managed venv at /app/.venv (exported as -# VIRTUAL_ENV); install sp_validation's deps into that same venv rather than -# spawning a second one under /sp_validation. -ENV UV_PROJECT_ENVIRONMENT=/app/.venv +# The base image installs into the system interpreter (/usr/local); use `uv pip` +# so the heavy scientific stack and our deps land where `python` resolves. +RUN uv pip install --no-cache-dir \ + snakemake + +# The base shapepipe image ships cs_util 0.1.9, and `uv pip install -e` does NOT +# upgrade an already-satisfied dependency to meet a *new* lower bound (astral-sh/uv +# #8410). sp_validation now needs `cs_util.size` (cs_util>=0.2.1), so upgrade it +# explicitly here — otherwise the editable install silently keeps 0.1.9 and the +# galaxy import smoke test fails. shear_psf_leakage@develop allows cs-util<0.3, +# so 0.2.x satisfies the whole graph. +RUN uv pip install --no-cache-dir --upgrade 'cs_util>=0.2.1' WORKDIR /sp_validation +COPY . /sp_validation -# uv.lock is the SSOT: `uv sync --frozen` installs exactly what it pins, so an -# image build can never silently re-resolve and drift a base-image version (the -# numpy-past-numba drift this lockfile exists to prevent). `--inexact` keeps the -# base image's ShapePipe stack (shapepipe, ngmix, galsim, …) — packages not in -# our lock — instead of pruning them. Copy the lock + manifest first so this -# layer caches independently of source edits. Extras: test (CI unit suite), -# glass (GLASS map-level mock — pulls glass.ext.camb + the cosmology wrapper), -# workflow (Snakemake + mpi4py runners). cs_util 0.2.2 (with cs_util.size) and a -# numba-safe numpy 2.4.6 come straight from the lock, so the old ad-hoc snakemake -# and cs_util `--upgrade` layers are gone. -COPY pyproject.toml uv.lock /sp_validation/ -RUN uv sync --frozen --inexact --no-install-project \ - --extra test --extra glass --extra workflow +# Install with the test + glass + blinding extras so the image can run the unit +# suite in CI, the GLASS map-level mock test, *and* the SACC/Smokescreen blinding +# stack. `glass` (Generator for Large Scale Structure) ships `glass.ext.camb`; +# `cosmology` provides the `Cosmology` wrapper (`Cosmology.from_camb`) GLASS +# consumes. The `[blinding]` extra (firecrown + smokescreen) needs the override +# file: firecrown declares conda-forge-only / unused sampler connectors as hard +# deps — see uv-overrides.txt for the full story. +RUN uv pip install --no-cache-dir --overrides uv-overrides.txt -e '.[test,glass,blinding]' -# Install sp_validation itself (editable) into the same venv; deps are already -# satisfied by the sync above. -COPY . /sp_validation -RUN uv pip install --no-deps -e . +# Same uv gotcha as the cs_util upgrade above (astral-sh/uv #8410): if the base +# image already carries a numpy that violates the [blinding] extra's new +# `numpy<2.5` cap (firecrown 1.15.1 breaks on numpy 2.5 at import), the +# editable install won't move it. Request the bound explicitly so the image is +# deterministic either way; numpy 2.4.x is ABI-compatible with the compiled +# stack (verified: pyccl/camb/treecorr/healpy/pymaster + fast suite). +RUN uv pip install --no-cache-dir 'numpy>=2.2,<2.5' + +# firecrown is distributed for conda-forge (where NumCosmo always exists) and +# hits NumCosmo at import time in a pip env, on paths unrelated to our use. +# This patches the installed tree (surgical, pinned-version-checked, loud on +# mismatch) and verifies `import firecrown.likelihood; import smokescreen`. +RUN python scripts/patch_firecrown.py diff --git a/README.md b/README.md index e14a2071..ef9e601b 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,26 @@ docker run --rm -it ghcr.io/cosmostat/sp_validation:develop python -c "import sp We do not currently build images for Apple Silicon/arm64; however the amd64 images should work on these systems, albeit with reduced performance. +## Local Installation + +Requires Python ≥ 3.12 (the floor is set by the blinding stack; the container +already runs 3.12). With [uv](https://docs.astral.sh/uv/): + +```bash +uv venv --python 3.12 +uv pip install -e '.[test]' +``` + +To also install the data-vector blinding stack (Smokescreen + firecrown, PRD +[#241](https://github.com/CosmoStat/sp_validation/issues/241)), pass the +dependency-override file — firecrown is not pip-resolvable without it (see +`uv-overrides.txt` for why): + +```bash +uv pip install --overrides uv-overrides.txt -e '.[test,blinding]' +python scripts/patch_firecrown.py # make pip-installed firecrown importable without NumCosmo +``` + ## Flow chart diff --git a/config/calibration/mask_v1.X.4_im_sim.yaml b/config/calibration/mask_v1.X.4_im_sim.yaml deleted file mode 100644 index 794566d4..00000000 --- a/config/calibration/mask_v1.X.4_im_sim.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# Config file for masking and calibration. -# Standard cuts without coverage mask, type v1.X.4. - -# General parameters (can also given on command line) -params: - input_path: shape_catalog_comprehensive_ngmix.hdf5 - cmatrices: False - sky_regions: False - verbose: True - -# Masks -## Using columns in 'dat' group (ShapePipe flags) -dat: - # SExtractor flags - - col_name: FLAGS - label: SE FLAGS - kind: equal - value: 0 - - # Number of epochs - - col_name: N_EPOCH - label: r"$n_{\rm epoch}$" - kind: greater_equal - value: 2 - - # Magnitude range - - col_name: mag - label: mag range - kind: range - value: [15, 30] - - # ngmix flags - - col_name: NGMIX_MOM_FAIL - label: "ngmix moments failure" - kind: equal - value: 0 - - # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 - label: "bad PSF ellipticity comp 1" - kind: not_equal - value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 - label: "bad PSF ellipticity comp 2" - kind: not_equal - value: -10 - -# Metacal parameters -metacal: - # Ellipticity dispersion - sigma_eps_prior: 0.34 - - # Signal-to-noise range - gal_snr_min: 10 - gal_snr_max: 500 - - # Relative-size (hlr / hlr_psf) range - gal_rel_size_min: 0.5 - gal_rel_size_max: 3 - - # Correct relative size for ellipticity? - gal_size_corr_ell: False - - # Weight for global response matrix, None for unweighted mean. - # Unweighted for image sims: no weights anywhere in sim m-bias (#227). - global_R_weight: null - - # Subtract additive bias (mean shear)? Use False for constant-shear - # image sims - additive_correction: False diff --git a/config/calibration/mask_v1.X.9_im_sim.overlay.yaml b/config/calibration/mask_v1.X.9_im_sim.overlay.yaml deleted file mode 100644 index 94c5b9ae..00000000 --- a/config/calibration/mask_v1.X.9_im_sim.overlay.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# Declared delta: image-sim mask/calibration config vs the data config. -# -# The image-sim calibration reuses the *data* mask config -# (mask_v1.X.9.yaml) and changes only what the sims genuinely differ on. -# Rather than maintain a second full copy that can silently drift from the -# base, this overlay states -- as a list of block operations on the base -# file -- exactly which pieces the sims drop or change, and one line of why -# for each. `im_compose_mask.py` applies these ops to mask_v1.X.9.yaml and -# reproduces mask_v1.X.9_im_sim.yaml byte-for-byte; a test locks that, so the -# runtime file and this declaration cannot diverge. -# -# Each op anchors to a block of base text (matched verbatim, and required to -# occur exactly once) and either drops it or replaces it. `why` is prose for -# the human reader; the compose ignores it. Ordering follows the base file. - -base: mask_v1.X.9.yaml - -ops: - # --- params ------------------------------------------------------------ - - why: >- - Sims read the ShapePipe FITS catalogue staged in the run dir, not the - survey-wide comprehensive HDF5 on /n17data. - replace: | - input_path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_comprehensive_struc_2024_v1.X.c.hdf5 - with: | - input_path: shape_catalog_comprehensive_ngmix.fits - - # --- dat cuts ---------------------------------------------------------- - - why: >- - No ShapePipe coverage/mask flags on sims: IMAFLAGS_ISO is a survey - artefact (external masks, bright-star haloes) the sims do not carry. - drop: |2 - - # ShapePipe flags - - col_name: IMAFLAGS_ISO - label: SP mask - kind: equal - value: 0 - - - why: >- - Same cuts, but flag the grammar: the sims run the ShapePipe-v2 PSF - columns (scalar G1/G2), so the comment is made explicit here. - replace: |2 - # invalid PSF ellipticities - with: |2 - # invalid PSF ellipticities (ShapePipe-v2 grammar: scalar G1/G2 components) - - # --- dat_ext (post-processing / coverage masks) ------------------------ - - why: >- - No coverage masks on sims: the whole dat_ext group (Stars, manual mask, - r-band footprint, Maximask) is survey post-processing with no analogue - in the simulated tiles. - drop: |2 - - ## Using columns in 'dat_ext' group (post-processing flags) - dat_ext: - - # Stars - - col_name: 4_Stars - label: "Stars" - kind: equal - value: False - - # Manual mask - - col_name: 8_Manual - label: "manual mask" - kind: equal - value: False - - # r-band footprint - - col_name: 64_r - label: "r-band imaging" - kind: equal - value: False - - # Maximask - - col_name: 1024_Maximask - label: "maximask" - kind: equal - value: False - - # --- metacal ----------------------------------------------------------- - - why: >- - Unweighted for image sims: no weights anywhere in the sim m-bias (#227), - and the w_des-weighted global R had only N_eff ~ 20-100 objects. - replace: |2 - # Weight for global response matrix, None for unweighted mean - global_R_weight: w - with: |2 - # Weight for global response matrix, None for unweighted mean. - # Unweighted for image sims: no weights anywhere in sim m-bias (#227), - # and the w_des-weighted R had N_eff ~ 20-100 objects. - global_R_weight: null - - # Subtract additive bias (mean shear)? Use False for constant-shear - # image sims - additive_correction: False diff --git a/config/calibration/mask_v1.X.9_im_sim.yaml b/config/calibration/mask_v1.X.9_im_sim.yaml deleted file mode 100644 index 1c6ff672..00000000 --- a/config/calibration/mask_v1.X.9_im_sim.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# Config file for masking and calibration. -# Less conservative cuts, type v1.X.9 (e.g. for matching with spectroscopic sample). - -# General parameters (can also given on command line) -params: - input_path: shape_catalog_comprehensive_ngmix.fits - cmatrices: False - sky_regions: False - verbose: True - -# Masks -## Using columns in 'dat' group (ShapePipe flags) -dat: - # SExtractor flags - - col_name: FLAGS - label: SE FLAGS - kind: smaller_equal - value: 2 - - # Duplicate objects - - col_name: overlap - label: tile overlap - kind: equal - value: True - - # Number of epochs - - col_name: N_EPOCH - label: r"$n_{\rm epoch}$" - kind: greater_equal - value: 1 - - # Magnitude range - - col_name: mag - label: mag range - kind: range - value: [15, 30] - - # ngmix flags - - col_name: NGMIX_MCAL_TYPES_FAIL - label: "ngmix moments failure" - kind: equal - value: 0 - - # invalid PSF ellipticities (ShapePipe-v2 grammar: scalar G1/G2 components) - - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR - label: "bad PSF ellipticity comp 1" - kind: not_equal - value: -10 - - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR - label: "bad PSF ellipticity comp 2" - kind: not_equal - value: -10 - -# Metacal parameters -metacal: - # Ellipticity dispersion - sigma_eps_prior: 0.34 - - # Signal-to-noise range - gal_snr_min: 5 - gal_snr_max: 500 - - # Relative-size (hlr / hlr_psf) range - gal_rel_size_min: 0.25 - gal_rel_size_max: 10 - - # Correct relative size for ellipticity? - gal_size_corr_ell: False - - # Weight for global response matrix, None for unweighted mean. - # Unweighted for image sims: no weights anywhere in sim m-bias (#227), - # and the w_des-weighted R had N_eff ~ 20-100 objects. - global_R_weight: null - - # Subtract additive bias (mean shear)? Use False for constant-shear - # image sims - additive_correction: False diff --git a/cosmo_inference/.gitignore b/cosmo_inference/.gitignore index cb5922b7..0b53f63e 100644 --- a/cosmo_inference/.gitignore +++ b/cosmo_inference/.gitignore @@ -1,7 +1,6 @@ plots/ .ipynb_checkpoints/ data/ -cosmosis_config/output/* *.png *.pdf *.sh \ No newline at end of file diff --git a/cosmo_inference/README.md b/cosmo_inference/README.md index 5d753010..6da1b94c 100644 --- a/cosmo_inference/README.md +++ b/cosmo_inference/README.md @@ -4,7 +4,7 @@ by Lisa Goh and Sacha Guerrini, CEA Paris-Saclay This folder contains the files neccessary to run the cosmological inference pipeline on the UNIONS galaxy catalogues. ### Requirements -To run the pipeline, one would need to have installed [CosmoSIS](https://cosmosis.readthedocs.io/en/latest/). To sample the PSF leakage parameters, the fork of [cosmosis-standard-library](https://github.com/sachaguer/cosmosis-standard-library/) of Sacha Guerrini has to be used. +To run the pipeline, one would need to have installed [CosmoSIS](https://cosmosis.readthedocs.io/en/latest/) and [CosmoCov](https://github.com/CosmoLike/CosmoCov). To PSF leakage parameters, the fork of [cosmosis-standard-library](https://github.com/sachaguer/cosmosis-standard-library/) of Sacha Guerrini has to be used. ### To Run The inference pipeline is now orchestrated through Python. Run the main Snakemake workflow from the parent directory: @@ -15,7 +15,7 @@ snakemake -j inference_fiducial This will automatically execute all steps: 1. Calculate 2PCF ($\xi_{pm}$) via `cosmo_val.py` -2. Compute covariance matrices using CosmoCov +2. Compute covariance matrices using CosmoCov 3. Prepare CosmoSIS data (FITS) via `cosmosis_fitting.py` 4. Run CosmoSIS inference diff --git a/cosmo_inference/cfis_pipeline.sh b/cosmo_inference/cfis_pipeline.sh new file mode 100644 index 00000000..dadab6fb --- /dev/null +++ b/cosmo_inference/cfis_pipeline.sh @@ -0,0 +1,66 @@ +#!/bin/bash +read -p 'SHEAR CATALOGUE: ' shear_cat +# read -p 'NZ CATALOGUE: ' nz_cat +read -p 'DATA ROOT: ' root +read -p 'OUT ROOT: ' out_root +# read -p 'BLIND:' blind +mkdir -p data/${root} + +# #################STEP 0: RUN NOTEBOOK TO ANALYSE CATALOGUE; DERIVE PLOTS################## +# #File: cfis_analysis.ipynb + +# ##################STEP 1: CALCULATE XIP/XIM (OUTPUTS TREECORR FITS CATALOG)############### +python treecorr_calc.py $shear_cat $root + +echo -e "2PCF's calculated!\n" + +# # ##################STEP 2: WRITE NZ's###################################################### +# python nz_writeout.py $nz_cat $root $blind + +# echo -e "nz's written out!\n" + +# # # # ##################STEP 3: ESTIMATE COVMATS################################################ + +# # #edit ini file +# mkdir -p data/${root}/covs + +# nz_file="data/${root}/nz_shapepipe_A.txt" + +# # sed -i "/shear_REDSHIFT_FILE/c shear_REDSHIFT_FILE : $nz_file" cosmocov.ini +# # sed -i "/clustering_REDSHIFT_FILE/c clustering_REDSHIFT_FILE : $nz_file" cosmocov.ini +# # sed -i "/outdir/c outdir : data/$root/covs/" cosmocov.ini + +# echo -e "Running CosmoCov...\n" + +# ##run cosmocov +# for i in {1..3}; +# do ../CosmoCov/covs/cov $i cosmocov.ini; +# done + +# # do postprocessing (plot covmat and write into txt file) +# f="data/${root}/covs/cov_${root}"; cat data/${root}/covs/out_cov* > $f; python cosmocov_process.py $f + +# # # # # ##################STEP 4: COMBINE########################################################## +# xip_cat="data/${root}/xiplus_${root}.fits" +# xim_cat="data/${root}/ximinus_${root}.fits" +# covmat="data/${root}/covs/cov_${root}.txt" + +# out_file="$PWD/data/${root}/cosmosis_${root}.fits" + +python cosmosis_fitting.py /n23data1/n06data/lgoh/scratch/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/data/SP_v1.4_A/xiplus_SP_v1.4_A.fits /n23data1/n06data/lgoh/scratch/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/data/SP_v1.4_A/ximinus_SP_v1.4_A.fits /n23data1/n06data/lgoh/scratch/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/data/SP_v1.4_A/covs/cov_SP_v1.4.txt /n23data1/n06data/lgoh/scratch/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/data/nz/nz_shapepipe_A.txt /n23data1/n06data/lgoh/scratch/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/data/SP_v1.4_A/cosmosis_SP_v1.4_A.fits + +# # # # ##################STEP 5: RUN COSMOSIS##################################################### +# echo -e "Running CosmoSIS...\n" + +# sed -i "/SCRATCH = /c SCRATCH = $WORK/UNIONS/chains/${out_root}/" cosmosis_config/cosmosis_pipeline.ini +# sed -i "/FITS_FILE = /c FITS_FILE = ${out_file}" cosmosis_config/cosmosis_pipeline.ini +# sed -i "/filename = /c filename = %(SCRATCH)s/samples_${out_root}.txt" cosmosis_config/cosmosis_pipeline.ini + +# #submit cosmosis job to run on cluster + +# sbatch -J cfis_${root} --output=$WORK/UNIONS/cfis_${out_root}.log slurm.sh + +# echo -e "-------------PIPELINE END----------------" + +# # ##################STEP 6: RUN NOTEBOOK TO ANALYSE CONTOURS (WITH GETDIST)################## +# #File: CFIS_plotting.ipynb \ No newline at end of file diff --git a/cosmo_inference/cosmocov_config/cosmocov.ini b/cosmo_inference/cosmocov_config/cosmocov.ini new file mode 100644 index 00000000..0afbdc64 --- /dev/null +++ b/cosmo_inference/cosmocov_config/cosmocov.ini @@ -0,0 +1,80 @@ +# +# Cosmological parameters +# +Omega_m : 0.25 +Omega_v : 0.75 +sigma_8 : 0.8 +n_spec : 0.95 +w0 : -1 +wa : 0 +omb : 0.044 +h0 : 0.7 + + +# Survey and galaxy parameters +# +# area in degrees +# n_gal,lens_n_gal in gals/arcmin^2 + +#FOR LENSFIT +#area : 2138 +#sourcephotoz : multihisto +#lensphotoz : multihisto +#source_tomobins : 1 +#lens_tomobins : 1 +#sigma_e : 0.41016433003564806 +#source_n_gal : 10.78 + +#FOR SHAPEPIPE +; area : 3218.19 +; sourcephotoz : multihisto +; lensphotoz : multihisto +; source_tomobins : 1 +; lens_tomobins : 1 +; sigma_e : 0.491712 +; source_n_gal : 8.42 + +#FOR SHAPEPIPE 1500 +; area : 1453 +; sourcephotoz : multihisto +; lensphotoz : multihisto +; source_tomobins : 1 +; lens_tomobins : 1 +; sigma_e : 0.4808326112068524 +; source_n_gal : 7.92 + +#FOR SHAPEPIPE v1.3/v1.4 +area : 2782 +sourcephotoz : multihisto +lensphotoz : multihisto +source_tomobins : 1 +lens_tomobins : 1 +sigma_e : 0.4370966656902571 +; source_n_gal: 7.6 #v1.3 +source_n_gal : 7.18 # v1.4.1 +lens_n_gal : 7.18 + +c_footprint_file: + + +# IA parameters +IA : 1 +A_ia : 0.0 +eta_ia : 0.0 + + +# Covariance paramters +# +# tmin,tmax in arcminutes +tmin : 0.1 +tmax : 250 +ntheta : 20 +ng : 1 +cng : 1 + + +#mkdir before running! +filename : out_cov +ss : true +ls : false +ll : false \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline.ini new file mode 100644 index 00000000..f6b11411 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline.ini @@ -0,0 +1,89 @@ +#parameters used elsewhere in this file +[DEFAULT] +# Specify the directory of your cosmological CosmoSIS library +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +modules = consistency camb load_nz_fits linear_alignment projection 2pt_shear add_xi_sys 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 +timing = T +debug = T + +[runtime] +sampler = metropolis +resume = T +verbosity = debug + +[output] +format = text +lock = F + +[metropolis] +samples = 10000000 + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=all +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=takahashi +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +get_kernel_peaks = F +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +like_name=2pt_like +add_xi_sys=T + +angle_range_XI_PLUS_1_1= 1.0 200.0 +angle_range_XI_MINUS_1_1= 1.0 200.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini new file mode 100644 index 00000000..eb3ab166 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini @@ -0,0 +1,108 @@ +#parameters used elsewhere in this file +[DEFAULT] +COSMOSIS_DIR = /n23data1/n06data/lgoh/scratch/cosmosis-standard-library_lisa + + +[pipeline] +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[polychord] +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[test] + +[output] +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +data_sets=XI_PLUS XI_MINUS +like_name=2pt_like + +angle_range_XI_PLUS_1_1= 10.0 200.0 +angle_range_XI_MINUS_1_1= 20.0 200.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_cell.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_cell.ini new file mode 100644 index 00000000..87f06064 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_cell.ini @@ -0,0 +1,105 @@ +#parameters used elsewhere in this file +[DEFAULT] +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] + + +[polychord] +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_psf.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_psf.ini new file mode 100644 index 00000000..f9f4da51 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_psf.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] + +[polychord] +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 12.0 83.0 +angle_range_XI_MINUS_1_1= 12.0 83.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini new file mode 100644 index 00000000..efcf75f7 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +FITS_FILE = data/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1/cosmosis_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.fits +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1 +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +priors = cosmosis_config/priors_psf.ini +values = cosmosis_config/values_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/test_new_pipeline + +[polychord] +polychord_outfile_root = SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1/samples_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +add_xi_sys=T +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini new file mode 100644 index 00000000..0af5b180 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +FITS_FILE = data/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1/cosmosis_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.fits +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1 +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +priors = cosmosis_config/priors_psf.ini +values = cosmosis_config/values_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/test_new_pipeline + +[polychord] +polychord_outfile_root = SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1/samples_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +add_xi_sys=T +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini new file mode 100644 index 00000000..d64ac64f --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 +FITS_FILE = data/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/cosmosis_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_psf.ini +priors = cosmosis_config/priors_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 + +[polychord] +polychord_outfile_root = SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/samples_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +add_xi_sys=T +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini new file mode 100644 index 00000000..34a7209f --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini @@ -0,0 +1,113 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 +FITS_FILE = data/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/cosmosis_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_psf.ini +priors = cosmosis_config/priors_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell + + +[polychord] +polychord_outfile_root = SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/samples_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT_CELL +cut_zeros=F +like_name=2pt_like \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_A_cell.ini new file mode 100755 index 00000000..70404b87 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_A_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_A +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_A/cosmosis_SP_v1.4.6.3_A.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_A_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_A/samples_SP_v1.4.6.3_A_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_A_cell.ini new file mode 100755 index 00000000..dad17f3d --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_A_cell.ini @@ -0,0 +1,111 @@ +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_A +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_A/cosmosis_SP_v1.4.6.3_leak_corr_A.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_A_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_A/samples_SP_v1.4.6.3_leak_corr_A_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode = power +lmax = 2508 +feedback = 0 +do_reionization = F +kmin = 1e-5 +kmax = 20.0 +nk = 200 +zmax = 5.0 +zmax_background = 5.0 +nz_background = 500 +halofit_version = mead2020_feedback +nonlinear = pk +neutrino_hierarchy = normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file = %(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear = T +position-shear = F +perbin = F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets = CELL_EE +data_file = %(FITS_FILE)s +gaussian_covariance = F +covmat_name = COVMAT +cut_zeros = F +like_name = 2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 + diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_A_cell.ini new file mode 100755 index 00000000..e8c48898 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_A_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_HMCode_nobar_A +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_HMCode_nobar_A/cosmosis_SP_v1.4.6.3_leak_corr_HMCode_nobar_A.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_HMCode_nobar_A_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_HMCode_nobar_A/samples_SP_v1.4.6.3_leak_corr_HMCode_nobar_A_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020 +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_A_cell.ini new file mode 100755 index 00000000..e686f245 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_A_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_OneCov_A +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_OneCov_A/cosmosis_SP_v1.4.6.3_leak_corr_OneCov_A.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_OneCov_A_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_OneCov_A/samples_SP_v1.4.6.3_leak_corr_OneCov_A_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_A_cell.ini new file mode 100755 index 00000000..dd46e3e7 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_A_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_halofit_A +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_halofit_A/cosmosis_SP_v1.4.6.3_leak_corr_halofit_A.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_halofit_A_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_halofit_A/samples_SP_v1.4.6.3_leak_corr_halofit_A_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=takahashi +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_A_cell.ini new file mode 100755 index 00000000..507b2f9a --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_A_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_include_large_scales_A +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_include_large_scales_A/cosmosis_SP_v1.4.6.3_leak_corr_include_large_scales_A.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_include_large_scales_A_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_include_large_scales_A/samples_SP_v1.4.6.3_leak_corr_include_large_scales_A_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 0.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_A_cell.ini new file mode 100755 index 00000000..028e875c --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_A_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=1Mpc_A +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=1Mpc_A/cosmosis_SP_v1.4.6.3_leak_corr_kmax=1Mpc_A.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=1Mpc_A_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=1Mpc_A/samples_SP_v1.4.6.3_leak_corr_kmax=1Mpc_A_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 500.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_A_cell.ini new file mode 100755 index 00000000..32c45ad5 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_A_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=3Mpc_A +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=3Mpc_A/cosmosis_SP_v1.4.6.3_leak_corr_kmax=3Mpc_A.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=3Mpc_A_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=3Mpc_A/samples_SP_v1.4.6.3_leak_corr_kmax=3Mpc_A_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 1800.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_A_cell.ini new file mode 100755 index 00000000..92d61b20 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_A_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=5Mpc_A +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=5Mpc_A/cosmosis_SP_v1.4.6.3_leak_corr_kmax=5Mpc_A.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=5Mpc_A_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=5Mpc_A/samples_SP_v1.4.6.3_leak_corr_kmax=5Mpc_A_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 2048.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_A_cell.ini new file mode 100755 index 00000000..aa3784c2 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_A_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_large_scales_A +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_large_scales_A/cosmosis_SP_v1.4.6.3_leak_corr_large_scales_A.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_large_scales_A_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_large_scales_A/samples_SP_v1.4.6.3_leak_corr_large_scales_A_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 800.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_A_cell.ini new file mode 100755 index 00000000..1aa71c5f --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_A_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_small_scales_A +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_small_scales_A/cosmosis_SP_v1.4.6.3_leak_corr_small_scales_A.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_small_scales_A_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_small_scales_A/samples_SP_v1.4.6.3_leak_corr_small_scales_A_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 800.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_B_cell.ini new file mode 100755 index 00000000..db67430b --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_B_cell.ini @@ -0,0 +1,111 @@ +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_B +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_B/cosmosis_SP_v1.4.6.3_B.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_B_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_B/samples_SP_v1.4.6.3_B_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode = power +lmax = 2508 +feedback = 0 +do_reionization = F +kmin = 1e-5 +kmax = 20.0 +nk = 200 +zmax = 5.0 +zmax_background = 5.0 +nz_background = 500 +halofit_version = mead2020_feedback +nonlinear = pk +neutrino_hierarchy = normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file = %(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear = T +position-shear = F +perbin = F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets = CELL_EE +data_file = %(FITS_FILE)s +gaussian_covariance = F +covmat_name = COVMAT +cut_zeros = F +like_name = 2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 + diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_B_cell.ini new file mode 100755 index 00000000..d1b15a8f --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_B_cell.ini @@ -0,0 +1,111 @@ +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_B +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_B/cosmosis_SP_v1.4.6.3_leak_corr_B.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_B_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_B/samples_SP_v1.4.6.3_leak_corr_B_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode = power +lmax = 2508 +feedback = 0 +do_reionization = F +kmin = 1e-5 +kmax = 20.0 +nk = 200 +zmax = 5.0 +zmax_background = 5.0 +nz_background = 500 +halofit_version = mead2020_feedback +nonlinear = pk +neutrino_hierarchy = normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file = %(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear = T +position-shear = F +perbin = F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets = CELL_EE +data_file = %(FITS_FILE)s +gaussian_covariance = F +covmat_name = COVMAT +cut_zeros = F +like_name = 2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 + diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_B_cell.ini new file mode 100755 index 00000000..ccc62f9f --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_B_cell.ini @@ -0,0 +1,111 @@ +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_HMCode_nobar_B +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_HMCode_nobar_B/cosmosis_SP_v1.4.6.3_leak_corr_HMCode_nobar_B.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_HMCode_nobar_B_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_HMCode_nobar_B/samples_SP_v1.4.6.3_leak_corr_HMCode_nobar_B_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode = power +lmax = 2508 +feedback = 0 +do_reionization = F +kmin = 1e-5 +kmax = 20.0 +nk = 200 +zmax = 5.0 +zmax_background = 5.0 +nz_background = 500 +halofit_version = mead2020 +nonlinear = pk +neutrino_hierarchy = normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file = %(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear = T +position-shear = F +perbin = F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets = CELL_EE +data_file = %(FITS_FILE)s +gaussian_covariance = F +covmat_name = COVMAT +cut_zeros = F +like_name = 2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 + diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_B_cell.ini new file mode 100755 index 00000000..20b637ea --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_B_cell.ini @@ -0,0 +1,111 @@ +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_OneCov_B +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_OneCov_B/cosmosis_SP_v1.4.6.3_leak_corr_OneCov_B.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_OneCov_B_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_OneCov_B/samples_SP_v1.4.6.3_leak_corr_OneCov_B_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode = power +lmax = 2508 +feedback = 0 +do_reionization = F +kmin = 1e-5 +kmax = 20.0 +nk = 200 +zmax = 5.0 +zmax_background = 5.0 +nz_background = 500 +halofit_version = mead2020_feedback +nonlinear = pk +neutrino_hierarchy = normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file = %(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear = T +position-shear = F +perbin = F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets = CELL_EE +data_file = %(FITS_FILE)s +gaussian_covariance = F +covmat_name = COVMAT +cut_zeros = F +like_name = 2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 + diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_B_cell.ini new file mode 100755 index 00000000..c3d99809 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_B_cell.ini @@ -0,0 +1,111 @@ +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_halofit_B +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_halofit_B/cosmosis_SP_v1.4.6.3_leak_corr_halofit_B.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_halofit_B_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_halofit_B/samples_SP_v1.4.6.3_leak_corr_halofit_B_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode = power +lmax = 2508 +feedback = 0 +do_reionization = F +kmin = 1e-5 +kmax = 20.0 +nk = 200 +zmax = 5.0 +zmax_background = 5.0 +nz_background = 500 +halofit_version = takahashi +nonlinear = pk +neutrino_hierarchy = normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file = %(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear = T +position-shear = F +perbin = F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets = CELL_EE +data_file = %(FITS_FILE)s +gaussian_covariance = F +covmat_name = COVMAT +cut_zeros = F +like_name = 2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 + diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_B_cell.ini new file mode 100755 index 00000000..d724c837 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_B_cell.ini @@ -0,0 +1,111 @@ +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_include_large_scales_B +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_include_large_scales_B/cosmosis_SP_v1.4.6.3_leak_corr_include_large_scales_B.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_include_large_scales_B_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_include_large_scales_B/samples_SP_v1.4.6.3_leak_corr_include_large_scales_B_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode = power +lmax = 2508 +feedback = 0 +do_reionization = F +kmin = 1e-5 +kmax = 20.0 +nk = 200 +zmax = 5.0 +zmax_background = 5.0 +nz_background = 500 +halofit_version = mead2020_feedback +nonlinear = pk +neutrino_hierarchy = normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file = %(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear = T +position-shear = F +perbin = F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets = CELL_EE +data_file = %(FITS_FILE)s +gaussian_covariance = F +covmat_name = COVMAT +cut_zeros = F +like_name = 2pt_like +angle_range_CELL_EE_1_1 = 0.0 1600.0 + diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_B_cell.ini new file mode 100755 index 00000000..5991d198 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_B_cell.ini @@ -0,0 +1,111 @@ +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=1Mpc_B +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=1Mpc_B/cosmosis_SP_v1.4.6.3_leak_corr_kmax=1Mpc_B.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=1Mpc_B_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=1Mpc_B/samples_SP_v1.4.6.3_leak_corr_kmax=1Mpc_B_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode = power +lmax = 2508 +feedback = 0 +do_reionization = F +kmin = 1e-5 +kmax = 20.0 +nk = 200 +zmax = 5.0 +zmax_background = 5.0 +nz_background = 500 +halofit_version = mead2020_feedback +nonlinear = pk +neutrino_hierarchy = normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file = %(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear = T +position-shear = F +perbin = F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets = CELL_EE +data_file = %(FITS_FILE)s +gaussian_covariance = F +covmat_name = COVMAT +cut_zeros = F +like_name = 2pt_like +angle_range_CELL_EE_1_1 = 300.0 500.0 + diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_B_cell.ini new file mode 100755 index 00000000..fb347172 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_B_cell.ini @@ -0,0 +1,111 @@ +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=3Mpc_B +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=3Mpc_B/cosmosis_SP_v1.4.6.3_leak_corr_kmax=3Mpc_B.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=3Mpc_B_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=3Mpc_B/samples_SP_v1.4.6.3_leak_corr_kmax=3Mpc_B_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode = power +lmax = 2508 +feedback = 0 +do_reionization = F +kmin = 1e-5 +kmax = 20.0 +nk = 200 +zmax = 5.0 +zmax_background = 5.0 +nz_background = 500 +halofit_version = mead2020_feedback +nonlinear = pk +neutrino_hierarchy = normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file = %(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear = T +position-shear = F +perbin = F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets = CELL_EE +data_file = %(FITS_FILE)s +gaussian_covariance = F +covmat_name = COVMAT +cut_zeros = F +like_name = 2pt_like +angle_range_CELL_EE_1_1 = 300.0 1800.0 + diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_B_cell.ini new file mode 100755 index 00000000..4005c10b --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_B_cell.ini @@ -0,0 +1,111 @@ +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=5Mpc_B +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=5Mpc_B/cosmosis_SP_v1.4.6.3_leak_corr_kmax=5Mpc_B.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=5Mpc_B_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=5Mpc_B/samples_SP_v1.4.6.3_leak_corr_kmax=5Mpc_B_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode = power +lmax = 2508 +feedback = 0 +do_reionization = F +kmin = 1e-5 +kmax = 20.0 +nk = 200 +zmax = 5.0 +zmax_background = 5.0 +nz_background = 500 +halofit_version = mead2020_feedback +nonlinear = pk +neutrino_hierarchy = normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file = %(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear = T +position-shear = F +perbin = F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets = CELL_EE +data_file = %(FITS_FILE)s +gaussian_covariance = F +covmat_name = COVMAT +cut_zeros = F +like_name = 2pt_like +angle_range_CELL_EE_1_1 = 300.0 2048.0 + diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_B_cell.ini new file mode 100755 index 00000000..95009a6a --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_B_cell.ini @@ -0,0 +1,111 @@ +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_large_scales_B +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_large_scales_B/cosmosis_SP_v1.4.6.3_leak_corr_large_scales_B.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_large_scales_B_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_large_scales_B/samples_SP_v1.4.6.3_leak_corr_large_scales_B_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode = power +lmax = 2508 +feedback = 0 +do_reionization = F +kmin = 1e-5 +kmax = 20.0 +nk = 200 +zmax = 5.0 +zmax_background = 5.0 +nz_background = 500 +halofit_version = mead2020_feedback +nonlinear = pk +neutrino_hierarchy = normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file = %(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear = T +position-shear = F +perbin = F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets = CELL_EE +data_file = %(FITS_FILE)s +gaussian_covariance = F +covmat_name = COVMAT +cut_zeros = F +like_name = 2pt_like +angle_range_CELL_EE_1_1 = 300.0 800.0 + diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_B_cell.ini new file mode 100755 index 00000000..fd6d8990 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_B_cell.ini @@ -0,0 +1,111 @@ +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_small_scales_B +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_small_scales_B/cosmosis_SP_v1.4.6.3_leak_corr_small_scales_B.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_small_scales_B_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_small_scales_B/samples_SP_v1.4.6.3_leak_corr_small_scales_B_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode = power +lmax = 2508 +feedback = 0 +do_reionization = F +kmin = 1e-5 +kmax = 20.0 +nk = 200 +zmax = 5.0 +zmax_background = 5.0 +nz_background = 500 +halofit_version = mead2020_feedback +nonlinear = pk +neutrino_hierarchy = normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file = %(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear = T +position-shear = F +perbin = F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets = CELL_EE +data_file = %(FITS_FILE)s +gaussian_covariance = F +covmat_name = COVMAT +cut_zeros = F +like_name = 2pt_like +angle_range_CELL_EE_1_1 = 800.0 1600.0 + diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_C_cell.ini new file mode 100755 index 00000000..584a5fbb --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_C_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_C +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_C/cosmosis_SP_v1.4.6.3_C.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_C_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_C/samples_SP_v1.4.6.3_C_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_C_cell.ini new file mode 100755 index 00000000..341a4b7f --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_C_cell.ini @@ -0,0 +1,111 @@ +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_C +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_C/cosmosis_SP_v1.4.6.3_leak_corr_C.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_C_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_C/samples_SP_v1.4.6.3_leak_corr_C_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode = power +lmax = 2508 +feedback = 0 +do_reionization = F +kmin = 1e-5 +kmax = 20.0 +nk = 200 +zmax = 5.0 +zmax_background = 5.0 +nz_background = 500 +halofit_version = mead2020_feedback +nonlinear = pk +neutrino_hierarchy = normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file = %(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear = T +position-shear = F +perbin = F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets = CELL_EE +data_file = %(FITS_FILE)s +gaussian_covariance = F +covmat_name = COVMAT +cut_zeros = F +like_name = 2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 + diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_C_cell.ini new file mode 100755 index 00000000..e2b478dd --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_C_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_HMCode_nobar_C +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_HMCode_nobar_C/cosmosis_SP_v1.4.6.3_leak_corr_HMCode_nobar_C.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_HMCode_nobar_C_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_HMCode_nobar_C/samples_SP_v1.4.6.3_leak_corr_HMCode_nobar_C_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020 +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_C_cell.ini new file mode 100755 index 00000000..66e4a192 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_C_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_OneCov_C +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_OneCov_C/cosmosis_SP_v1.4.6.3_leak_corr_OneCov_C.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_OneCov_C_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_OneCov_C/samples_SP_v1.4.6.3_leak_corr_OneCov_C_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_C_cell.ini new file mode 100755 index 00000000..0be94bbd --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_C_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_halofit_C +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_halofit_C/cosmosis_SP_v1.4.6.3_leak_corr_halofit_C.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_halofit_C_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_halofit_C/samples_SP_v1.4.6.3_leak_corr_halofit_C_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=takahashi +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_C_cell.ini new file mode 100755 index 00000000..718ce25d --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_C_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_include_large_scales_C +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_include_large_scales_C/cosmosis_SP_v1.4.6.3_leak_corr_include_large_scales_C.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_include_large_scales_C_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_include_large_scales_C/samples_SP_v1.4.6.3_leak_corr_include_large_scales_C_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 0.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_C_cell.ini new file mode 100755 index 00000000..69f977d0 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_C_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=1Mpc_C +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=1Mpc_C/cosmosis_SP_v1.4.6.3_leak_corr_kmax=1Mpc_C.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=1Mpc_C_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=1Mpc_C/samples_SP_v1.4.6.3_leak_corr_kmax=1Mpc_C_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 500.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_C_cell.ini new file mode 100755 index 00000000..3ef25704 --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_C_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=3Mpc_C +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=3Mpc_C/cosmosis_SP_v1.4.6.3_leak_corr_kmax=3Mpc_C.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=3Mpc_C_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=3Mpc_C/samples_SP_v1.4.6.3_leak_corr_kmax=3Mpc_C_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 1800.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_C_cell.ini new file mode 100755 index 00000000..0b82ecbf --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_C_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=5Mpc_C +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=5Mpc_C/cosmosis_SP_v1.4.6.3_leak_corr_kmax=5Mpc_C.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=5Mpc_C_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=5Mpc_C/samples_SP_v1.4.6.3_leak_corr_kmax=5Mpc_C_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 2048.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_C_cell.ini new file mode 100755 index 00000000..3382193d --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_C_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_large_scales_C +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_large_scales_C/cosmosis_SP_v1.4.6.3_leak_corr_large_scales_C.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_large_scales_C_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_large_scales_C/samples_SP_v1.4.6.3_leak_corr_large_scales_C_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 300.0 800.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_C_cell.ini new file mode 100755 index 00000000..4bc8fe6d --- /dev/null +++ b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_C_cell.ini @@ -0,0 +1,114 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_small_scales_C +FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_small_scales_C/cosmosis_SP_v1.4.6.3_leak_corr_small_scales_C.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_ia.ini +priors = cosmosis_config/priors.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/ + + +[polychord] +polychord_outfile_root = SP_v1.4.6.3_leak_corr_small_scales_C_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_small_scales_C/samples_SP_v1.4.6.3_leak_corr_small_scales_C_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_CELL_EE_1_1 = 800.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/priors.ini b/cosmo_inference/cosmosis_config/priors.ini new file mode 100644 index 00000000..e1e4a50e --- /dev/null +++ b/cosmo_inference/cosmosis_config/priors.ini @@ -0,0 +1,11 @@ +[intrinsic_alignment_parameters] +A = gaussian 0.83 0.7 + +[cosmological_parameters] +ombh2 = gaussian 0.0244 0.00038 + +[shear_calibration_parameters] +m1 = gaussian -0.057 0.014 + +[nofz_shifts] +bias_1 = gaussian -0.030 0.018 diff --git a/cosmo_inference/cosmosis_config/priors_mock.ini b/cosmo_inference/cosmosis_config/priors_mock.ini new file mode 100644 index 00000000..5a42d2bb --- /dev/null +++ b/cosmo_inference/cosmosis_config/priors_mock.ini @@ -0,0 +1,15 @@ +[psf_leakage_parameters] +alpha = gaussian 0.0 0.022 +beta = gaussian 0.0 0.1148 + +[intrinsic_alignment_parameters] +A = gaussian 0.0 0.7 + +[cosmological_parameters] +ombh2 = gaussian 0.0244 0.00038 + +[shear_calibration_parameters] +m1 = gaussian 0.0 0.01 + +[nofz_shifts] +bias_1 = gaussian 0.0 0.018 diff --git a/cosmo_inference/cosmosis_config/priors_mock_cell.ini b/cosmo_inference/cosmosis_config/priors_mock_cell.ini new file mode 100644 index 00000000..6381417d --- /dev/null +++ b/cosmo_inference/cosmosis_config/priors_mock_cell.ini @@ -0,0 +1,11 @@ +[intrinsic_alignment_parameters] +A = gaussian 0.0 0.7 + +[cosmological_parameters] +ombh2 = gaussian 0.0244 0.00038 + +[shear_calibration_parameters] +m1 = gaussian 0.0 0.01 + +[nofz_shifts] +bias_1 = gaussian 0.0 0.018 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/priors_mock_cell_no_sys.ini b/cosmo_inference/cosmosis_config/priors_mock_cell_no_sys.ini new file mode 100644 index 00000000..781ae8f7 --- /dev/null +++ b/cosmo_inference/cosmosis_config/priors_mock_cell_no_sys.ini @@ -0,0 +1,5 @@ +[cosmological_parameters] +ombh2 = gaussian 0.0244 0.00038 + +[nofz_shifts] +bias_1 = gaussian 0.0 0.013 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/priors_psf.ini b/cosmo_inference/cosmosis_config/priors_psf.ini new file mode 100644 index 00000000..37b0e029 --- /dev/null +++ b/cosmo_inference/cosmosis_config/priors_psf.ini @@ -0,0 +1,15 @@ +[psf_leakage_parameters] +alpha = gaussian 0.0051 0.022 +beta = gaussian 0.8098 0.1148 + +[intrinsic_alignment_parameters] +A = gaussian 0.83 0.7 + +[cosmological_parameters] +ombh2 = gaussian 0.0244 0.00038 + +[shear_calibration_parameters] +m1 = gaussian -0.0057 0.014 + +[nofz_shifts] +bias_1 = gaussian -0.030 0.018 diff --git a/cosmo_inference/cosmosis_config/values.ini b/cosmo_inference/cosmosis_config/values.ini new file mode 100644 index 00000000..39d1ec22 --- /dev/null +++ b/cosmo_inference/cosmosis_config/values.ini @@ -0,0 +1,27 @@ +[cosmological_parameters] +tau = 0.0544 +w = -1.0 +mnu = 0.06 +omega_k = 0.0 +wa = 0.0 +omch2 = 0.10496564589028712 +h0 = 0.7703672811295145 +ombh2 = 0.024364520846452055 +n_s = 1.0378056660322685 +s_8_input = 0.867063981537897 + +[halo_model_parameters] + +[intrinsic_alignment_parameters] +a = 1.2105355520872163 + +[shear_calibration_parameters] +m1 = -0.0019669717507113187 + +[nofz_shifts] +bias_1 = -0.05240296008081707 + +[psf_leakage_parameters] +alpha = 0.017313482956287624 +beta = 1.0863942321436328 + diff --git a/cosmo_inference/cosmosis_config/values_empty.ini b/cosmo_inference/cosmosis_config/values_empty.ini new file mode 100644 index 00000000..019ede84 --- /dev/null +++ b/cosmo_inference/cosmosis_config/values_empty.ini @@ -0,0 +1,27 @@ +[cosmological_parameters] +omch2 = 0.12565412726665712 +ombh2 = 0.022190866236551653 +h0 = 0.7030358726770478 +n_s = 0.9289664070330077 +tau = 0.11917908882774889 +s_8_input = 0.8305214945570943 + +[halo_model_parameters] +logt_agn = 7.557408270897992 + +[intrinsic_alignment_parameters] +a = 1.1275705902391073 + +[shear_calibration_parameters] +m1 = -0.055917714943670135 + +[nofz_shifts] +bias_1 = -0.0043115110715487015 + +[psf_leakage_parameters] +alpha = 0.004915858299931063 +beta = 0.811523998498723 + +[planck] +a_planck = 0.9998087705267193 + diff --git a/cosmo_inference/cosmosis_config/values_ia.ini b/cosmo_inference/cosmosis_config/values_ia.ini new file mode 100644 index 00000000..33389111 --- /dev/null +++ b/cosmo_inference/cosmosis_config/values_ia.ini @@ -0,0 +1,26 @@ +[cosmological_parameters] +omch2 = 0.051 0.120 0.255 +h0 = 0.64 0.7 0.82 +ombh2 = 0.019 0.023 0.026 +n_s = 0.84 0.96 1.1 +S_8_input = 0.1 0.8 1.3 + +tau = 0.0544 +w = -1.0 +mnu = 0.06 +omega_k = 0.0 +wa = 0.0 + +[halo_model_parameters] +logT_AGN = 7.3 7.5 8.0 + +[intrinsic_alignment_parameters] +A = -5.0 1.0 5.0 + +[shear_calibration_parameters] +m1 = -0.1 0.0 0.1 + +[nofz_shifts] +bias_1 = -0.1 0.0 0.1 + + diff --git a/cosmo_inference/cosmosis_config/values_ia_no_sys.ini b/cosmo_inference/cosmosis_config/values_ia_no_sys.ini new file mode 100644 index 00000000..da390032 --- /dev/null +++ b/cosmo_inference/cosmosis_config/values_ia_no_sys.ini @@ -0,0 +1,26 @@ +[cosmological_parameters] +omch2 = 0.051 0.120 0.255 +h0 = 0.64 0.7 0.82 +ombh2 = 0.019 0.023 0.026 +n_s = 0.84 0.96 1.1 +S_8_input = 0.1 0.8 1.3 + +tau = 0.0544 +w = -1.0 +mnu = 0.06 +omega_k = 0.0 +wa = 0.0 + +[halo_model_parameters] +logT_AGN = 7.3 7.5 8.0 + +[intrinsic_alignment_parameters] +A = 0.0 + +[shear_calibration_parameters] +m1 = 0.0 + +[nofz_shifts] +bias_1 = -0.1 0.0 0.1 + + diff --git a/cosmo_inference/cosmosis_config/values_ia_test.ini b/cosmo_inference/cosmosis_config/values_ia_test.ini new file mode 100644 index 00000000..30b2c422 --- /dev/null +++ b/cosmo_inference/cosmosis_config/values_ia_test.ini @@ -0,0 +1,27 @@ +[cosmological_parameters] +omch2 = 0.051 0.11869577244577488 0.255 +h0 = 0.64 0.6766 0.82 +ombh2 = 0.019 0.0224178568132 0.026 +n_s = 0.84 0.9665 1.1 +#S_8_input = 0.1 0.81 1.3 +S_8_input = 0.1 0.8231408713507062 1.3 + +tau = 0.054 +w = -1.0 +mnu = 0.06 +omega_k = 0.0 +wa = 0.0 + +[halo_model_parameters] +logT_AGN = 7.3 7.8 8.0 + +[intrinsic_alignment_parameters] +A = -5.0 0.0 5.0 + +[shear_calibration_parameters] +m1 = -0.1 0.0 0.1 + +[nofz_shifts] +bias_1 = -0.1 0.0 0.1 + + diff --git a/cosmo_inference/cosmosis_config/values_psf.ini b/cosmo_inference/cosmosis_config/values_psf.ini new file mode 100644 index 00000000..7826eaf7 --- /dev/null +++ b/cosmo_inference/cosmosis_config/values_psf.ini @@ -0,0 +1,30 @@ +[cosmological_parameters] +#omega_m = 0.05 0.25 0.6 +omch2 = 0.051 0.12249999999999998 0.255 +h0 = 0.64 0.70 0.82 +ombh2 = 0.019 0.024499999999999997 0.026 +n_s = 0.84 0.96 1.1 +S_8_input = 0.1 0.79563645 1.3 + +tau = 0.0544 +w = -1.0 +mnu = 0.06 +omega_k = 0.0 +wa = 0.0 + +[halo_model_parameters] +logT_AGN = 7.3 7.4755796676459387 8.0 + +[intrinsic_alignment_parameters] +A = -5.0 0 5.0 + +[shear_calibration_parameters] +m1 = -0.1 0.0 0.1 + +[nofz_shifts] +bias_1 = -0.1 0.0 0.1 + +[psf_leakage_parameters] +alpha = -0.1 0.0 0.1 +beta = -2.0 0.0 2.0 + diff --git a/cosmo_inference/cosmosis_config/values_template.ini b/cosmo_inference/cosmosis_config/values_template.ini new file mode 100644 index 00000000..60ac60c6 --- /dev/null +++ b/cosmo_inference/cosmosis_config/values_template.ini @@ -0,0 +1,23 @@ +[cosmological_parameters] +omch2 = 0.01 0.12 0.3 +h0 = 0.55 0.7 0.91 +ombh2 = 0.01 0.023 0.07 +n_s = 0.87 0.96 1.07 +a_s = 0.5e-09 2.9e-09 5.0e-09 + +tau = 0.0544 +w = -1.0 +mnu = 0.06 +omega_k = 0.0 +wa = 0.0 + +[halo_model_parameters] +logt_agn = 6.5 7.81 8.5 + +[nofz_shifts] +bias_1 = -2.0 0.0 2.0 + +[intrinsic_alignment_parameters] +A = -3.0 0.0 3.0 + + diff --git a/cosmo_inference/get_chi2.ipynb b/cosmo_inference/get_chi2.ipynb new file mode 100644 index 00000000..69d026a2 --- /dev/null +++ b/cosmo_inference/get_chi2.ipynb @@ -0,0 +1,1391 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import configparser\n", + "import os\n", + "import subprocess\n", + "\n", + "import healpy as hp\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import scipy.stats as stats\n", + "import seaborn as sns\n", + "from astropy.io import fits\n", + "from getdist import plots\n", + "from IPython.display import Markdown, display\n", + "from scipy.interpolate import interp1d\n", + "\n", + "%matplotlib inline\n", + "# import uncertainties\n", + "\n", + "# Use paper style and seaborn with husl palette\n", + "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", + "# Set default palette - will be updated per plot as needed\n", + "sns.set_palette(\"husl\")\n", + "\n", + "g = plots.get_subplot_plotter(width_inch=30)\n", + "g.settings.axes_fontsize = 30\n", + "g.settings.axes_labelsize = 30\n", + "g.settings.alpha_filled_add = 0.7\n", + "g.settings.legend_fontsize = 40\n", + "\n", + "\n", + "# SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE\n", + "root_dir = \"/n09data/guerrini/output_chains/\"\n", + "\n", + "catalog_version = \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1\"\n", + "\n", + "path_ini_files = \"/home/guerrini/sp_validation/cosmo_inference/cosmosis_config/\"\n", + "\n", + "roots = [\n", + " f\"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_{int(i)}.0_80.0_10.0_80.0\"\n", + " for i in [3, 5, 7, 10, 11]\n", + "]\n", + "\n", + "\"\"\" roots = [\n", + " \"SP_v1.4.5_A\",\n", + " #\"SP_v1.4.5_A_no_IA\",\n", + " #\"SP_v1.4.5_A_no_dz\",\n", + " #\"SP_v1.4.5_A_no_m_bias\",\n", + " \"SP_v1.4.5_A_sc_3_150\",\n", + " \"SP_v1.4.5_A_sc_3_60\",\n", + " \"SP_v1.4.5_A_sc_10_150\",\n", + " \"SP_v1.4.5_A_sc_10_60\",\n", + " \"SP_v1.4.5_A_sc_5_150\",\n", + " \"SP_v1.4.5_A_sc_7_150\",\n", + " #\"SP_v1.4.5_A_no_leakage\"\n", + "] \"\"\"\n", + "\n", + "\"\"\" roots = [\n", + " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0\",\n", + " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0_no_alpha_beta\"\n", + "] \"\"\"\n", + "\n", + "\n", + "properties = {}\n", + "\n", + "for root in roots:\n", + " config = configparser.ConfigParser()\n", + " config.optionxform = str # Preserve case sensitivity of option names\n", + " config.read(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\")\n", + "\n", + " add_xi_sys = config[\"2pt_like\"][\"add_xi_sys\"]\n", + " add_xi_sys = add_xi_sys == \"T\"\n", + " lower_bound_xi_plus, upper_bound_xi_plus = map(\n", + " float, config[\"2pt_like\"][\"angle_range_XI_PLUS_1_1\"].split()\n", + " )\n", + " lower_bound_xi_minus, upper_bound_xi_minus = map(\n", + " float, config[\"2pt_like\"][\"angle_range_XI_MINUS_1_1\"].split()\n", + " )\n", + "\n", + " properties[root] = {\n", + " \"add_xi_sys\": add_xi_sys,\n", + " \"lower_bound_xi_plus\": lower_bound_xi_plus,\n", + " \"upper_bound_xi_plus\": upper_bound_xi_plus,\n", + " \"lower_bound_xi_minus\": lower_bound_xi_minus,\n", + " \"upper_bound_xi_minus\": upper_bound_xi_minus,\n", + " }\n", + "\n", + "\n", + "print(roots)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Retrieve the chains" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# MAKE PARAMNAMES FILE\n", + "\n", + "for root in roots:\n", + " with open(root_dir + \"{}/samples_{}.txt\".format(\"/\" + root, root), \"r\") as file:\n", + " params = file.readline()[1:].split(\"\\t\")[:-4]\n", + " file.close()\n", + "\n", + " with open(\n", + " root_dir + \"{}/getdist_{}.paramnames\".format(\"/\" + root, root), \"w\"\n", + " ) as file:\n", + " for i in range(len(params)):\n", + " if len(params[i].split(\"--\")) > 1:\n", + " file.write(params[i].split(\"--\")[1] + \"\\n\")\n", + " else:\n", + " file.write(params[i].split(\"--\")[0] + \"\\n\")\n", + " file.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# READ CHAIN\n", + "\n", + "chains = []\n", + "\n", + "for root in roots:\n", + " samples = np.loadtxt(root_dir + \"{}/samples_{}.txt\".format(root, root))\n", + " print(len(samples))\n", + " if \"nautilus\" in root:\n", + " samples = np.column_stack(\n", + " (np.exp(samples[:, -3]), samples[:, -1] - samples[:, -2], samples[:, 0:-3])\n", + " )\n", + " else:\n", + " samples = np.column_stack((samples[:, -1], samples[:, -3], samples[:, 0:-4]))\n", + " np.savetxt(root_dir + \"{}/getdist_{}.txt\".format(root, root), samples)\n", + "\n", + " chain = g.samples_for_root(\n", + " root_dir + \"{}/getdist_{}\".format(root, root),\n", + " cache=False,\n", + " settings={\"ignore_rows\": 0, \"smooth_scale_2D\": 0.3, \"smooth_scale_1D\": 0.3},\n", + " )\n", + "\n", + " chains.append(chain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "name_list = [\n", + " \"OMEGA_M\",\n", + " \"ombh2\",\n", + " \"h0\",\n", + " \"n_s\",\n", + " \"SIGMA_8\",\n", + " \"s_8_input\",\n", + " \"logt_agn\",\n", + " \"a\",\n", + " \"m1\",\n", + " \"bias_1\",\n", + " \"alpha\",\n", + " \"beta\",\n", + "]\n", + "label_list = [\n", + " r\"\\Omega_m\",\n", + " r\"\\omega_b h^2\",\n", + " \"h_0\",\n", + " \"n_s\",\n", + " r\"\\sigma_8\",\n", + " \"S_8\",\n", + " \"log T_{AGN}\",\n", + " \"A_{IA}\",\n", + " \"m_1\",\n", + " r\"\\Delta z_1\",\n", + " \"\\\\alpha_{PSF}\",\n", + " \"\\\\beta_{PSF}\",\n", + "]\n", + "\n", + "for chain in chains:\n", + " param_names = chain.getParamNames()\n", + " for name, label in zip(name_list, label_list):\n", + " param_names.parWithName(name).label = label" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Extract the best fit parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "best_fit = {}\n", + "\n", + "for root, chain in zip(roots, chains):\n", + " print(root)\n", + " likestats = chain.getLikeStats()\n", + " bestfit_idx = np.argmax(chain.loglikes)\n", + " maxlike = chain.loglikes[bestfit_idx]\n", + " print(f\"Maximum Likelihood: {maxlike:.5g}\")\n", + " best_fit[root] = {\"likelihood\": maxlike}\n", + " for i, par in enumerate(likestats.names):\n", + " best_fit[root].update(\n", + " {par.name: np.average(chain.samples[:, i], weights=chain.weights)}\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run `Cosmosis` in test mode to get the data vectors" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if not os.path.exists(path_ini_files + \"/values_empty.ini\"):\n", + " content = \"\"\"[cosmological_parameters]\n", + "\n", + "tau = 0.0544\n", + "w = -1.0\n", + "massive_nu = 1\n", + "massless_nu = 2.046\n", + "omega_k = 0.0\n", + "wa = 0.0\n", + "\n", + "[halo_model_parameters]\n", + "\n", + "[intrinsic_alignment_parameters]\n", + "\n", + "[shear_calibration_parameters]\n", + "\n", + "[nofz_shifts]\n", + "\n", + "[psf_leakage_parameters]\n", + "\"\"\"\n", + "\n", + " with open(path_ini_files + \"/values_empty.ini\", \"w\") as f:\n", + " f.write(content)\n", + " f.close()\n", + "\n", + " print(\"File created successfully\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "section_map = {\n", + " \"omch2\": \"cosmological_parameters\",\n", + " \"ombh2\": \"cosmological_parameters\",\n", + " \"h0\": \"cosmological_parameters\",\n", + " \"n_s\": \"cosmological_parameters\",\n", + " \"s_8_input\": \"cosmological_parameters\",\n", + " \"logt_agn\": \"halo_model_parameters\",\n", + " \"a\": \"intrinsic_alignment_parameters\",\n", + " \"m1\": \"shear_calibration_parameters\",\n", + " \"bias_1\": \"nofz_shifts\",\n", + " \"alpha\": \"psf_leakage_parameters\",\n", + " \"beta\": \"psf_leakage_parameters\",\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "env = os.environ.copy()\n", + "env[\"LD_LIBRARY_PATH\"] = (\n", + " \"/home/guerrini/.conda/envs/sp_validation/lib/python3.9/site-packages/cosmosis/datablock:\"\n", + " + env.get(\"LD_LIBRARY_PATH\", \"\")\n", + ")\n", + "\n", + "for root in roots:\n", + " print(root)\n", + " config = configparser.ConfigParser()\n", + " config.optionxform = str # Preserve case sensitivity of option names\n", + " config.read(path_ini_files + \"/values_empty.ini\")\n", + " for param, value in best_fit[root].items():\n", + " section = section_map.get(param)\n", + " if section is None:\n", + " continue\n", + " if section not in config:\n", + " config.add_section(section)\n", + " config[section][param] = str(value)\n", + "\n", + " with open(path_ini_files + \"/values_empty.ini\", \"w\") as configfile:\n", + " config.write(configfile)\n", + "\n", + " # Modify the ini file to run in test mode at the best fit\n", + " config = configparser.ConfigParser()\n", + " config.optionxform = str # Preserve case sensitivity of option names\n", + " config.read(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\")\n", + "\n", + " sampler = config[\"runtime\"][\"sampler\"]\n", + " config[\"runtime\"][\"sampler\"] = \"test\"\n", + " values = config[\"pipeline\"][\"values\"]\n", + " config[\"pipeline\"][\"values\"] = path_ini_files + \"/values_empty.ini\"\n", + "\n", + " with open(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\", \"w\") as configfile:\n", + " config.write(configfile)\n", + "\n", + " # Run cosmosis\n", + " result = subprocess.run(\n", + " [\"cosmosis\", \"cosmosis_config/cosmosis_pipeline_{}.ini\".format(root)],\n", + " env=env,\n", + " capture_output=True,\n", + " text=True,\n", + " )\n", + " print(f\"STDOUT:\\n{result.stdout}\")\n", + " print(f\"STDERR:\\n{result.stderr}\")\n", + "\n", + " # Modify the ini file to the previous one\n", + " config[\"pipeline\"][\"values\"] = values\n", + " config[\"runtime\"][\"sampler\"] = sampler\n", + "\n", + " with open(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\", \"w\") as configfile:\n", + " config.write(configfile)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute the $\\chi^2$" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "output_folder = \"/n09data/guerrini/output_chains/\"\n", + "\n", + "metrics = {}\n", + "\n", + "for root in roots:\n", + " print(root)\n", + "\n", + " add_xi_sys = properties[root][\"add_xi_sys\"]\n", + " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", + " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", + " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", + " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", + "\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " shear_xi_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " shear_xi_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_sys_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + "\n", + " # Read model tau_stats\n", + " theta_tau = np.loadtxt(\n", + " output_folder + \"best_fit/{}/tau_0_plus/theta.txt\".format(root)\n", + " )\n", + " theta_tau_arcmin = theta_tau * 180 * 60 / np.pi\n", + " tau_0_model = np.loadtxt(\n", + " output_folder + \"best_fit/{}/tau_0_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " tau_2_model = np.loadtxt(\n", + " output_folder + \"best_fit/{}/tau_2_plus/bin_1_1.txt\".format(root)\n", + " )\n", + "\n", + " # Read the data\n", + " data = fits.open(f\"data/{catalog_version}/cosmosis_{catalog_version}.fits\")\n", + "\n", + " theta_data = data[\"XI_PLUS\"].data[\"ANG\"]\n", + " xi_plus_data = data[\"XI_PLUS\"].data[\"VALUE\"]\n", + " xi_minus_data = data[\"XI_MINUS\"].data[\"VALUE\"]\n", + " tau_0_data = data[\"TAU_0_PLUS\"].data[\"VALUE\"]\n", + " tau_2_data = data[\"TAU_2_PLUS\"].data[\"VALUE\"]\n", + "\n", + " # Load the covariance\n", + " cov = data[\"COVMAT\"].data\n", + " cov_xi = cov[0 : 2 * len(xi_plus_data), 0 : 2 * len(xi_plus_data)]\n", + " cov_tau = cov[2 * len(xi_plus_data) :, 2 * len(xi_plus_data) :]\n", + "\n", + " # interpolate the model\n", + " interp_xi_plus = interp1d(\n", + " theta_arcmin, shear_xi_plus, kind=\"cubic\", fill_value=\"extrapolate\"\n", + " )\n", + " interp_xi_minus = interp1d(\n", + " theta_arcmin, shear_xi_minus, kind=\"cubic\", fill_value=\"extrapolate\"\n", + " )\n", + "\n", + " xi_plus_model = interp_xi_plus(theta_data)\n", + " if add_xi_sys:\n", + " xi_plus_model += xi_sys_plus\n", + " xi_minus_model = interp_xi_minus(theta_data)\n", + " if add_xi_sys:\n", + " xi_minus_model += xi_sys_minus\n", + "\n", + " # Concatenate the data vector\n", + " xi_data = np.concatenate((xi_plus_data, xi_minus_data))\n", + " xi_model = np.concatenate((xi_plus_model, xi_minus_model))\n", + "\n", + " tau_data = np.concatenate((tau_0_data, tau_2_data))\n", + " tau_model = np.concatenate((tau_0_model, tau_2_model))\n", + "\n", + " # Apply scale cuts\n", + " mask_xi_plus = (theta_data > lower_bound_xi_plus) & (\n", + " theta_data < upper_bound_xi_plus\n", + " )\n", + " mask_xi_minus = (theta_data > lower_bound_xi_minus) & (\n", + " theta_data < upper_bound_xi_minus\n", + " )\n", + " mask = np.concatenate((mask_xi_plus, mask_xi_minus))\n", + "\n", + " xi_data = xi_data[mask]\n", + " xi_model = xi_model[mask]\n", + " cov_xi = cov_xi[mask][:, mask]\n", + "\n", + " xi_plus_chi2 = np.dot(\n", + " (xi_model - xi_data), np.dot(np.linalg.inv(cov_xi), (xi_model - xi_data))\n", + " )\n", + " tau_chi2 = np.dot(\n", + " (tau_model - tau_data), np.dot(np.linalg.inv(cov_tau), (tau_model - tau_data))\n", + " )\n", + " n_dof_xi = np.sum(mask)\n", + " n_dof_tau = len(tau_0_data) + len(tau_2_data)\n", + " p_value_xi = 1 - stats.chi2.cdf(xi_plus_chi2, n_dof_xi)\n", + " p_value_tau = 1 - stats.chi2.cdf(tau_chi2, n_dof_tau)\n", + " chi2_tot = xi_plus_chi2 + tau_chi2\n", + " n_dof_tot = n_dof_xi + n_dof_tau\n", + " p_value_tot = 1 - stats.chi2.cdf(chi2_tot, n_dof_tot)\n", + "\n", + " metrics[root] = {\n", + " \"chi2_xi\": xi_plus_chi2,\n", + " \"n_dof_xi\": n_dof_xi,\n", + " \"p_value_xi\": p_value_xi,\n", + " \"chi2_tau\": tau_chi2,\n", + " \"n_dof_tau\": n_dof_tau,\n", + " \"p_value_tau\": p_value_tau,\n", + " \"chi2_tot\": chi2_tot,\n", + " \"n_dof_tot\": n_dof_tot,\n", + " \"p_value_tot\": p_value_tot,\n", + " }\n", + " print(\"Done!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def get_latex_table(metrics):\n", + " latex_lines = [\n", + " r\"\\begin{tabular}{lccc|ccc|ccc}\",\n", + " r\"\\hline\",\n", + " r\"Root & $\\chi^2_{\\xi^+}$/dof & $p_{\\xi^+}$ & \"\n", + " r\"$\\chi^2_\\tau$/dof & $p_\\tau$ & $\\chi^2_{\\text{tot}}$/dof & $p_{\\text{tot}}$ \\\\\",\n", + " r\"\\hline\",\n", + " ]\n", + "\n", + " for root, vals in metrics.items():\n", + " escaped = root.replace(\"_\", r\"\\_\")\n", + " line = (\n", + " f\"{escaped} & \"\n", + " f\"{vals['chi2_xi']:.2f}/{vals['n_dof_xi']} & {vals['p_value_xi']:.5f} & \"\n", + " f\"{vals['chi2_tau']:.2f}/{vals['n_dof_tau']} & {vals['p_value_tau']:.5f} & \"\n", + " f\"{vals['chi2_tot']:.2f}/{vals['n_dof_tot']} & {vals['p_value_tot']:.5f} \\\\\\\\\"\n", + " )\n", + " latex_lines.append(line)\n", + "\n", + " latex_lines.append(r\"\\hline\")\n", + " latex_lines.append(r\"\\end{tabular}\")\n", + "\n", + " # Print LaTeX table\n", + " print(\"\\n\".join(latex_lines))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "get_latex_table(metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def display_markdown(metrics):\n", + " # Build Markdown table\n", + " header = (\n", + " \"| Root | $\\\\chi^2$ (ξ⁺) / dof | p-val (ξ⁺) | $\\\\chi^2$ (τ) / dof | p-val (τ) | $\\\\chi^2$ (tot) / dof | p-val (tot) |\\n\"\n", + " \"|------|----------------|------------|---------------|------------|------------------|--------------|\\n\"\n", + " )\n", + "\n", + " rows = []\n", + " for root, vals in metrics.items():\n", + " row = f\"| `{root}` \"\n", + " row += f\"| {vals['chi2_xi']:.2f} / {vals['n_dof_xi']} \"\n", + " row += f\"| {vals['p_value_xi']:.5f} \"\n", + " row += f\"| {vals['chi2_tau']:.2f} / {vals['n_dof_tau']} \"\n", + " row += f\"| {vals['p_value_tau']:.5f} \"\n", + " row += f\"| {vals['chi2_tot']:.2f} / {vals['n_dof_tot']} \"\n", + " row += f\"| {vals['p_value_tot']:.5f} |\"\n", + " rows.append(row)\n", + "\n", + " # Display in Jupyter\n", + " display(Markdown(header + \"\\n\".join(rows)))\n", + " return header + \"\\n\".join(rows)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "markdown_source = display_markdown(metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "markdown_source" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot the best-fit of each model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = fits.open(\n", + " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_version}/cosmosis_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.fits\"\n", + ")\n", + "xi_plus = data[\"XI_PLUS\"].data\n", + "xi_minus = data[\"XI_MINUS\"].data\n", + "cov_mat = data[\"COVMAT\"].data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(15, 15))\n", + "\n", + "plt.subplot(211)\n", + "\n", + "plt.errorbar(\n", + " xi_plus[\"ANG\"],\n", + " xi_plus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + ")\n", + "\n", + "plt.ylabel(r\"$\\xi_{+}$\", fontsize=26)\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "\n", + "plt.subplot(212)\n", + "\n", + "plt.errorbar(\n", + " xi_minus[\"ANG\"],\n", + " xi_minus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + ")\n", + "\n", + "plt.xlabel(r\"$\\theta$ [arcmin]\", fontsize=26)\n", + "plt.ylabel(r\"$\\xi_{-}$\", fontsize=26)\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.legend(fontsize=15)\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_best_fit(\n", + " root_to_plot,\n", + " colours,\n", + " savefile,\n", + " theta_min=1.0,\n", + " theta_max=250.0,\n", + " multiply_theta=False,\n", + " plot_xi_sys=True,\n", + "):\n", + " data = fits.open(\n", + " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_version}/cosmosis_{catalog_version}.fits\"\n", + " )\n", + " xi_plus = data[\"XI_PLUS\"].data\n", + " xi_minus = data[\"XI_MINUS\"].data\n", + " cov_mat = data[\"COVMAT\"].data\n", + "\n", + " plt.figure(figsize=(15, 15))\n", + "\n", + " plt.subplot(211)\n", + "\n", + " y_plot_xi_plus = (\n", + " xi_plus[\"VALUE\"] if not multiply_theta else xi_plus[\"ANG\"] * xi_plus[\"VALUE\"]\n", + " )\n", + " y_errorbar = (\n", + " xi_plus[\"ANG\"] * np.sqrt(np.diag(cov_mat))[:20]\n", + " if multiply_theta\n", + " else np.sqrt(np.diag(cov_mat))[:20]\n", + " )\n", + " plt.errorbar(\n", + " xi_plus[\"ANG\"],\n", + " y_plot_xi_plus,\n", + " yerr=y_errorbar,\n", + " fmt=\"o\",\n", + " label=f\"{catalog_version} data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + " )\n", + "\n", + " for root, color in zip(root_to_plot, colours):\n", + " add_xi_sys = properties[root][\"add_xi_sys\"]\n", + " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", + " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", + " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", + " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", + "\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " shear_xi_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " shear_xi_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_sys_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + " theta_xi_sys = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", + " )\n", + " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", + "\n", + " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", + " xi_plus_model = shear_xi_plus[mask]\n", + " if add_xi_sys:\n", + " xi_plus_model += np.interp(\n", + " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_plus\n", + " )\n", + "\n", + " y_plot = theta_arcmin[mask] * xi_plus_model if multiply_theta else xi_plus_model\n", + " plt.plot(theta_arcmin[mask], y_plot, color=color, label=root, alpha=0.5)\n", + " if plot_xi_sys and add_xi_sys:\n", + " y_plot_xi_sys = (\n", + " theta_xi_sys_arcmin * xi_sys_plus if multiply_theta else xi_sys_plus\n", + " )\n", + " plt.plot(\n", + " theta_xi_sys_arcmin,\n", + " y_plot_xi_sys,\n", + " color=color,\n", + " linestyle=\"-.\",\n", + " alpha=0.5,\n", + " )\n", + " plt.axvline(x=lower_bound_xi_plus, color=color, linestyle=\"--\", alpha=0.3)\n", + " plt.axvline(x=upper_bound_xi_plus, color=color, linestyle=\"--\", alpha=0.3)\n", + "\n", + " y_label = r\"$\\xi_{+}$\" if not multiply_theta else r\"$\\theta \\xi_{+}$\"\n", + " plt.ylabel(y_label, fontsize=26)\n", + " plt.xscale(\"log\")\n", + " plt.yscale(\"log\")\n", + " plt.legend(loc=\"lower left\", fontsize=8)\n", + "\n", + " plt.subplot(212)\n", + "\n", + " y_plot_xi_minus = (\n", + " xi_minus[\"VALUE\"] if not multiply_theta else xi_minus[\"ANG\"] * xi_minus[\"VALUE\"]\n", + " )\n", + " y_errorbar = (\n", + " xi_minus[\"ANG\"] * np.sqrt(np.diag(cov_mat))[20:40]\n", + " if multiply_theta\n", + " else np.sqrt(np.diag(cov_mat))[20:40]\n", + " )\n", + " plt.errorbar(\n", + " xi_minus[\"ANG\"],\n", + " y_plot_xi_minus,\n", + " yerr=y_errorbar,\n", + " fmt=\"o\",\n", + " label=f\"{catalog_version} data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + " )\n", + "\n", + " for root, color in zip(root_to_plot, colours):\n", + " add_xi_sys = properties[root][\"add_xi_sys\"]\n", + " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", + " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", + " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", + " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", + "\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " shear_xi_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " shear_xi_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_sys_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + " theta_xi_sys = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", + " )\n", + " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", + "\n", + " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", + " xi_minus_model = shear_xi_minus[mask]\n", + " if add_xi_sys:\n", + " xi_minus_model += np.interp(\n", + " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_minus\n", + " )\n", + "\n", + " y_plot = (\n", + " theta_arcmin[mask] * xi_minus_model if multiply_theta else xi_minus_model\n", + " )\n", + " plt.plot(theta_arcmin[mask], y_plot, color=color, label=root, alpha=0.5)\n", + " if plot_xi_sys and add_xi_sys:\n", + " y_plot_xi_sys = (\n", + " theta_xi_sys_arcmin * xi_sys_minus if multiply_theta else xi_sys_minus\n", + " )\n", + " plt.plot(\n", + " theta_xi_sys_arcmin,\n", + " y_plot_xi_sys,\n", + " color=color,\n", + " linestyle=\"-.\",\n", + " alpha=0.5,\n", + " )\n", + " plt.axvline(x=lower_bound_xi_minus, color=color, linestyle=\"--\", alpha=0.3)\n", + " plt.axvline(x=upper_bound_xi_minus, color=color, linestyle=\"--\", alpha=0.3)\n", + "\n", + " plt.xlabel(r\"$\\theta$ [arcmin]\", fontsize=26)\n", + " y_label = r\"$\\xi_{-}$\" if not multiply_theta else r\"$\\theta \\xi_{-}$\"\n", + " plt.ylabel(y_label, fontsize=26)\n", + " plt.xscale(\"log\")\n", + " plt.yscale(\"log\")\n", + " plt.legend(loc=\"lower left\", fontsize=8)\n", + "\n", + " if savefile is not None:\n", + " plt.savefig(savefile, bbox_inches=\"tight\")\n", + "\n", + " plt.show()\n", + "\n", + "\n", + "def plot_best_fit_ratio(\n", + " root_to_plot, colours, savefile, theta_min=1.0, theta_max=250.0\n", + "):\n", + " data = fits.open(\n", + " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_version}/cosmosis_{catalog_version}.fits\"\n", + " )\n", + " xi_plus = data[\"XI_PLUS\"].data\n", + " xi_minus = data[\"XI_MINUS\"].data\n", + " cov_mat = data[\"COVMAT\"].data\n", + "\n", + " plt.figure(figsize=(15, 15))\n", + "\n", + " plt.subplot(211)\n", + "\n", + " root = roots[0]\n", + " add_xi_sys = properties[root][\"add_xi_sys\"]\n", + " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", + " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", + " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", + " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", + "\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " shear_xi_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " shear_xi_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_sys_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + " theta_xi_sys = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", + " )\n", + " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", + "\n", + " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", + " xi_plus_model_fiducial = shear_xi_plus[mask]\n", + " if add_xi_sys:\n", + " xi_plus_model_fiducial += np.interp(\n", + " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_plus\n", + " )\n", + "\n", + " plt.errorbar(\n", + " xi_plus[\"ANG\"],\n", + " xi_plus[\"VALUE\"]\n", + " / np.interp(xi_plus[\"ANG\"], theta_arcmin[mask], xi_plus_model_fiducial),\n", + " yerr=np.sqrt(np.diag(cov_mat))[:20]\n", + " / np.abs(np.interp(xi_plus[\"ANG\"], theta_arcmin[mask], xi_plus_model_fiducial)),\n", + " fmt=\"o\",\n", + " label=f\"{catalog_version} data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + " )\n", + "\n", + " for root, color in zip(root_to_plot, colours):\n", + " add_xi_sys = properties[root][\"add_xi_sys\"]\n", + " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", + " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", + " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", + " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", + "\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " shear_xi_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " shear_xi_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_sys_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + " theta_xi_sys = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", + " )\n", + " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", + "\n", + " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", + " xi_plus_model = shear_xi_plus[mask]\n", + " if add_xi_sys:\n", + " xi_plus_model += np.interp(\n", + " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_plus\n", + " )\n", + "\n", + " alpha = 1.0 if root == roots[0] else 0.5\n", + " plt.plot(\n", + " theta_arcmin[mask],\n", + " xi_plus_model / xi_plus_model_fiducial,\n", + " color=color,\n", + " label=root,\n", + " alpha=alpha,\n", + " )\n", + " plt.axvline(x=lower_bound_xi_plus, color=color, linestyle=\"--\", alpha=0.3)\n", + " plt.axvline(x=upper_bound_xi_plus, color=color, linestyle=\"--\", alpha=0.3)\n", + "\n", + " plt.ylabel(r\"$\\xi_{+}/\\xi_{+, \\text{fid}}$\", fontsize=26)\n", + " plt.xscale(\"log\")\n", + " # plt.yscale('log')\n", + " plt.legend(loc=\"lower left\", fontsize=8)\n", + "\n", + " plt.subplot(212)\n", + "\n", + " root = roots[0]\n", + " add_xi_sys = properties[root][\"add_xi_sys\"]\n", + " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", + " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", + " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", + " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", + "\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " shear_xi_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " shear_xi_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_sys_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + " theta_xi_sys = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", + " )\n", + " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", + "\n", + " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", + " xi_minus_model_fiducial = shear_xi_minus[mask]\n", + " if add_xi_sys:\n", + " xi_minus_model_fiducial += np.interp(\n", + " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_minus\n", + " )\n", + "\n", + " plt.errorbar(\n", + " xi_minus[\"ANG\"],\n", + " xi_minus[\"VALUE\"]\n", + " / np.interp(xi_minus[\"ANG\"], theta_arcmin[mask], xi_minus_model_fiducial),\n", + " yerr=np.sqrt(np.diag(cov_mat))[20:40]\n", + " / np.abs(\n", + " np.interp(xi_minus[\"ANG\"], theta_arcmin[mask], xi_minus_model_fiducial)\n", + " ),\n", + " fmt=\"o\",\n", + " label=f\"{catalog_version} data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + " )\n", + "\n", + " for root, color in zip(root_to_plot, colours):\n", + " add_xi_sys = properties[root][\"add_xi_sys\"]\n", + " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", + " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", + " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", + " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", + "\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " shear_xi_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " shear_xi_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_sys_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + " theta_xi_sys = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", + " )\n", + " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", + "\n", + " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", + " xi_minus_model = shear_xi_minus[mask]\n", + " if add_xi_sys:\n", + " xi_minus_model += np.interp(\n", + " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_minus\n", + " )\n", + "\n", + " alpha = 1.0 if root == roots[0] else 0.5\n", + " plt.plot(\n", + " theta_arcmin[mask],\n", + " xi_minus_model / xi_minus_model_fiducial,\n", + " color=color,\n", + " label=root,\n", + " alpha=alpha,\n", + " )\n", + " plt.axvline(x=lower_bound_xi_minus, color=color, linestyle=\"--\", alpha=0.3)\n", + " plt.axvline(x=upper_bound_xi_minus, color=color, linestyle=\"--\", alpha=0.3)\n", + "\n", + " plt.xlabel(r\"$\\theta$ [arcmin]\", fontsize=26)\n", + " plt.ylabel(r\"$\\xi_{-}/\\xi_{-, \\text{fid}}$\", fontsize=26)\n", + " plt.xscale(\"log\")\n", + " plt.ylim(0, 2)\n", + " # plt.yscale('log')\n", + " plt.legend(loc=\"lower left\", fontsize=8)\n", + "\n", + " if savefile is not None:\n", + " plt.savefig(savefile, bbox_inches=\"tight\")\n", + "\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "root_to_plot = [\n", + " \"SP_v1.4.5_A\",\n", + " # \"SP_v1.4.5_A_no_IA\",\n", + " # \"SP_v1.4.5_A_no_dz\",\n", + " # \"SP_v1.4.5_A_no_m_bias\",\n", + " \"SP_v1.4.5_A_sc_3_150\",\n", + " \"SP_v1.4.5_A_sc_3_60\",\n", + " \"SP_v1.4.5_A_sc_10_150\",\n", + " \"SP_v1.4.5_A_sc_10_60\",\n", + " \"SP_v1.4.5_A_sc_5_150\",\n", + " \"SP_v1.4.5_A_sc_7_150\",\n", + " # \"SP_v1.4.5_A_no_leakage\"\n", + "]\n", + "\n", + "root_to_plot = [\n", + " f\"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_{int(i)}.0_80.0_10.0_80.0\"\n", + " for i in [3, 5, 7, 10, 11]\n", + "]\n", + "\n", + "\"\"\" root_to_plot = [\n", + " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0\",\n", + " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0_no_alpha_beta\"\n", + "] \"\"\"\n", + "\n", + "\n", + "colours = [\n", + " \"cornflowerblue\",\n", + " \"salmon\",\n", + " \"darkorange\",\n", + " \"forestgreen\",\n", + " \"turquoise\",\n", + " \"darkviolet\",\n", + " \"crimson\",\n", + " \"gold\",\n", + " \"lightcoral\",\n", + " \"mediumseagreen\",\n", + " \"lightsteelblue\",\n", + " \"black\",\n", + " \"silver\",\n", + " \"peru\",\n", + " \"maroon\",\n", + " \"olive\",\n", + "]\n", + "\n", + "savefile = None\n", + "\n", + "plot_best_fit(root_to_plot, colours, savefile, multiply_theta=True, plot_xi_sys=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "root_to_plot = [\n", + " \"SP_v1.4.5_A\",\n", + " # \"SP_v1.4.5_A_no_IA\",\n", + " # \"SP_v1.4.5_A_no_dz\",\n", + " # \"SP_v1.4.5_A_no_m_bias\",\n", + " \"SP_v1.4.5_A_sc_3_150\",\n", + " \"SP_v1.4.5_A_sc_3_60\",\n", + " \"SP_v1.4.5_A_sc_10_150\",\n", + " \"SP_v1.4.5_A_sc_10_60\",\n", + " \"SP_v1.4.5_A_sc_5_150\",\n", + " \"SP_v1.4.5_A_sc_7_150\",\n", + " # \"SP_v1.4.5_A_no_leakage\"\n", + "]\n", + "\n", + "\"\"\" root_to_plot = [\n", + " f\"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_{int(i)}.0_80.0_10.0_80.0\" for i in [3, 5, 7, 10, 11]\n", + "] \"\"\"\n", + "\n", + "root_to_plot = [\n", + " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0\",\n", + " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0_no_alpha_beta\",\n", + "]\n", + "\n", + "\n", + "colours = [\n", + " \"red\",\n", + " \"salmon\",\n", + " \"darkorange\",\n", + " \"forestgreen\",\n", + " \"turquoise\",\n", + " \"darkviolet\",\n", + " \"crimson\",\n", + " \"gold\",\n", + " \"lightcoral\",\n", + " \"mediumseagreen\",\n", + " \"lightsteelblue\",\n", + " \"black\",\n", + " \"silver\",\n", + " \"peru\",\n", + " \"maroon\",\n", + " \"olive\",\n", + "]\n", + "\n", + "savefile = \"best_fit_ratio_w_wo_leakage.png\"\n", + "\n", + "plot_best_fit_ratio(root_to_plot, colours, savefile)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_best_fit_tau(root_to_plot, colours, savefile, theta_min=1.0, theta_max=250.0):\n", + " data = fits.open(\n", + " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_version}/cosmosis_{catalog_version}.fits\"\n", + " )\n", + " tau_0 = data[\"TAU_0_PLUS\"].data\n", + " tau_2 = data[\"TAU_2_PLUS\"].data\n", + " cov_mat = data[\"COVMAT\"].data\n", + "\n", + " plt.figure(figsize=(15, 15))\n", + "\n", + " plt.subplot(211)\n", + "\n", + " plt.errorbar(\n", + " tau_0[\"ANG\"],\n", + " tau_0[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[40:60],\n", + " fmt=\"o\",\n", + " label=f\"{catalog_version} data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + " )\n", + "\n", + " for root, color in zip(root_to_plot, colours):\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/tau_0_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " tau_0_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/tau_0_plus/bin_1_1.txt\".format(root)\n", + " )\n", + "\n", + " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", + "\n", + " plt.plot(\n", + " theta_arcmin[mask], tau_0_plus[mask], color=color, label=root, alpha=0.5\n", + " )\n", + "\n", + " plt.ylabel(r\"$\\tau_0$\", fontsize=26)\n", + " plt.xscale(\"log\")\n", + " # plt.yscale('log')\n", + " plt.legend(loc=\"upper right\", fontsize=8)\n", + "\n", + " plt.subplot(212)\n", + "\n", + " y_plot_tau_2 = tau_2[\"ANG\"] * tau_2[\"VALUE\"]\n", + " y_errorbar = tau_2[\"ANG\"] * np.sqrt(np.diag(cov_mat))[60:80]\n", + " plt.errorbar(\n", + " tau_2[\"ANG\"],\n", + " y_plot_tau_2,\n", + " yerr=y_errorbar,\n", + " fmt=\"o\",\n", + " label=f\"{catalog_version} data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + " )\n", + "\n", + " for root, color in zip(root_to_plot, colours):\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/tau_2_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " tau_2_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/tau_2_plus/bin_1_1.txt\".format(root)\n", + " )\n", + "\n", + " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", + "\n", + " plt.plot(\n", + " theta_arcmin[mask],\n", + " theta_arcmin[mask] * tau_2_plus[mask],\n", + " color=color,\n", + " label=root,\n", + " alpha=0.5,\n", + " )\n", + "\n", + " plt.xlabel(r\"$\\theta$ [arcmin]\", fontsize=26)\n", + " plt.ylabel(r\"$\\theta \\tau_2$\", fontsize=26)\n", + " plt.xscale(\"log\")\n", + " # plt.yscale('log')\n", + " plt.legend(loc=\"upper left\", fontsize=8)\n", + "\n", + " if savefile is not None:\n", + " plt.savefig(savefile, bbox_inches=\"tight\")\n", + "\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "root_to_plot = [\n", + " \"SP_v1.4.5_A\",\n", + " # \"SP_v1.4.5_A_no_IA\",\n", + " # \"SP_v1.4.5_A_no_dz\",\n", + " # \"SP_v1.4.5_A_no_m_bias\",\n", + " \"SP_v1.4.5_A_sc_3_150\",\n", + " \"SP_v1.4.5_A_sc_3_60\",\n", + " \"SP_v1.4.5_A_sc_10_150\",\n", + " \"SP_v1.4.5_A_sc_10_60\",\n", + " \"SP_v1.4.5_A_sc_5_150\",\n", + " \"SP_v1.4.5_A_sc_7_150\",\n", + " # \"SP_v1.4.5_A_no_leakage\"\n", + "]\n", + "\n", + "root_to_plot = [\n", + " f\"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_{int(i)}.0_80.0_10.0_80.0\"\n", + " for i in [3, 5, 7, 10, 11]\n", + "]\n", + "\n", + "colours = [\n", + " \"red\",\n", + " \"salmon\",\n", + " \"darkorange\",\n", + " \"forestgreen\",\n", + " \"turquoise\",\n", + " \"darkviolet\",\n", + " \"crimson\",\n", + " \"gold\",\n", + " \"lightcoral\",\n", + " \"mediumseagreen\",\n", + " \"lightsteelblue\",\n", + " \"black\",\n", + " \"silver\",\n", + " \"peru\",\n", + " \"maroon\",\n", + " \"olive\",\n", + "]\n", + "\n", + "savefile = \"best_fit_tau_new_binning.png\"\n", + "\n", + "plot_best_fit_tau(root_to_plot, colours, savefile)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pseudo_cell = fits.open(\n", + " \"/home/guerrini/sp_validation/cosmo_val/output/pseudo_cl_SP_v1.4.5.fits\"\n", + ")[1].data\n", + "cov_pseudo_cell = fits.open(\n", + " \"/home/guerrini/sp_validation/cosmo_val/output/pseudo_cl_cov_SP_v1.4.5.fits\"\n", + ")\n", + "\n", + "theory_ell = np.loadtxt(\n", + " \"/n09data/guerrini/output_chains/best_fit/SP_v1.4.5_A/shear_cl/ell.txt\"\n", + ")\n", + "theory_cell = np.loadtxt(\n", + " \"/n09data/guerrini/output_chains/best_fit/SP_v1.4.5_A/shear_cl/bin_1_1.txt\"\n", + ")\n", + "\n", + "pw = hp.pixwin(1024, lmax=2048)\n", + "\n", + "plt.errorbar(\n", + " pseudo_cell[\"ELL\"],\n", + " pseudo_cell[\"ELL\"] * pseudo_cell[\"EE\"],\n", + " yerr=pseudo_cell[\"ELL\"] * np.sqrt(np.diag(cov_pseudo_cell[\"COVAR_EE_EE\"].data)),\n", + " capsize=2,\n", + " c=\"k\",\n", + " fmt=\"o\",\n", + " markersize=2,\n", + ")\n", + "\n", + "mask = (theory_ell > 0.1) & (theory_ell < 2048)\n", + "plt.plot(\n", + " theory_ell[mask],\n", + " theory_ell[mask]\n", + " * theory_cell[mask]\n", + " * np.interp(theory_ell[mask], np.arange(0, 2049), pw) ** 2,\n", + " c=\"r\",\n", + " label=\"best-fit $\\\\theta \\\\in [3-200]$\",\n", + ")\n", + "\n", + "plt.xlabel(r\"$\\ell$\", fontsize=26)\n", + "plt.ylabel(r\"$\\ell C_\\ell^{EE}$\", fontsize=26)\n", + "plt.legend()\n", + "plt.savefig(\"SP_v1.4.5_A_cell.png\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cov_pseudo_cell.info()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sp_validation_3.11", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/cosmo_inference/get_chi2_cell.ipynb b/cosmo_inference/get_chi2_cell.ipynb new file mode 100644 index 00000000..c97748d1 --- /dev/null +++ b/cosmo_inference/get_chi2_cell.ipynb @@ -0,0 +1,1527 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Trick to plot with tex\n", + "os.environ[\"LD_LIBRARY_PATH\"] = \"\"\n", + "os.environ[\"CONDA_PREFIX\"] = \"/home/guerrini/.conda/envs/sp_validation\"\n", + "\n", + "import configparser\n", + "import subprocess\n", + "\n", + "import healpy as hp\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.scale as mscale\n", + "import matplotlib.ticker as ticker\n", + "import matplotlib.transforms as mtransforms\n", + "import numpy as np\n", + "import scipy.stats as stats\n", + "import seaborn as sns\n", + "from astropy.io import fits\n", + "from getdist import plots\n", + "from IPython.display import Markdown, display\n", + "from scipy.interpolate import interp1d\n", + "\n", + "plt.style.use(\"../papers/harmonic/matplotlib_config/paper.mplstyle\")\n", + "\n", + "plt.rcParams[\"text.usetex\"] = True\n", + "\n", + "sns.set_palette(\"husl\")\n", + "\n", + "\n", + "class SquareRootScale(mscale.ScaleBase):\n", + " \"\"\"\n", + " ScaleBase class for generating square root scale.\n", + "\n", + " Usage example: axis.set_yscale('squareroot')\n", + "\n", + " \"\"\"\n", + "\n", + " name = \"squareroot\"\n", + "\n", + " def __init__(self, axis, **kwargs):\n", + " mscale.ScaleBase.__init__(self, axis, **kwargs)\n", + "\n", + " def set_default_locators_and_formatters(self, axis):\n", + " axis.set_major_locator(ticker.AutoLocator())\n", + " axis.set_major_formatter(ticker.ScalarFormatter())\n", + " axis.set_minor_locator(ticker.NullLocator())\n", + " axis.set_minor_formatter(ticker.NullFormatter())\n", + "\n", + " def limit_range_for_scale(self, vmin, vmax, minpos):\n", + " return max(0.0, vmin), vmax\n", + "\n", + " class SquareRootTransform(mtransforms.Transform):\n", + " input_dims = 1\n", + " output_dims = 1\n", + " is_separable = True\n", + "\n", + " def transform_non_affine(self, a):\n", + " return np.array(a) ** 0.5\n", + "\n", + " def inverted(self):\n", + " return SquareRootScale.InvertedSquareRootTransform()\n", + "\n", + " class InvertedSquareRootTransform(mtransforms.Transform):\n", + " input_dims = 1\n", + " output_dims = 1\n", + " is_separable = True\n", + "\n", + " def transform(self, a):\n", + " return np.array(a) ** 2\n", + "\n", + " def inverted(self):\n", + " return SquareRootScale.SquareRootTransform()\n", + "\n", + " def get_transform(self):\n", + " return self.SquareRootTransform()\n", + "\n", + "\n", + "mscale.register_scale(SquareRootScale)\n", + "%matplotlib inline\n", + "# import uncertainties\n", + "\n", + "plt.rc(\"mathtext\", fontset=\"stix\")\n", + "plt.rc(\"font\", family=\"sans-serif\")\n", + "\n", + "g = plots.get_subplot_plotter(width_inch=30)\n", + "g.settings.axes_fontsize = 30\n", + "g.settings.axes_labelsize = 30\n", + "g.settings.alpha_filled_add = 0.7\n", + "g.settings.legend_fontsize = 40\n", + "\n", + "\n", + "# SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE\n", + "root_dir = \"/n09data/guerrini/output_chains/\"\n", + "\n", + "catalog_version = \"SP_v1.4.6_leak_corr_cell\"\n", + "catalog_version_real_space = \"SP_v1.4.6_leak_corr_A_10_80\"\n", + "\n", + "path_ini_files = \"/home/guerrini/sp_validation/cosmo_inference/cosmosis_config/\"\n", + "\n", + "roots = [\n", + " \"SP_v1.4.6_leak_corr_A_lmin=300_lmax=1600_cell\",\n", + " \"SP_v1.4.6_leak_corr_B_lmin=300_lmax=1600_cell\",\n", + " \"SP_v1.4.6_leak_corr_C_lmin=300_lmax=1600_cell\",\n", + " \"SP_v1.4.6_leak_corr_A_10_80\",\n", + " # f\"SP_v1.4.6_leak_corr_A_kmax=5Mpc_cell\",\n", + " \"SP_v1.4.6_leak_corr_A_kmax=3Mpc_cell\",\n", + " \"SP_v1.4.6_leak_corr_A_kmax=1Mpc_cell\",\n", + " \"SP_v1.4.6_leak_corr_A_include_large_scales_cell\",\n", + " \"SP_v1.4.6_leak_corr_A_small_scales_cell\",\n", + " \"SP_v1.4.6_leak_corr_A_large_scales_cell\",\n", + " \"SP_v1.4.6_leak_corr_A_halofit_cell\",\n", + " \"SP_v1.4.6_leak_corr_A_HMCode_nobar_cell\",\n", + " \"SP_v1.4.6_leak_corr_A_OneCov_cell\",\n", + " \"SP_v1.4.6_A_fid_cell\",\n", + "]\n", + "\n", + "labels = [\n", + " r\"UNIONS $C_\\ell$, Blind A\",\n", + " r\"UNIONS $C_\\ell$, Blind B\",\n", + " r\"UNIONS $C_\\ell$, Blind C\",\n", + " r\"UNIONS $\\xi_\\pm(\\vartheta)$, (Goh et al., 2026)\",\n", + " # rf\"$k_\\mathrm{{max}}=5 h$ Mpc$^{{-1}}$, $\\ell_\\mathrm{{max}}=2048$\",\n", + " r\"$k_\\mathrm{max}=3 h$ Mpc$^{-1}$, $\\ell_\\mathrm{max}=1800$\",\n", + " r\"$k_\\mathrm{max}=1 h$ Mpc$^{-1}$, $\\ell_\\mathrm{max}=500$\",\n", + " r\"Include Large Scales, $\\ell_\\mathrm{max}=1600$\",\n", + " \"Small Scales only\",\n", + " \"Large Scales only\",\n", + " r\"Halofit\",\n", + " r\"HMCode no baryons\",\n", + " \"OneCovariance only\",\n", + " \"No leakage correction\",\n", + "]\n", + "\n", + "bases = [\n", + " \"harmonic\",\n", + " \"harmonic\",\n", + " \"harmonic\",\n", + " \"configuration\",\n", + " \"harmonic\",\n", + " \"harmonic\",\n", + " \"harmonic\",\n", + " \"harmonic\",\n", + " \"harmonic\",\n", + " \"harmonic\",\n", + " \"harmonic\",\n", + " \"harmonic\",\n", + " \"harmonic\",\n", + " \"harmonic\",\n", + "]\n", + "\n", + "\n", + "properties = {}\n", + "\n", + "for i, root in enumerate(roots):\n", + " config = configparser.ConfigParser()\n", + " config.optionxform = str # Preserve case sensitivity of option names\n", + " config.read(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\")\n", + "\n", + " try:\n", + " lower_bound_cell_ee, upper_bound_cell_ee = map(\n", + " float, config[\"2pt_like\"][\"angle_range_CELL_EE_1_1\"].split()\n", + " )\n", + "\n", + " properties[root] = {\n", + " \"lower_bound_cell_ee\": lower_bound_cell_ee,\n", + " \"upper_bound_cell_ee\": upper_bound_cell_ee,\n", + " }\n", + " except KeyError:\n", + " properties[root] = {\"lower_bound_cell_ee\": 0.0, \"upper_bound_cell_ee\": 2048.0}\n", + "\n", + " if bases[i] == \"configuration\":\n", + " # Also save the scale cuts in theta for xi\n", + " add_xi_sys = config[\"2pt_like\"][\"add_xi_sys\"]\n", + " add_xi_sys = add_xi_sys == \"T\"\n", + " lower_bound_xi_plus, upper_bound_xi_plus = map(\n", + " float, config[\"2pt_like\"][\"angle_range_XI_PLUS_1_1\"].split()\n", + " )\n", + " lower_bound_xi_minus, upper_bound_xi_minus = map(\n", + " float, config[\"2pt_like\"][\"angle_range_XI_MINUS_1_1\"].split()\n", + " )\n", + "\n", + " properties[root].update(\n", + " {\n", + " \"add_xi_sys\": add_xi_sys,\n", + " \"lower_bound_xi_plus\": lower_bound_xi_plus,\n", + " \"upper_bound_xi_plus\": upper_bound_xi_plus,\n", + " \"lower_bound_xi_minus\": lower_bound_xi_minus,\n", + " \"upper_bound_xi_minus\": upper_bound_xi_minus,\n", + " }\n", + " )\n", + "\n", + "\n", + "print(roots)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Retrieve the chains" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# MAKE PARAMNAMES FILE\n", + "\n", + "for root in roots:\n", + " with open(root_dir + \"{}/samples_{}.txt\".format(\"/\" + root, root), \"r\") as file:\n", + " params = file.readline()[1:].split(\"\\t\")[:-4]\n", + " file.close()\n", + "\n", + " with open(\n", + " root_dir + \"{}/getdist_{}.paramnames\".format(\"/\" + root, root), \"w\"\n", + " ) as file:\n", + " for i in range(len(params)):\n", + " if len(params[i].split(\"--\")) > 1:\n", + " file.write(params[i].split(\"--\")[1] + \"\\n\")\n", + " else:\n", + " file.write(params[i].split(\"--\")[0] + \"\\n\")\n", + " file.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# READ CHAIN\n", + "\n", + "chains = []\n", + "\n", + "for root in roots:\n", + " samples = np.loadtxt(root_dir + \"{}/samples_{}.txt\".format(root, root))\n", + " print(len(samples))\n", + " if \"nautilus\" in root:\n", + " samples = np.column_stack(\n", + " (np.exp(samples[:, -3]), samples[:, -1] - samples[:, -2], samples[:, 0:-3])\n", + " )\n", + " else:\n", + " samples = np.column_stack((samples[:, -1], samples[:, -3], samples[:, 0:-4]))\n", + " np.savetxt(root_dir + \"{}/getdist_{}.txt\".format(root, root), samples)\n", + "\n", + " chain = g.samples_for_root(\n", + " root_dir + \"{}/getdist_{}\".format(root, root),\n", + " cache=False,\n", + " settings={\"ignore_rows\": 0, \"smooth_scale_2D\": 0.3, \"smooth_scale_1D\": 0.3},\n", + " )\n", + "\n", + " chains.append(chain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "name_list = [\n", + " \"OMEGA_M\",\n", + " \"ombh2\",\n", + " \"h0\",\n", + " \"n_s\",\n", + " \"SIGMA_8\",\n", + " \"s_8_input\",\n", + " \"logt_agn\",\n", + " \"a\",\n", + " \"m1\",\n", + " \"bias_1\",\n", + "]\n", + "label_list = [\n", + " r\"\\Omega_m\",\n", + " r\"\\omega_b h^2\",\n", + " \"h_0\",\n", + " \"n_s\",\n", + " r\"\\sigma_8\",\n", + " \"S_8\",\n", + " \"log T_{AGN}\",\n", + " \"A_{IA}\",\n", + " \"m_1\",\n", + " r\"\\Delta z_1\",\n", + "]\n", + "\n", + "for chain in chains:\n", + " param_names = chain.getParamNames()\n", + " for name, label in zip(name_list, label_list):\n", + " param_names.parWithName(name).label = label" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Extract the best fit parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "best_fit = {}\n", + "\n", + "for root, chain in zip(roots, chains):\n", + " print(root)\n", + " likestats = chain.getLikeStats()\n", + " bestfit_idx = np.argmax(chain.loglikes)\n", + " maxlike = chain.loglikes[bestfit_idx]\n", + " print(f\"Maximum Likelihood: {maxlike:.5g}\")\n", + " best_fit[root] = {\"likelihood\": maxlike}\n", + " margestats = chain.getMargeStats()\n", + " s8_stats = margestats.parWithName(\"S_8\")\n", + " sigma8_stats = margestats.parWithName(\"SIGMA_8\")\n", + " omegam_stats = margestats.parWithName(\"OMEGA_M\")\n", + " a_ia_stats = margestats.parWithName(\"a\")\n", + "\n", + " best_fit[root].update(\n", + " {\n", + " \"S_8_mean\": s8_stats.mean,\n", + " \"S_8_lower\": s8_stats.mean - s8_stats.limits[0].lower,\n", + " \"S_8_upper\": s8_stats.limits[0].upper - s8_stats.mean,\n", + " \"sigma_8_mean\": sigma8_stats.mean,\n", + " \"sigma_8_lower\": sigma8_stats.mean - sigma8_stats.limits[0].lower,\n", + " \"sigma_8_upper\": sigma8_stats.limits[0].upper - sigma8_stats.mean,\n", + " \"omega_m_mean\": omegam_stats.mean,\n", + " \"omega_m_lower\": omegam_stats.mean - omegam_stats.limits[0].lower,\n", + " \"omega_m_upper\": omegam_stats.limits[0].upper - omegam_stats.mean,\n", + " \"A_IA_mean\": a_ia_stats.mean,\n", + " \"A_IA_lower\": a_ia_stats.mean - a_ia_stats.limits[0].lower,\n", + " \"A_IA_upper\": a_ia_stats.limits[0].upper - a_ia_stats.mean,\n", + " }\n", + " )\n", + " try:\n", + " t_agn_stats = margestats.parWithName(\"logt_agn\")\n", + " best_fit[root].update(\n", + " {\n", + " \"logt_agn_mean\": t_agn_stats.mean,\n", + " \"logt_agn_lower\": t_agn_stats.mean - t_agn_stats.limits[0].lower,\n", + " \"logt_agn_upper\": t_agn_stats.limits[0].upper - t_agn_stats.mean,\n", + " }\n", + " )\n", + " except Exception:\n", + " pass\n", + " for i, par in enumerate(likestats.names):\n", + " best_fit[root].update(\n", + " {par.name: np.average(chain.samples[:, i], weights=chain.weights)}\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run `Cosmosis` in test mode to get the data vectors" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if not os.path.exists(path_ini_files + \"/values_empty.ini\"):\n", + " content = \"\"\"[cosmological_parameters]\n", + "\n", + "tau = 0.0544\n", + "w = -1.0\n", + "mnu = 0.06\n", + "omega_k = 0.0\n", + "wa = 0.0\n", + "\n", + "[halo_model_parameters]\n", + "\n", + "[intrinsic_alignment_parameters]\n", + "\n", + "[shear_calibration_parameters]\n", + "\n", + "[nofz_shifts]\n", + "\n", + "[psf_leakage_parameters]\n", + "\"\"\"\n", + "\n", + " with open(path_ini_files + \"/values_empty.ini\", \"w\") as f:\n", + " f.write(content)\n", + " f.close()\n", + "\n", + " print(\"File created successfully\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "section_map = {\n", + " \"omch2\": \"cosmological_parameters\",\n", + " \"ombh2\": \"cosmological_parameters\",\n", + " \"h0\": \"cosmological_parameters\",\n", + " \"n_s\": \"cosmological_parameters\",\n", + " \"s_8_input\": \"cosmological_parameters\",\n", + " \"logt_agn\": \"halo_model_parameters\",\n", + " \"a\": \"intrinsic_alignment_parameters\",\n", + " \"m1\": \"shear_calibration_parameters\",\n", + " \"bias_1\": \"nofz_shifts\",\n", + " \"alpha\": \"psf_leakage_parameters\",\n", + " \"beta\": \"psf_leakage_parameters\",\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "env = os.environ.copy()\n", + "env[\"LD_LIBRARY_PATH\"] = (\n", + " \"/home/guerrini/.conda/envs/sp_validation/lib/python3.9/site-packages/cosmosis/datablock:\"\n", + " + env.get(\"LD_LIBRARY_PATH\", \"\")\n", + ")\n", + "\n", + "for root in roots:\n", + " print(root)\n", + " config = configparser.ConfigParser()\n", + " config.optionxform = str # Preserve case sensitivity of option names\n", + " config.read(path_ini_files + \"/values_empty.ini\")\n", + " for param, value in best_fit[root].items():\n", + " section = section_map.get(param)\n", + " if section is None:\n", + " continue\n", + " if section not in config:\n", + " config.add_section(section)\n", + " config[section][param] = str(value)\n", + "\n", + " with open(path_ini_files + \"/values_empty.ini\", \"w\") as configfile:\n", + " config.write(configfile)\n", + "\n", + " # Modify the ini file to run in test mode at the best fit\n", + " config = configparser.ConfigParser()\n", + " config.optionxform = str # Preserve case sensitivity of option names\n", + " config.read(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\")\n", + "\n", + " sampler = config[\"runtime\"][\"sampler\"]\n", + " config[\"runtime\"][\"sampler\"] = \"test\"\n", + " values = config[\"pipeline\"][\"values\"]\n", + " config[\"pipeline\"][\"values\"] = path_ini_files + \"/values_empty.ini\"\n", + "\n", + " with open(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\", \"w\") as configfile:\n", + " config.write(configfile)\n", + "\n", + " # Run cosmosis\n", + " result = subprocess.run(\n", + " [\"cosmosis\", \"cosmosis_config/cosmosis_pipeline_{}.ini\".format(root)],\n", + " env=env,\n", + " capture_output=True,\n", + " text=True,\n", + " )\n", + " print(f\"STDOUT:\\n{result.stdout}\")\n", + " print(f\"STDERR:\\n{result.stderr}\")\n", + "\n", + " # Modify the ini file to the previous one\n", + " config[\"pipeline\"][\"values\"] = values\n", + " config[\"runtime\"][\"sampler\"] = sampler\n", + "\n", + " with open(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\", \"w\") as configfile:\n", + " config.write(configfile)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute the $\\chi^2$" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "output_folder = \"/n09data/guerrini/output_chains/\"\n", + "\n", + "metrics = {}\n", + "\n", + "for i, root in enumerate(roots):\n", + " print(root)\n", + "\n", + " base = bases[i]\n", + "\n", + " if base == \"harmonic\":\n", + " # Remove cell from the end of root\n", + " root_cell_removed = root.replace(\"_cell\", \"\")\n", + "\n", + " lower_bound_cell_ee = properties[root][\"lower_bound_cell_ee\"]\n", + " upper_bound_cell_ee = properties[root][\"upper_bound_cell_ee\"]\n", + " print(upper_bound_cell_ee)\n", + "\n", + " # Read the results\n", + " ell = np.loadtxt(output_folder + \"best_fit/{}/shear_cl/ell.txt\".format(root))\n", + " shear_cl = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_cl/bin_1_1.txt\".format(root)\n", + " )\n", + "\n", + " # Read the data\n", + " data = fits.open(f\"data/{root_cell_removed}/cosmosis_{root}.fits\")\n", + "\n", + " ell_data = data[\"CELL_EE\"].data[\"ANG\"]\n", + " cell_data = data[\"CELL_EE\"].data[\"VALUE\"]\n", + "\n", + " # Load the covariance\n", + " cov = data[\"COVMAT\"].data\n", + " cov_cell = cov\n", + "\n", + " # interpolate the model\n", + " interp_cell_ee = interp1d(ell, shear_cl, kind=\"cubic\", fill_value=\"extrapolate\")\n", + "\n", + " cell_model = interp_cell_ee(ell_data)\n", + "\n", + " # Apply scale cuts\n", + " mask_cell = (ell_data > lower_bound_cell_ee) & (ell_data < upper_bound_cell_ee)\n", + " cell_data = cell_data[mask_cell]\n", + " cell_model = cell_model[mask_cell]\n", + " cov_cell = cov_cell[mask_cell][:, mask_cell]\n", + "\n", + " cell_chi2 = np.dot(\n", + " (cell_model - cell_data),\n", + " np.dot(np.linalg.inv(cov_cell), (cell_model - cell_data)),\n", + " )\n", + " n_dof_cell = np.sum(mask_cell)\n", + " print(n_dof_cell)\n", + " n_dof_cell -= 9\n", + " p_value_cell = 1 - stats.chi2.cdf(cell_chi2, n_dof_cell)\n", + "\n", + " metrics[root] = {\n", + " \"chi2\": cell_chi2,\n", + " \"n_dof\": n_dof_cell,\n", + " \"p_value\": p_value_cell,\n", + " }\n", + "\n", + " elif base == \"configuration\":\n", + " add_xi_sys = properties[root][\"add_xi_sys\"]\n", + " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", + " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", + " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", + " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", + "\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " shear_xi_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " shear_xi_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_sys_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + "\n", + " # Read model tau_stats\n", + " theta_tau = np.loadtxt(\n", + " output_folder + \"best_fit/{}/tau_0_plus/theta.txt\".format(root)\n", + " )\n", + " theta_tau_arcmin = theta_tau * 180 * 60 / np.pi\n", + " tau_0_model = np.loadtxt(\n", + " output_folder + \"best_fit/{}/tau_0_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " tau_2_model = np.loadtxt(\n", + " output_folder + \"best_fit/{}/tau_2_plus/bin_1_1.txt\".format(root)\n", + " )\n", + "\n", + " # Read the data\n", + " data = fits.open(\n", + " f\"data/{catalog_version_real_space}/cosmosis_{catalog_version_real_space}.fits\"\n", + " )\n", + "\n", + " theta_data = data[\"XI_PLUS\"].data[\"ANG\"]\n", + " xi_plus_data = data[\"XI_PLUS\"].data[\"VALUE\"]\n", + " xi_minus_data = data[\"XI_MINUS\"].data[\"VALUE\"]\n", + " tau_0_data = data[\"TAU_0_PLUS\"].data[\"VALUE\"]\n", + " tau_2_data = data[\"TAU_2_PLUS\"].data[\"VALUE\"]\n", + "\n", + " # Load the covariance\n", + " cov = data[\"COVMAT\"].data\n", + " cov_xi = cov[0 : 2 * len(xi_plus_data), 0 : 2 * len(xi_plus_data)]\n", + " cov_tau = cov[2 * len(xi_plus_data) :, 2 * len(xi_plus_data) :]\n", + "\n", + " # interpolate the model\n", + " interp_xi_plus = interp1d(\n", + " theta_arcmin, shear_xi_plus, kind=\"cubic\", fill_value=\"extrapolate\"\n", + " )\n", + " interp_xi_minus = interp1d(\n", + " theta_arcmin, shear_xi_minus, kind=\"cubic\", fill_value=\"extrapolate\"\n", + " )\n", + "\n", + " xi_plus_model = interp_xi_plus(theta_data)\n", + " if add_xi_sys:\n", + " xi_plus_model += xi_sys_plus\n", + " xi_minus_model = interp_xi_minus(theta_data)\n", + " if add_xi_sys:\n", + " xi_minus_model += xi_sys_minus\n", + "\n", + " # Concatenate the data vector\n", + " xi_data = np.concatenate((xi_plus_data, xi_minus_data))\n", + " xi_model = np.concatenate((xi_plus_model, xi_minus_model))\n", + "\n", + " tau_data = np.concatenate((tau_0_data, tau_2_data))\n", + " tau_model = np.concatenate((tau_0_model, tau_2_model))\n", + "\n", + " # Apply scale cuts\n", + " mask_xi_plus = (theta_data > lower_bound_xi_plus) & (\n", + " theta_data < upper_bound_xi_plus\n", + " )\n", + " mask_xi_minus = (theta_data > lower_bound_xi_minus) & (\n", + " theta_data < upper_bound_xi_minus\n", + " )\n", + " mask = np.concatenate((mask_xi_plus, mask_xi_minus))\n", + "\n", + " xi_data = xi_data[mask]\n", + " xi_model = xi_model[mask]\n", + " cov_xi = cov_xi[mask][:, mask]\n", + "\n", + " xi_plus_chi2 = np.dot(\n", + " (xi_model - xi_data), np.dot(np.linalg.inv(cov_xi), (xi_model - xi_data))\n", + " )\n", + " tau_chi2 = np.dot(\n", + " (tau_model - tau_data),\n", + " np.dot(np.linalg.inv(cov_tau), (tau_model - tau_data)),\n", + " )\n", + " n_dof_xi = np.sum(mask)\n", + " n_dof_xi -= 11\n", + " n_dof_tau = len(tau_0_data) + len(tau_2_data)\n", + " p_value_xi = 1 - stats.chi2.cdf(xi_plus_chi2, n_dof_xi)\n", + " p_value_tau = 1 - stats.chi2.cdf(tau_chi2, n_dof_tau)\n", + " chi2_tot = xi_plus_chi2 + tau_chi2\n", + " n_dof_tot = n_dof_xi + n_dof_tau\n", + " p_value_tot = 1 - stats.chi2.cdf(chi2_tot, n_dof_tot)\n", + "\n", + " metrics[root] = {\"chi2\": xi_plus_chi2, \"n_dof\": n_dof_xi, \"p_value\": p_value_xi}\n", + "\n", + " print(\"Done!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def get_latex_table(metrics):\n", + " latex_lines = [\n", + " r\"\\begin{tabular}{|l|c|c|c|c|c|c|c|}\",\n", + " r\"\\hline\",\n", + " r\"Experiment name & $S_8$ & $\\Omega_m$ & $\\sigma_8$ & $A_\\mathrm{IA}$ & $\\log T_\\mathrm{AGN}$ & $\\chi^2$/dof & PTE \\\\ \",\n", + " r\"\\hline\",\n", + " ]\n", + "\n", + " for i, (root, vals) in enumerate(metrics.items()):\n", + " label = labels[i]\n", + " best_fit_vals = best_fit[root]\n", + " log_t_agn_mean = best_fit_vals.get(\"logt_agn_mean\", None)\n", + "\n", + " if log_t_agn_mean is None:\n", + " logt_agn_mean_str = \"N/A\"\n", + " else:\n", + " logt_agn_mean_str = f\"${best_fit_vals['logt_agn_mean']:.3f}^{{+{best_fit_vals['logt_agn_upper']:.3f}}}_{{-{best_fit_vals['logt_agn_lower']:.3f}}}$\"\n", + " line = (\n", + " f\"{label} & \"\n", + " rf\" ${best_fit_vals['S_8_mean']:.3f}^{{+{best_fit_vals['S_8_upper']:.3f}}}_{{-{best_fit_vals['S_8_lower']:.3f}}}$ & \"\n", + " rf\" ${best_fit_vals['omega_m_mean']:.3f}^{{+{best_fit_vals['omega_m_upper']:.3f}}}_{{-{best_fit_vals['omega_m_lower']:.3f}}}$ & \"\n", + " rf\" ${best_fit_vals['sigma_8_mean']:.3f}^{{+{best_fit_vals['sigma_8_upper']:.3f}}}_{{-{best_fit_vals['sigma_8_lower']:.3f}}}$ & \"\n", + " rf\" ${best_fit_vals['A_IA_mean']:.3f}^{{+{best_fit_vals['A_IA_upper']:.3f}}}_{{-{best_fit_vals['A_IA_lower']:.3f}}}$ & \"\n", + " rf\" {logt_agn_mean_str} & \"\n", + " f\"{vals['chi2']:.2f}/{vals['n_dof']} & {vals['p_value']:.5f} \\\\\\\\\"\n", + " )\n", + " latex_lines.append(line)\n", + "\n", + " latex_lines.append(r\"\\hline\")\n", + " latex_lines.append(r\"\\end{tabular}\")\n", + "\n", + " # Print LaTeX table\n", + " print(\"\\n\".join(latex_lines))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "get_latex_table(metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def display_markdown(metrics):\n", + " # Build Markdown table\n", + " header = (\n", + " \"| Root | $\\\\chi^2$ ($C_\\\\ell$) / dof | p-val ($C_\\\\ell$) |\\n\"\n", + " \"|------|----------------|------------|\\n\"\n", + " )\n", + "\n", + " rows = []\n", + " for root, vals in metrics.items():\n", + " row = f\"| `{root}` \"\n", + " row += f\"| {vals['chi2']:.2f} / {vals['n_dof']} \"\n", + " row += f\"| {vals['p_value']:.5f} \"\n", + " rows.append(row)\n", + "\n", + " # Display in Jupyter\n", + " display(Markdown(header + \"\\n\".join(rows)))\n", + " return header + \"\\n\".join(rows)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "markdown_source = display_markdown(metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "markdown_source" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot the best-fit of each model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "catalog_version = \"SP_v1.4.6_leak_corr_A_lmin=300_lmax=1600\"\n", + "data = fits.open(\n", + " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_version}/cosmosis_{catalog_version}_cell.fits\"\n", + ")\n", + "cell_ee = data[\"CELL_EE\"].data\n", + "cov_mat = data[\"COVMAT\"].data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig, ax = plt.subplots(1, 1, figsize=(15, 8))\n", + "\n", + "ell = cell_ee[\"ANG\"]\n", + "cell = cell_ee[\"VALUE\"]\n", + "\n", + "ax.errorbar(\n", + " ell,\n", + " ell * cell,\n", + " yerr=ell * np.sqrt(np.diag(cov)),\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + ")\n", + "ax.set_xlabel(r\"$\\ell$\")\n", + "ax.set_ylabel(r\"$\\ell C_\\ell$\")\n", + "ax.set_xlim(ell.min() - 10, ell.max() + 100)\n", + "ax.set_xscale(\"squareroot\")\n", + "ax.set_xticks(np.array([100, 400, 900, 1600]))\n", + "ax.minorticks_on()\n", + "ax.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", + "minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)]\n", + "ax.xaxis.set_ticks(minor_ticks, minor=True)\n", + "\n", + "plt.legend(fontsize=15)\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_best_fit(\n", + " data_points,\n", + " root_to_plot,\n", + " line_args,\n", + " savefile,\n", + " ell_min=10.0,\n", + " ell_max=2048.0,\n", + " multiply_ell=True,\n", + " loc_legend=\"best\",\n", + " bbox_to_anchor=None,\n", + " label_data=\"Fiducial data\",\n", + " labels=None,\n", + "):\n", + " data = fits.open(\n", + " f\"/home/guerrini/sp_validation/cosmo_inference/data/{data_points}/cosmosis_{data_points}_cell.fits\"\n", + " )\n", + " cell_ee = data[\"CELL_EE\"].data\n", + " cov_mat = data[\"COVMAT\"].data\n", + "\n", + " if labels is None:\n", + " labels = root_to_plot\n", + "\n", + " fig, ax = plt.subplots(1, 1, figsize=(8, 5))\n", + "\n", + " ell, cell = cell_ee[\"ANG\"], cell_ee[\"VALUE\"]\n", + " ax.errorbar(\n", + " ell,\n", + " ell * cell,\n", + " yerr=ell * np.sqrt(np.diag(cov_mat)),\n", + " fmt=\"o\",\n", + " label=label_data,\n", + " color=\"black\",\n", + " capsize=2,\n", + " )\n", + "\n", + " for idx, (label, root) in enumerate(zip(labels, root_to_plot)):\n", + " # Read the results\n", + " ell = np.loadtxt(output_folder + \"best_fit/{}/shear_cl/ell.txt\".format(root))\n", + " shear_cl = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_cl/bin_1_1.txt\".format(root)\n", + " )\n", + "\n", + " mask = (ell > ell_min) & (ell < ell_max)\n", + "\n", + " ax.plot(\n", + " ell[mask],\n", + " ell[mask] * shear_cl[mask] if multiply_ell else shear_cl[mask],\n", + " label=label,\n", + " **line_args[idx],\n", + " )\n", + "\n", + " # Plot the scale cuts for different k_max\n", + " ax.axvline(x=1800, color=\"black\", linestyle=\"--\", alpha=0.5)\n", + " ax.axvline(x=2048, color=\"black\", linestyle=\"--\", alpha=1.0)\n", + " ax.axvline(x=500, color=\"black\", linestyle=\"--\", alpha=0.3)\n", + "\n", + " # Add labels directly under the tick\n", + " ax.text(\n", + " 1740,\n", + " 0.90,\n", + " r\"$k_\\mathrm{max} = 3 h$ Mpc$^{-1}$\",\n", + " transform=ax.get_xaxis_transform(),\n", + " ha=\"center\",\n", + " va=\"top\",\n", + " fontsize=14,\n", + " rotation=90,\n", + " )\n", + "\n", + " ax.text(\n", + " 1978,\n", + " 0.90,\n", + " r\"$k_\\mathrm{max} = 5 h$ Mpc$^{-1}$\",\n", + " transform=ax.get_xaxis_transform(),\n", + " ha=\"center\",\n", + " va=\"top\",\n", + " fontsize=14,\n", + " rotation=90,\n", + " )\n", + "\n", + " ax.text(\n", + " 470,\n", + " 0.90,\n", + " r\"$k_\\mathrm{max} = 1 h$ Mpc$^{-1}$\",\n", + " transform=ax.get_xaxis_transform(),\n", + " ha=\"center\",\n", + " va=\"top\",\n", + " fontsize=14,\n", + " rotation=90,\n", + " )\n", + "\n", + " ell, cell = cell_ee[\"ANG\"], cell_ee[\"VALUE\"]\n", + " ax.set_ylabel(r\"$\\ell C_\\ell \\times 10^{-7}$\", fontsize=20)\n", + " ax.set_xlabel(r\"Multipole $\\ell$\", fontsize=20)\n", + " ax.set_xlim(ell.min() - 10, ell.max() + 100)\n", + " ax.set_xscale(\"squareroot\")\n", + " ax.set_xticks(np.array([100, 400, 900, 1600]))\n", + " ax.minorticks_on()\n", + " ax.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", + " minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)]\n", + " ax.xaxis.set_ticks(minor_ticks, minor=True)\n", + " ax.tick_params(axis=\"both\", which=\"major\", labelsize=14)\n", + " ax.tick_params(axis=\"both\", which=\"minor\", labelsize=10)\n", + " ax.yaxis.get_offset_text().set_visible(False)\n", + "\n", + " plt.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor, fontsize=11)\n", + "\n", + " if savefile is not None:\n", + " plt.savefig(savefile, bbox_inches=\"tight\")\n", + "\n", + " plt.show()\n", + "\n", + "\n", + "def plot_best_fit_ratio(\n", + " root_to_plot, colours, savefile, theta_min=1.0, theta_max=250.0\n", + "):\n", + " data = fits.open(\n", + " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_version}/cosmosis_{catalog_version}.fits\"\n", + " )\n", + " xi_plus = data[\"XI_PLUS\"].data\n", + " xi_minus = data[\"XI_MINUS\"].data\n", + " cov_mat = data[\"COVMAT\"].data\n", + "\n", + " plt.figure(figsize=(15, 15))\n", + "\n", + " plt.subplot(211)\n", + "\n", + " root = roots[0]\n", + " add_xi_sys = properties[root][\"add_xi_sys\"]\n", + " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", + " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", + " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", + " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", + "\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " shear_xi_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " shear_xi_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_sys_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + " theta_xi_sys = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", + " )\n", + " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", + "\n", + " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", + " xi_plus_model_fiducial = shear_xi_plus[mask]\n", + " if add_xi_sys:\n", + " xi_plus_model_fiducial += np.interp(\n", + " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_plus\n", + " )\n", + "\n", + " plt.errorbar(\n", + " xi_plus[\"ANG\"],\n", + " xi_plus[\"VALUE\"]\n", + " / np.interp(xi_plus[\"ANG\"], theta_arcmin[mask], xi_plus_model_fiducial),\n", + " yerr=np.sqrt(np.diag(cov_mat))[:20]\n", + " / np.abs(np.interp(xi_plus[\"ANG\"], theta_arcmin[mask], xi_plus_model_fiducial)),\n", + " fmt=\"o\",\n", + " label=f\"{catalog_version} data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + " )\n", + "\n", + " for root, color in zip(root_to_plot, colours):\n", + " add_xi_sys = properties[root][\"add_xi_sys\"]\n", + " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", + " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", + " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", + " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", + "\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " shear_xi_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " shear_xi_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_sys_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + " theta_xi_sys = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", + " )\n", + " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", + "\n", + " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", + " xi_plus_model = shear_xi_plus[mask]\n", + " if add_xi_sys:\n", + " xi_plus_model += np.interp(\n", + " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_plus\n", + " )\n", + "\n", + " alpha = 1.0 if root == roots[0] else 0.5\n", + " plt.plot(\n", + " theta_arcmin[mask],\n", + " xi_plus_model / xi_plus_model_fiducial,\n", + " color=color,\n", + " label=root,\n", + " alpha=alpha,\n", + " )\n", + " plt.axvline(x=lower_bound_xi_plus, color=color, linestyle=\"--\", alpha=0.3)\n", + " plt.axvline(x=upper_bound_xi_plus, color=color, linestyle=\"--\", alpha=0.3)\n", + "\n", + " plt.ylabel(r\"$\\xi_{+}/\\xi_{+, \\text{fid}}$\", fontsize=26)\n", + " plt.xscale(\"log\")\n", + " # plt.yscale('log')\n", + " plt.legend(loc=\"lower left\", fontsize=8)\n", + "\n", + " plt.subplot(212)\n", + "\n", + " root = roots[0]\n", + " add_xi_sys = properties[root][\"add_xi_sys\"]\n", + " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", + " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", + " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", + " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", + "\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " shear_xi_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " shear_xi_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_sys_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + " theta_xi_sys = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", + " )\n", + " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", + "\n", + " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", + " xi_minus_model_fiducial = shear_xi_minus[mask]\n", + " if add_xi_sys:\n", + " xi_minus_model_fiducial += np.interp(\n", + " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_minus\n", + " )\n", + "\n", + " plt.errorbar(\n", + " xi_minus[\"ANG\"],\n", + " xi_minus[\"VALUE\"]\n", + " / np.interp(xi_minus[\"ANG\"], theta_arcmin[mask], xi_minus_model_fiducial),\n", + " yerr=np.sqrt(np.diag(cov_mat))[20:40]\n", + " / np.abs(\n", + " np.interp(xi_minus[\"ANG\"], theta_arcmin[mask], xi_minus_model_fiducial)\n", + " ),\n", + " fmt=\"o\",\n", + " label=f\"{catalog_version} data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + " )\n", + "\n", + " for root, color in zip(root_to_plot, colours):\n", + " add_xi_sys = properties[root][\"add_xi_sys\"]\n", + " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", + " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", + " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", + " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", + "\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " shear_xi_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " shear_xi_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_sys_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + " theta_xi_sys = np.loadtxt(\n", + " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", + " )\n", + " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", + "\n", + " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", + " xi_minus_model = shear_xi_minus[mask]\n", + " if add_xi_sys:\n", + " xi_minus_model += np.interp(\n", + " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_minus\n", + " )\n", + "\n", + " alpha = 1.0 if root == roots[0] else 0.5\n", + " plt.plot(\n", + " theta_arcmin[mask],\n", + " xi_minus_model / xi_minus_model_fiducial,\n", + " color=color,\n", + " label=root,\n", + " alpha=alpha,\n", + " )\n", + " plt.axvline(x=lower_bound_xi_minus, color=color, linestyle=\"--\", alpha=0.3)\n", + " plt.axvline(x=upper_bound_xi_minus, color=color, linestyle=\"--\", alpha=0.3)\n", + "\n", + " plt.xlabel(r\"$\\theta$ [arcmin]\", fontsize=26)\n", + " plt.ylabel(r\"$\\xi_{-}/\\xi_{-, \\text{fid}}$\", fontsize=26)\n", + " plt.xscale(\"log\")\n", + " plt.ylim(0, 2)\n", + " # plt.yscale('log')\n", + " plt.legend(loc=\"lower left\", fontsize=8)\n", + "\n", + " if savefile is not None:\n", + " plt.savefig(savefile, bbox_inches=\"tight\")\n", + "\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.rcParams.update({\"font.family\": \"serif\"})\n", + "\n", + "root_to_plot = [\n", + " \"SP_v1.4.6_leak_corr_A_lmin=300_lmax=1600_cell\",\n", + " \"SP_v1.4.6_leak_corr_A_halofit_cell\",\n", + " \"SP_v1.4.6_leak_corr_A_10_80\",\n", + "]\n", + "\n", + "labels = [\n", + " r\"UNIONS $C_\\ell$, Blind A\",\n", + " r\"UNIONS $C_\\ell$, Halofit\",\n", + " r\"UNIONS $\\xi_\\pm(\\vartheta)$ (Goh et al., 2026)\",\n", + "]\n", + "\n", + "line_args = [\n", + " {\"color\": \"royalblue\", \"linestyle\": \"-\"},\n", + " {\"color\": \"royalblue\", \"linestyle\": \"--\"},\n", + " {\"color\": \"orange\", \"linestyle\": \"-\"},\n", + "]\n", + "\n", + "log_legend = \"lower center\"\n", + "bbox_to_anchor = (0.685, 0.70)\n", + "\n", + "savefile = \"../papers/harmonic/plots/paperplot_Cell_EE_and_best_fit.png\"\n", + "\n", + "plot_best_fit(\n", + " catalog_version,\n", + " root_to_plot,\n", + " line_args,\n", + " savefile,\n", + " labels=labels,\n", + " loc_legend=log_legend,\n", + " bbox_to_anchor=bbox_to_anchor,\n", + ")\n", + "\n", + "savefile = \"../papers/harmonic/plots/paperplot_Cell_EE_and_best_fit.pdf\"\n", + "\n", + "plot_best_fit(\n", + " catalog_version,\n", + " root_to_plot,\n", + " line_args,\n", + " savefile,\n", + " labels=labels,\n", + " loc_legend=log_legend,\n", + " bbox_to_anchor=bbox_to_anchor,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "root_to_plot = [\n", + " \"SP_v1.4.6_leak_corr_A_small_scales_cell\",\n", + " \"SP_v1.4.6_leak_corr_A_large_scales_cell\",\n", + "]\n", + "\n", + "labels = [r\"Small scales only\", r\"Large scales only\"]\n", + "\n", + "line_args = [\n", + " {\"color\": \"royalblue\", \"linestyle\": \"-\"},\n", + " {\"color\": \"royalblue\", \"linestyle\": \"--\"},\n", + "]\n", + "\n", + "savefile = \"../papers/harmonic/plots/small_vs_large_scale.png\"\n", + "\n", + "plot_best_fit(catalog_version, root_to_plot, line_args, savefile, labels=labels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# DEPRECATED code\n", + "\n", + "root_to_plot = [\n", + " \"SP_v1.4.5_A\",\n", + " # \"SP_v1.4.5_A_no_IA\",\n", + " # \"SP_v1.4.5_A_no_dz\",\n", + " # \"SP_v1.4.5_A_no_m_bias\",\n", + " \"SP_v1.4.5_A_sc_3_150\",\n", + " \"SP_v1.4.5_A_sc_3_60\",\n", + " \"SP_v1.4.5_A_sc_10_150\",\n", + " \"SP_v1.4.5_A_sc_10_60\",\n", + " \"SP_v1.4.5_A_sc_5_150\",\n", + " \"SP_v1.4.5_A_sc_7_150\",\n", + " # \"SP_v1.4.5_A_no_leakage\"\n", + "]\n", + "\n", + "\"\"\" root_to_plot = [\n", + " f\"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_{int(i)}.0_80.0_10.0_80.0\" for i in [3, 5, 7, 10, 11]\n", + "] \"\"\"\n", + "\n", + "root_to_plot = [\n", + " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0\",\n", + " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0_no_alpha_beta\",\n", + "]\n", + "\n", + "\n", + "colours = [\n", + " \"red\",\n", + " \"salmon\",\n", + " \"darkorange\",\n", + " \"forestgreen\",\n", + " \"turquoise\",\n", + " \"darkviolet\",\n", + " \"crimson\",\n", + " \"gold\",\n", + " \"lightcoral\",\n", + " \"mediumseagreen\",\n", + " \"lightsteelblue\",\n", + " \"black\",\n", + " \"silver\",\n", + " \"peru\",\n", + " \"maroon\",\n", + " \"olive\",\n", + "]\n", + "\n", + "savefile = \"best_fit_ratio_w_wo_leakage.png\"\n", + "\n", + "plot_best_fit_ratio(root_to_plot, colours, savefile)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_best_fit_tau(root_to_plot, colours, savefile, theta_min=1.0, theta_max=250.0):\n", + " data = fits.open(\n", + " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_version}/cosmosis_{catalog_version}.fits\"\n", + " )\n", + " tau_0 = data[\"TAU_0_PLUS\"].data\n", + " tau_2 = data[\"TAU_2_PLUS\"].data\n", + " cov_mat = data[\"COVMAT\"].data\n", + "\n", + " plt.figure(figsize=(15, 15))\n", + "\n", + " plt.subplot(211)\n", + "\n", + " plt.errorbar(\n", + " tau_0[\"ANG\"],\n", + " tau_0[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[40:60],\n", + " fmt=\"o\",\n", + " label=f\"{catalog_version} data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + " )\n", + "\n", + " for root, color in zip(root_to_plot, colours):\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/tau_0_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " tau_0_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/tau_0_plus/bin_1_1.txt\".format(root)\n", + " )\n", + "\n", + " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", + "\n", + " plt.plot(\n", + " theta_arcmin[mask], tau_0_plus[mask], color=color, label=root, alpha=0.5\n", + " )\n", + "\n", + " plt.ylabel(r\"$\\tau_0$\", fontsize=26)\n", + " plt.xscale(\"log\")\n", + " # plt.yscale('log')\n", + " plt.legend(loc=\"upper right\", fontsize=8)\n", + "\n", + " plt.subplot(212)\n", + "\n", + " y_plot_tau_2 = tau_2[\"ANG\"] * tau_2[\"VALUE\"]\n", + " y_errorbar = tau_2[\"ANG\"] * np.sqrt(np.diag(cov_mat))[60:80]\n", + " plt.errorbar(\n", + " tau_2[\"ANG\"],\n", + " y_plot_tau_2,\n", + " yerr=y_errorbar,\n", + " fmt=\"o\",\n", + " label=f\"{catalog_version} data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + " )\n", + "\n", + " for root, color in zip(root_to_plot, colours):\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"best_fit/{}/tau_2_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " tau_2_plus = np.loadtxt(\n", + " output_folder + \"best_fit/{}/tau_2_plus/bin_1_1.txt\".format(root)\n", + " )\n", + "\n", + " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", + "\n", + " plt.plot(\n", + " theta_arcmin[mask],\n", + " theta_arcmin[mask] * tau_2_plus[mask],\n", + " color=color,\n", + " label=root,\n", + " alpha=0.5,\n", + " )\n", + "\n", + " plt.xlabel(r\"$\\theta$ [arcmin]\", fontsize=26)\n", + " plt.ylabel(r\"$\\theta \\tau_2$\", fontsize=26)\n", + " plt.xscale(\"log\")\n", + " # plt.yscale('log')\n", + " plt.legend(loc=\"upper left\", fontsize=8)\n", + "\n", + " if savefile is not None:\n", + " plt.savefig(savefile, bbox_inches=\"tight\")\n", + "\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "root_to_plot = [\n", + " \"SP_v1.4.5_A\",\n", + " # \"SP_v1.4.5_A_no_IA\",\n", + " # \"SP_v1.4.5_A_no_dz\",\n", + " # \"SP_v1.4.5_A_no_m_bias\",\n", + " \"SP_v1.4.5_A_sc_3_150\",\n", + " \"SP_v1.4.5_A_sc_3_60\",\n", + " \"SP_v1.4.5_A_sc_10_150\",\n", + " \"SP_v1.4.5_A_sc_10_60\",\n", + " \"SP_v1.4.5_A_sc_5_150\",\n", + " \"SP_v1.4.5_A_sc_7_150\",\n", + " # \"SP_v1.4.5_A_no_leakage\"\n", + "]\n", + "\n", + "root_to_plot = [\n", + " f\"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_{int(i)}.0_80.0_10.0_80.0\"\n", + " for i in [3, 5, 7, 10, 11]\n", + "]\n", + "\n", + "colours = [\n", + " \"red\",\n", + " \"salmon\",\n", + " \"darkorange\",\n", + " \"forestgreen\",\n", + " \"turquoise\",\n", + " \"darkviolet\",\n", + " \"crimson\",\n", + " \"gold\",\n", + " \"lightcoral\",\n", + " \"mediumseagreen\",\n", + " \"lightsteelblue\",\n", + " \"black\",\n", + " \"silver\",\n", + " \"peru\",\n", + " \"maroon\",\n", + " \"olive\",\n", + "]\n", + "\n", + "savefile = \"best_fit_tau_new_binning.png\"\n", + "\n", + "plot_best_fit_tau(root_to_plot, colours, savefile)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pseudo_cell = fits.open(\n", + " \"/home/guerrini/sp_validation/cosmo_val/output/pseudo_cl_SP_v1.4.5.fits\"\n", + ")[1].data\n", + "cov_pseudo_cell = fits.open(\n", + " \"/home/guerrini/sp_validation/cosmo_val/output/pseudo_cl_cov_SP_v1.4.5.fits\"\n", + ")\n", + "\n", + "theory_ell = np.loadtxt(\n", + " \"/n09data/guerrini/output_chains/best_fit/SP_v1.4.5_A/shear_cl/ell.txt\"\n", + ")\n", + "theory_cell = np.loadtxt(\n", + " \"/n09data/guerrini/output_chains/best_fit/SP_v1.4.5_A/shear_cl/bin_1_1.txt\"\n", + ")\n", + "\n", + "pw = hp.pixwin(1024, lmax=2048)\n", + "\n", + "plt.errorbar(\n", + " pseudo_cell[\"ELL\"],\n", + " pseudo_cell[\"ELL\"] * pseudo_cell[\"EE\"],\n", + " yerr=pseudo_cell[\"ELL\"] * np.sqrt(np.diag(cov_pseudo_cell[\"COVAR_EE_EE\"].data)),\n", + " capsize=2,\n", + " c=\"k\",\n", + " fmt=\"o\",\n", + " markersize=2,\n", + ")\n", + "\n", + "mask = (theory_ell > 0.1) & (theory_ell < 2048)\n", + "plt.plot(\n", + " theory_ell[mask],\n", + " theory_ell[mask]\n", + " * theory_cell[mask]\n", + " * np.interp(theory_ell[mask], np.arange(0, 2049), pw) ** 2,\n", + " c=\"r\",\n", + " label=\"best-fit $\\\\theta \\\\in [3-200]$\",\n", + ")\n", + "\n", + "plt.xlabel(r\"$\\ell$\", fontsize=26)\n", + "plt.ylabel(r\"$\\ell C_\\ell^{EE}$\", fontsize=26)\n", + "plt.legend()\n", + "plt.savefig(\"SP_v1.4.5_A_cell.png\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cov_pseudo_cell.info()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sp_validation", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/S8_om_sigma8_whisker.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/S8_om_sigma8_whisker.ipynb new file mode 100644 index 00000000..7a4fd5d2 --- /dev/null +++ b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/S8_om_sigma8_whisker.ipynb @@ -0,0 +1,645 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Whisker plot\n", + "\n", + "This notebook plots the whisker plot of $S_8$, $\\Omega_m$ and $\\sigma_8$" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "\n", + "# Trick to plot with tex\n", + "os.environ[\"LD_LIBRARY_PATH\"] = \"\"\n", + "os.environ[\"CONDA_PREFIX\"] = \"/home/guerrini/.conda/envs/sp_validation_3.11\"\n", + "\n", + "sys.path.append(\"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/notebooks/\")\n", + "\n", + "import sys\n", + "import warnings\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import seaborn as sns\n", + "from getdist import plots\n", + "\n", + "sys.path.append(\"/home/guerrini/sp_validation/cosmo_inference/scripts\")\n", + "\n", + "import chain_postprocessing as cp\n", + "\n", + "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", + "\n", + "plt.rc(\"text\", usetex=True)\n", + "\n", + "sns.set_palette(\"husl\")\n", + "\n", + "g = plots.get_subplot_plotter(width_inch=30)\n", + "g.settings.axes_fontsize = 60\n", + "g.settings.axes_labelsize = 60\n", + "g.settings.alpha_filled_add = 0.7\n", + "g.settings.legend_fontsize = 60\n", + "\n", + "%matplotlib inline\n", + "\n", + "# SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE\n", + "root_dir = \"/n09data/guerrini/output_chains/\"\n", + "root_external = f\"{root_dir}/ext_data/\"\n", + "blind = \"B\"\n", + "\n", + "roots = [\n", + " f\"SP_v1.4.6.3_{blind}_fiducial_config\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}\",\n", + " \"Planck18\",\n", + " \"DES Y6\",\n", + " \"KiDS-Legacy_bandpowers\",\n", + " \"KiDS-Legacy_cosebis\",\n", + " \"KiDS-Legacy_xipm\",\n", + " \"HSC_Y3\",\n", + " \"HSC_Y3_cell\",\n", + " f\"SP_v1.4.6.3_{blind}_small_scales_config\",\n", + " f\"SP_v1.4.6.3_{blind}_flat_alpha_beta_config\",\n", + " f\"SP_v1.4.6.3_{blind}_no_xi_sys_config\",\n", + " f\"SP_v1.4.6.3_{blind}_no_leak_corr_config\",\n", + " f\"SP_v1.4.6.3_{blind}_flat_delta_z_config\",\n", + " f\"SP_v1.4.6.3_{blind}_no_delta_z_config\",\n", + " f\"SP_v1.4.6.3_{blind}_flat_ia_config\",\n", + " f\"SP_v1.4.6.3_{blind}_no_ia_config\",\n", + " f\"SP_v1.4.6.3_{blind}_no_m_bias_config\",\n", + " f\"SP_v1.4.6.3_{blind}_unmasked_covmat_config\",\n", + " f\"SP_v1.4.6.3_{blind}_halofit_config\",\n", + " f\"SP_v1.4.6.3_{blind}_no_baryons_config\",\n", + " f\"SP_v1.4.6.3_{blind}_nautilus_config\",\n", + " f\"SP_v1.4.6.3_{blind}_planck_config\",\n", + " f\"SP_v1.4.6.3_{blind}_planck_desi_config\",\n", + "]\n", + "\n", + "legend_labels = [\n", + " r\"UNIONS-3500 $\\xi_{\\pm}(\\theta)$ (This work)\",\n", + " r\"UNIONS-3500 $C_\\ell$ (Guerrini et al. 2026)\",\n", + " r\"$\\textit{Planck}$ 2018\",\n", + " r\"DES Y6 $\\xi_{\\pm}$, NLA\",\n", + " r\"KiDS-Legacy Bandpowers ($C_{\\rm E}$)\",\n", + " r\"KiDS-Legacy COSEBIs ($E_n$)\",\n", + " r\"KiDS-Legacy $\\xi_{\\pm}(\\theta)$\",\n", + " r\"HSC-Y3 $\\xi_{\\pm}(\\theta)$\",\n", + " r\"HSC-Y3 $C_\\ell$\",\n", + " r\"$\\xi_+$ small scales, $\\theta$=[5,83] arcmin\",\n", + " r\"Flat $\\alpha_{\\rm{PSF}}$ and $\\beta_{\\rm{PSF}}$ priors\",\n", + " r\"No $\\xi^{\\rm sys}_{\\pm}$\",\n", + " r\"No leakage correction\",\n", + " r\"Flat $\\Delta z$ priors\",\n", + " r\"No $\\Delta z$\",\n", + " r\"Flat $A_{\\rm IA}$ prior\",\n", + " r\"No $A_{\\rm IA}$\",\n", + " r\"No $m$ bias\",\n", + " r\"Unmasked covmat\",\n", + " r\"$\\texttt{Halofit}$\",\n", + " r\"$\\texttt{HMCode}$ no baryons\",\n", + " r\"Nautilus sampler\",\n", + " r\"UNIONS-3500 + $\\textit{Planck}$\",\n", + " r\"UNIONS-3500 + $\\textit{Planck}$ + DESI BAO\",\n", + "]\n", + "\n", + "categories = [\n", + " \"configuration\",\n", + " \"harmonic\",\n", + " \"external\",\n", + " \"external\",\n", + " \"external\",\n", + " \"external\",\n", + " \"external\",\n", + " \"external\",\n", + " \"external\",\n", + " \"configuration\",\n", + " \"configuration\",\n", + " \"configuration\",\n", + " \"configuration\",\n", + " \"configuration\",\n", + " \"configuration\",\n", + " \"configuration\",\n", + " \"configuration\",\n", + " \"configuration\",\n", + " \"configuration\",\n", + " \"configuration\",\n", + " \"configuration\",\n", + " \"configuration\",\n", + " \"configuration\",\n", + " \"configuration\",\n", + "]\n", + "colours = [\n", + " \"darkorange\",\n", + " \"royalblue\",\n", + " \"violet\",\n", + " \"black\",\n", + " \"black\",\n", + " \"black\",\n", + " \"black\",\n", + " \"black\",\n", + " \"black\",\n", + " \"forestgreen\",\n", + " \"forestgreen\",\n", + " \"forestgreen\",\n", + " \"forestgreen\",\n", + " \"forestgreen\",\n", + " \"forestgreen\",\n", + " \"forestgreen\",\n", + " \"forestgreen\",\n", + " \"forestgreen\",\n", + " \"forestgreen\",\n", + " \"forestgreen\",\n", + " \"forestgreen\",\n", + " \"forestgreen\",\n", + " \"forestgreen\",\n", + " \"forestgreen\",\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "chains = []\n", + "for i, root in enumerate(roots):\n", + " category = categories[i]\n", + " if root == \"DES Y6\":\n", + " continue\n", + " if category != \"external\":\n", + " if category == \"configuration\":\n", + " path_samples = os.path.join(root_dir, f\"{root}/samples_{root}.txt\")\n", + " path_getdist = os.path.join(root_dir, f\"{root}/getdist_{root}\")\n", + " elif category == \"harmonic\":\n", + " path_samples = os.path.join(\n", + " root_dir, f\"{root}/{root}/samples_{root}_cell.txt\"\n", + " )\n", + " path_getdist = os.path.join(root_dir, f\"{root}/{root}/getdist_{root}\")\n", + " elif category == \"external_compute_sample\":\n", + " path_samples = os.path.join(root_dir, f\"ext_data/{root}/samples_{root}.txt\")\n", + " path_getdist = os.path.join(root_dir, f\"ext_data/{root}/getdist_{root}\")\n", + " else:\n", + " raise ValueError(f\"The category, {category}, of {root} is not correct\")\n", + " if \"nautilus\" not in root:\n", + " cp.load_samples_and_write_paramnames(\n", + " path_samples, path_getdist + \".paramnames\"\n", + " )\n", + " cp.write_samples_getdist_format(path_samples, path_getdist + \".txt\")\n", + " else:\n", + " cp.load_samples_and_write_paramnames(\n", + " path_samples, path_getdist + \".paramnames\", chain_type=\"nautilus\"\n", + " )\n", + " cp.write_samples_getdist_format(\n", + " path_samples, path_getdist + \".txt\", chain_type=\"nautilus\"\n", + " )\n", + " chains.append(cp.load_chain(path_getdist, smoothing_scale=0.5))\n", + " else:\n", + " path_getdist = os.path.join(root_dir, f\"ext_data/{root}/getdist_{root}\")\n", + " chains.append(cp.load_chain(path_getdist))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "name_list = [\n", + " \"OMEGA_M\",\n", + " \"ombh2\",\n", + " \"h0\",\n", + " \"n_s\",\n", + " \"SIGMA_8\",\n", + " \"S_8\",\n", + " \"s_8_input\",\n", + " \"logt_agn\",\n", + " \"a\",\n", + " \"m1\",\n", + " \"bias_1\",\n", + "]\n", + "label_list = [\n", + " r\"\\Omega_{\\rm m}\",\n", + " r\"\\omega_b h^2\",\n", + " r\"h_0\",\n", + " r\"n_s\",\n", + " r\"\\sigma_8\",\n", + " r\"S_8\",\n", + " r\"S_8\",\n", + " r\"\\log T_{\\rm AGN}\",\n", + " r\"A_{\\rm IA}\",\n", + " r\"m_1\",\n", + " r\"\\Delta z_1\",\n", + "]\n", + "\n", + "for i, chain in enumerate(chains):\n", + " print(legend_labels[i])\n", + " param_names = chain.getParamNames()\n", + " for name, label in zip(name_list, label_list):\n", + " try:\n", + " param_names.parWithName(name).label = label\n", + " except Exception:\n", + " warnings.warn(f\"Parameter {name} not found in chain {roots[i]}.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Micro management of external chains\n", + "\n", + "# Account for the missing parameter conventions\n", + "\n", + "idx = roots.index(\"KiDS-Legacy_xipm\")\n", + "cp.derive_parameter_S8(chains[idx])\n", + "\n", + "idx = roots.index(\"KiDS-Legacy_bandpowers\")\n", + "cp.derive_parameter_S8(chains[idx])\n", + "\n", + "idx = roots.index(\"KiDS-Legacy_cosebis\")\n", + "cp.derive_parameter_S8(chains[idx])\n", + "\n", + "# OMEGA_M not in HSC_Y3_cell\n", + "idx = roots.index(\"HSC_Y3_cell\")\n", + "cp.adjust_paramname_chain(chains[idx], \"omega_m\", \"OMEGA_M\", r\"\\Omega_{\\rm m}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "param_values = np.array(\n", + " [\n", + " \"# Expt\",\n", + " \"Colour\",\n", + " \"S8_Mean\",\n", + " \"S8_low\",\n", + " \"S8_high\",\n", + " \"sigma_8_Mean\",\n", + " \"sigma_8_low\",\n", + " \"sigma_8_high\",\n", + " \"Omega_m_Mean\",\n", + " \"Omega_m_low\",\n", + " \"Omega_m_high\",\n", + " ]\n", + ")\n", + "escaped = np.char.replace(legend_labels, \"\\\\\", \"\\\\\\\\\")\n", + "\n", + "for i, root in enumerate(roots):\n", + " chain = chains[i]\n", + " if root == \"DES Y6\":\n", + " param_values = np.vstack(\n", + " (\n", + " param_values,\n", + " [\n", + " escaped[i],\n", + " colours[i],\n", + " 0.798,\n", + " 0.015,\n", + " 0.014,\n", + " 0.763,\n", + " 0.057,\n", + " 0.050,\n", + " 0.332,\n", + " 0.040,\n", + " 0.035,\n", + " ],\n", + " )\n", + " )\n", + " else:\n", + " best_fit_params = cp.extract_best_fit_params(chain, best_fit_method=\"2Dkde\")\n", + " margestats = chain.getMargeStats()\n", + "\n", + " s8_stats = margestats.parWithName(\"S_8\")\n", + " sigma8_stats = margestats.parWithName(\"SIGMA_8\")\n", + " omegam_stats = margestats.parWithName(\"OMEGA_M\")\n", + "\n", + " param_values = np.vstack(\n", + " (\n", + " param_values,\n", + " [\n", + " escaped[i],\n", + " colours[i],\n", + " best_fit_params[\"S_8\"],\n", + " best_fit_params[\"S_8\"] - s8_stats.limits[0].lower,\n", + " s8_stats.limits[0].upper - best_fit_params[\"S_8\"],\n", + " best_fit_params[\"SIGMA_8\"],\n", + " best_fit_params[\"SIGMA_8\"] - sigma8_stats.limits[0].lower,\n", + " sigma8_stats.limits[0].upper - best_fit_params[\"SIGMA_8\"],\n", + " best_fit_params[\"OMEGA_M\"],\n", + " best_fit_params[\"OMEGA_M\"] - omegam_stats.limits[0].lower,\n", + " omegam_stats.limits[0].upper - best_fit_params[\"OMEGA_M\"],\n", + " ],\n", + " )\n", + " )\n", + "print(param_values)\n", + "np.savetxt(\n", + " f\"{root_dir}/param_values.txt\",\n", + " param_values,\n", + " fmt=[\"%s\" for i in range(11)],\n", + " delimiter=\";\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# Load the value of the parameters\n", + "cosmo = np.loadtxt(\n", + " f\"{root_dir}/param_values.txt\",\n", + " dtype={\n", + " \"names\": (\n", + " \"Expt\",\n", + " \"colour\",\n", + " \"s8_mean\",\n", + " \"s8_low\",\n", + " \"s8_high\",\n", + " \"sigma8_mean\",\n", + " \"sigma8_low\",\n", + " \"sigma8_high\",\n", + " \"omegam_mean\",\n", + " \"omegam_low\",\n", + " \"omegam_high\",\n", + " ),\n", + " \"formats\": (\n", + " \"U250\",\n", + " \"U20\",\n", + " \"U20\",\n", + " \"U20\",\n", + " \"U20\",\n", + " \"U20\",\n", + " \"U20\",\n", + " \"U20\",\n", + " \"U20\",\n", + " \"U20\",\n", + " \"U20\",\n", + " ),\n", + " },\n", + " skiprows=1,\n", + " delimiter=\";\",\n", + ")\n", + "expt = np.char.replace(cosmo[\"Expt\"], \"\\\\\\\\\", \"\\\\\")\n", + "colours = cosmo[\"colour\"]\n", + "s8_mean = cosmo[\"s8_mean\"].astype(np.float64)\n", + "s8_low = cosmo[\"s8_low\"].astype(np.float64)\n", + "s8_high = cosmo[\"s8_high\"].astype(np.float64)\n", + "sigma8_mean = cosmo[\"sigma8_mean\"].astype(np.float64)\n", + "sigma8_low = cosmo[\"sigma8_low\"].astype(np.float64)\n", + "sigma8_high = cosmo[\"sigma8_high\"].astype(np.float64)\n", + "omegam_mean = cosmo[\"omegam_mean\"].astype(np.float64)\n", + "omegam_low = cosmo[\"omegam_low\"].astype(np.float64)\n", + "omegam_high = cosmo[\"omegam_high\"].astype(np.float64)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "from matplotlib.gridspec import GridSpec\n", + "\n", + "fig = plt.figure(figsize=(13, 8))\n", + "gs = GridSpec(1, 3, width_ratios=[1, 0.5, 0.5])\n", + "ax1 = fig.add_subplot(gs[0])\n", + "ax2 = fig.add_subplot(gs[1], sharey=ax1)\n", + "ax3 = fig.add_subplot(gs[2], sharey=ax1)\n", + "\n", + "axs = [ax1, ax2, ax3]\n", + "\n", + "params = [\n", + " (s8_mean, s8_low, s8_high, r\"$S_8$\"),\n", + " (sigma8_mean, sigma8_low, sigma8_high, r\"$\\sigma_8$\"),\n", + " (omegam_mean, omegam_low, omegam_high, r\"$\\Omega_{\\rm m}$\"),\n", + "]\n", + "reference = r\"UNIONS-3500 $\\xi_{\\pm}(\\theta)$ (This work)\"\n", + "\n", + "separation_after = [\n", + " r\"UNIONS-3500 $C_\\ell$ (Guerrini et al. 2026)\",\n", + " r\"HSC-Y3 $C_\\ell$\",\n", + " r\"$\\xi_+$ small scales, $\\theta$=[5,83] arcmin\",\n", + " r\"Unmasked covmat\",\n", + " r\"$\\texttt{HMCode}$ no baryons\",\n", + " r\"Nautilus sampler\",\n", + "]\n", + "list_section_index = [r\"(ii)\", r\"(iii)\", r\"(iv)\", r\"(v)\", r\"(vi)\", r\"(vii)\"]\n", + "\n", + "preliminary_watermark = False\n", + "blind_axes = False\n", + "row_spacing = 0.2\n", + "\n", + "index_ref = np.where(expt == reference)[0][0]\n", + "\n", + "y = np.arange(len(expt))\n", + "for ax, param in zip(axs, params):\n", + " means, lows, highs, label = param\n", + " for i, mean, low, high, color in zip(y, means, lows, highs, colours):\n", + " ax.errorbar(\n", + " mean,\n", + " 0.05 + i * row_spacing,\n", + " xerr=np.array([low, high])[:, None],\n", + " fmt=\"o\",\n", + " color=color,\n", + " ecolor=color,\n", + " elinewidth=2,\n", + " capsize=3,\n", + " )\n", + " ax.set_xlabel(label, fontsize=14)\n", + "\n", + " ax.grid(False)\n", + " ax.tick_params(axis=\"y\", left=False, labelleft=False)\n", + " if label == r\"$S_8$\":\n", + " ax.axvspan(\n", + " s8_mean[index_ref] - s8_low[index_ref],\n", + " s8_mean[index_ref] + s8_high[index_ref],\n", + " color=colours[index_ref],\n", + " alpha=0.2,\n", + " )\n", + " ax.set_xlim(0.6, 1.35)\n", + " if blind_axes:\n", + " ref_tick = np.mean(s8_mean[:4])\n", + " ax.set_xticks([ref_tick + i * 0.1 for i in range(-5, 5)], labels=[])\n", + " elif label == r\"$\\sigma_8$\":\n", + " ax.axvspan(\n", + " sigma8_mean[index_ref] - sigma8_low[index_ref],\n", + " sigma8_mean[index_ref] + sigma8_high[index_ref],\n", + " color=colours[index_ref],\n", + " alpha=0.2,\n", + " )\n", + " ax.set_xlim(0.5, 1.35)\n", + " if blind_axes:\n", + " ref_tick = np.mean(sigma8_mean[:4])\n", + " ax.set_xticks([ref_tick + i * 0.2 for i in range(-2, 2)], labels=[])\n", + " elif label == r\"$\\Omega_{\\rm m}$\":\n", + " ax.axvspan(\n", + " omegam_mean[index_ref] - omegam_low[index_ref],\n", + " omegam_mean[index_ref] + omegam_high[index_ref],\n", + " color=colours[index_ref],\n", + " alpha=0.2,\n", + " )\n", + " ax.set_xlim(0.1, 0.5)\n", + " if blind_axes:\n", + " ref_tick = np.mean(omegam_mean[:4])\n", + " ax.set_xticks([ref_tick + i * 0.1 for i in range(-2, 3)], labels=[])\n", + "\n", + "\n", + "ax1.set_yticks(0.01 + y * row_spacing)\n", + "ax1.set_yticklabels([])\n", + "for label, color in zip(expt, colours):\n", + " if \"This work\" in label:\n", + " label_bold = (\n", + " r\"$\\bf{UNIONS}$-$\\bf{3500}$ $\\xi_{\\pm}(\\theta)$ $\\bf{(This\\ work)}$\"\n", + " )\n", + " ax1.text(\n", + " -0.6,\n", + " 0.05 + row_spacing * np.where(expt == label)[0][0],\n", + " label_bold,\n", + " fontsize=12,\n", + " ha=\"left\",\n", + " va=\"center\",\n", + " color=color,\n", + " )\n", + " else:\n", + " ax1.text(\n", + " -0.6,\n", + " 0.05 + row_spacing * np.where(expt == label)[0][0],\n", + " label,\n", + " fontsize=12,\n", + " ha=\"left\",\n", + " va=\"center\",\n", + " color=color,\n", + " )\n", + " if label != reference:\n", + " index = np.where(expt == label)[0][0]\n", + " s8_tension = cp.get_sigma_tension(\n", + " s8_mean[index],\n", + " s8_low[index],\n", + " s8_high[index],\n", + " s8_mean[index_ref],\n", + " s8_low[index_ref],\n", + " s8_high[index_ref],\n", + " )\n", + " sign_str = \"+\" if s8_tension > 0 else \"-\"\n", + " ax1.text(\n", + " 1.32,\n", + " 0.05 + row_spacing * index,\n", + " rf\"${sign_str}{np.abs(s8_tension):.2f}\" + r\"\\, \\sigma$\",\n", + " fontsize=10,\n", + " ha=\"right\",\n", + " va=\"center\",\n", + " color=color,\n", + " )\n", + "# Add separation lines\n", + "for i, sep in enumerate(separation_after):\n", + " print(sep)\n", + " index_sep = np.where(expt == sep)[0][0]\n", + " ax2.axhline(\n", + " row_spacing * (index_sep + 1) - 0.07,\n", + " color=\"black\",\n", + " linestyle=\"dotted\",\n", + " linewidth=1,\n", + " )\n", + " ax3.axhline(\n", + " row_spacing * (index_sep + 1) - 0.07,\n", + " color=\"black\",\n", + " linestyle=\"dotted\",\n", + " linewidth=1,\n", + " )\n", + " ax1.axhline(\n", + " row_spacing * (index_sep + 1) - 0.07,\n", + " xmin=-1.8,\n", + " color=\"black\",\n", + " linestyle=\"dotted\",\n", + " linewidth=1,\n", + " clip_on=False,\n", + " )\n", + " ax1.text(\n", + " -0.61,\n", + " row_spacing * (index_sep + 1) + 0.05,\n", + " list_section_index[i],\n", + " fontsize=12,\n", + " fontweight=\"bold\",\n", + " va=\"center\",\n", + " ha=\"right\",\n", + " )\n", + "\n", + "\n", + "# --- Add section label (i)) ---\n", + "ax1.text(-0.61, 0.05, r\"(i)\", fontsize=12, fontweight=\"bold\", va=\"center\", ha=\"right\")\n", + "\n", + "if preliminary_watermark:\n", + " plt.figtext(\n", + " 0.5,\n", + " 0.5,\n", + " \"PRELIMINARY\",\n", + " fontsize=50,\n", + " color=\"gray\",\n", + " ha=\"center\",\n", + " va=\"center\",\n", + " alpha=0.3,\n", + " rotation=330,\n", + " )\n", + "\n", + "plt.gca().invert_yaxis()\n", + "\n", + "plt.tight_layout()\n", + "\n", + "# plt.savefig(\"./plots/whisker_plot.png\", dpi=300)\n", + "# #Save pdf\n", + "plt.savefig(\"../Plots/S8_whisker_plot.pdf\", bbox_inches=\"tight\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "my_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/best_fit_xipm.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/best_fit_xipm.ipynb new file mode 100644 index 00000000..d6ed4c01 --- /dev/null +++ b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/best_fit_xipm.ipynb @@ -0,0 +1,607 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Best-fit $\\xi_\\pm$\n", + "\n", + "This notebook plots the best-fit 2PCFs for the fiducial and other cases" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "\n", + "sys.path.append(\"/home/guerrini/sp_validation/cosmo_inference/scripts\")\n", + "\n", + "import chain_postprocessing as cp\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.scale as mscale\n", + "import numpy as np\n", + "import seaborn as sns\n", + "from astropy.io import fits\n", + "from getdist import plots\n", + "\n", + "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", + "\n", + "from sp_validation.rho_tau import SquareRootScale\n", + "\n", + "mscale.register_scale(SquareRootScale)\n", + "\n", + "plt.rcParams[\"text.usetex\"] = True\n", + "\n", + "sns.set_palette(\"husl\")\n", + "\n", + "g = plots.get_subplot_plotter(width_inch=30)\n", + "g.settings.axes_fontsize = 40\n", + "g.settings.axes_labelsize = 40\n", + "g.settings.alpha_filled_add = 0.7\n", + "g.settings.legend_fontsize = 50\n", + "\n", + "# Directory where the chains are located\n", + "root_dir = \"/n09data/guerrini/output_chains\"\n", + "\n", + "# THE BLIND TO USE FOR THE PLOTS\n", + "blind = \"B\"\n", + "catalog_version = \"SP_v1.4.6.3\"\n", + "fiducial_root_cell = f\"SP_v1.4.6.3_leak_corr_{blind}\"\n", + "label_fiducial_cell = r\"UNIONS $C_{\\ell}$\"\n", + "fiducial_root_xi_data = f\"SP_v1.4.6.3_leak_corr_{blind}_masked\"\n", + "fiducial_root_xi_chains = f\"SP_v1.4.6.3_{blind}_fiducial_config\"\n", + "label_fiducial_xi = r\"UNIONS $\\xi_{\\pm}$\"\n", + "\n", + "# Path to the ini files used\n", + "path_ini_files = \"/home/guerrini/sp_validation/cosmo_inference/cosmosis_config\"\n", + "path_datavectors = \"/home/guerrini/sp_validation/cosmo_inference/data/\"\n", + "path_output_chains = \"/n09data/guerrini/output_chains/\"\n", + "\n", + "\n", + "data_cell = fits.open(\n", + " os.path.join(\n", + " path_datavectors, f\"{fiducial_root_cell}/cosmosis_{fiducial_root_cell}.fits\"\n", + " )\n", + ")\n", + "\n", + "data_xi = fits.open(\n", + " os.path.join(\n", + " path_datavectors,\n", + " f\"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}/cosmosis_{fiducial_root_xi_data}.fits\",\n", + " )\n", + ")\n", + "\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "# Perform the computation for the fiducial of Cell\n", + "path_samples_fiducial_cell = os.path.join(\n", + " path_output_chains,\n", + " fiducial_root_cell,\n", + " fiducial_root_cell,\n", + " f\"samples_{fiducial_root_cell}_cell.txt\",\n", + ")\n", + "path_gd_fiducial_cell = os.path.join(\n", + " path_output_chains,\n", + " fiducial_root_cell,\n", + " fiducial_root_cell,\n", + " f\"getdist_{fiducial_root_cell}_cell\",\n", + ")\n", + "cp.load_samples_and_write_paramnames(\n", + " path_samples_fiducial_cell, path_gd_fiducial_cell + \".paramnames\"\n", + ")\n", + "cp.write_samples_getdist_format(\n", + " path_samples_fiducial_cell, path_gd_fiducial_cell + \".txt\", chain_type=\"polychord\"\n", + ")\n", + "\n", + "chain_fiducial_cell = cp.load_chain(path_gd_fiducial_cell, smoothing_scale=0.5)\n", + "\n", + "best_fit_params_fiducial_cell = cp.extract_best_fit_params(\n", + " chain_fiducial_cell, best_fit_method=\"2Dkde\"\n", + ")\n", + "\n", + "cp.compute_best_fit(\n", + " path_ini_files,\n", + " best_fit_params_fiducial_cell,\n", + " fiducial_root_cell,\n", + " is_harmonic=True,\n", + " blind=blind,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "# Perform the computation for the fiducial of xi\n", + "path_samples_fiducial_xi = os.path.join(\n", + " path_output_chains,\n", + " fiducial_root_xi_chains,\n", + " f\"samples_{fiducial_root_xi_chains}.txt\",\n", + ")\n", + "\n", + "path_gd_fiducial_xi = os.path.join(\n", + " path_output_chains, fiducial_root_xi_chains, f\"getdist_{fiducial_root_xi_chains}\"\n", + ")\n", + "cp.load_samples_and_write_paramnames(\n", + " path_samples_fiducial_xi, path_gd_fiducial_xi + \".paramnames\"\n", + ")\n", + "cp.write_samples_getdist_format(\n", + " path_samples_fiducial_xi, path_gd_fiducial_xi + \".txt\", chain_type=\"polychord\"\n", + ")\n", + "\n", + "chain_fiducial_xi = cp.load_chain(path_gd_fiducial_xi, smoothing_scale=0.5)\n", + "\n", + "best_fit_params_fiducial_xi = cp.extract_best_fit_params(\n", + " chain_fiducial_xi, best_fit_method=\"2Dkde\"\n", + ")\n", + "\n", + "ini_file_root = os.path.join(\n", + " path_ini_files,\n", + " f\"config_space_v1.4.6.3_fiducial/pipeline/blind_{blind}/fiducial.ini\",\n", + ")\n", + "cp.compute_best_fit(\n", + " path_ini_files,\n", + " best_fit_params_fiducial_xi,\n", + " fiducial_root_xi_chains,\n", + " is_harmonic=False,\n", + " blind=blind,\n", + " ini_file_root=ini_file_root,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Make the plot for the best-fit datavector for Cell EE\n", + "root_to_plot = [\n", + " fiducial_root_xi_chains,\n", + " fiducial_root_cell,\n", + "]\n", + "\n", + "labels = [\n", + " r\"UNIONS $\\xi_\\pm(\\theta)$\",\n", + " r\"UNIONS $C_\\ell$\",\n", + "]\n", + "\n", + "line_args = [\n", + " {\"color\": \"royalblue\", \"linestyle\": \"-\"},\n", + " {\"color\": \"orange\", \"linestyle\": \"-\"},\n", + "]\n", + "\n", + "properties = {}\n", + "\n", + "properties = cp.update_properties_w_roots(\n", + " properties, fiducial_root_cell, path_ini_files, with_configuration=False\n", + ")\n", + "properties = cp.update_properties_w_roots(\n", + " properties,\n", + " fiducial_root_xi_chains,\n", + " path_ini_files,\n", + " with_configuration=True,\n", + " path_to_this_ini=ini_file_root,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "root_to_plot = [fiducial_root_cell, fiducial_root_xi_chains]\n", + "labels = [r\"Best fit $C_\\ell$\", r\"Best fit $\\xi_\\pm(\\theta)$\"]\n", + "path_best_fit_xi_theta = os.path.join(\n", + " path_output_chains, fiducial_root_xi_chains, \"best_fit/shear_xi_plus/theta.txt\"\n", + ")\n", + "\n", + "theta_rad = np.loadtxt(path_best_fit_xi_theta)\n", + "theta_min = 1\n", + "theta_max = 250\n", + "\n", + "cp.compute_best_fit_xi_from_cell(\n", + " path_output_chains, fiducial_root_cell, best_fit_params_fiducial_cell, theta_rad\n", + ")\n", + "\n", + "data = fits.open(\n", + " os.path.join(\n", + " path_datavectors,\n", + " f\"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}/cosmosis_{fiducial_root_xi_data}.fits\",\n", + " )\n", + ")\n", + "bbox_to_anchor_xip = (0.685, 0.09)\n", + "bbox_to_anchor_xim = (0.3, 0.65)\n", + "xi_p_data = data[\"XI_PLUS\"].data\n", + "xi_m_data = data[\"XI_MINUS\"].data\n", + "cov_mat = data[\"COVMAT\"].data\n", + "\n", + "# Plot hyperparameter\n", + "loc_legend = \"lower center\"\n", + "\n", + "fig, [ax, ax2] = plt.subplots(1, 2, figsize=(20, 8))\n", + "\n", + "theta, xi_p, xi_m = xi_p_data[\"ANG\"], xi_p_data[\"VALUE\"], xi_m_data[\"VALUE\"]\n", + "ax.errorbar(\n", + " theta,\n", + " theta * xi_p,\n", + " yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])),\n", + " fmt=\"o\",\n", + " label=r\"UNIONS $\\xi_+$ data\",\n", + " color=\"black\",\n", + " capsize=2,\n", + ")\n", + "ax2.errorbar(\n", + " theta,\n", + " theta * xi_m,\n", + " yerr=theta\n", + " * np.sqrt(\n", + " np.diag(cov_mat[len(theta) : 2 * len(theta), len(theta) : 2 * len(theta)])\n", + " ),\n", + " fmt=\"o\",\n", + " label=r\"UNIONS $\\xi_-$ data\",\n", + " color=\"black\",\n", + " capsize=2,\n", + ")\n", + "\n", + "for idx, (label, root) in enumerate(zip(labels, root_to_plot)):\n", + " # Read the results\n", + " theta = (\n", + " (\n", + " np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " )\n", + " * 180\n", + " / np.pi\n", + " * 60\n", + " )\n", + " xi_plus = np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_minus = np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " if r\"$C_\\ell$\" not in label:\n", + " xi_sys_plus = np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + " theta_xi_sys = (\n", + " np.loadtxt(path_output_chains + \"{}/best_fit/xi_sys/theta.txt\".format(root))\n", + " * 180\n", + " / np.pi\n", + " * 60\n", + " )\n", + "\n", + " xi_sys_plus = np.interp(theta, theta_xi_sys, xi_sys_plus)\n", + " xi_sys_minus = np.interp(theta, theta_xi_sys, xi_sys_minus)\n", + " xi_plus += xi_sys_plus\n", + " xi_minus += xi_sys_minus\n", + "\n", + " mask = (theta > theta_min) & (theta < theta_max)\n", + " theta = theta[mask]\n", + " ax.plot(\n", + " theta,\n", + " theta * xi_plus[mask],\n", + " label=r\"Best fit $\\xi_+(\\theta)$\",\n", + " **line_args[idx],\n", + " lw=2.5,\n", + " )\n", + " ax.plot(\n", + " theta,\n", + " theta * xi_sys_plus[mask],\n", + " label=r\"Best fit $\\xi^{\\rm sys}_{+}(\\theta)$\",\n", + " c=\"r\",\n", + " )\n", + " ax2.plot(\n", + " theta,\n", + " theta * xi_minus[mask],\n", + " label=r\"Best fit $\\xi_-(\\theta)$\",\n", + " **line_args[idx],\n", + " lw=2.5,\n", + " )\n", + " ax2.plot(\n", + " theta,\n", + " theta * xi_sys_minus[mask],\n", + " label=r\"Best fit $\\xi^{\\rm sys}_{-}(\\theta)$\",\n", + " c=\"r\",\n", + " )\n", + "\n", + " else:\n", + " mask = (theta > theta_min) & (theta < theta_max)\n", + " theta = theta[mask]\n", + " ax.plot(theta, theta * xi_plus[mask], label=label, **line_args[idx], lw=2.5)\n", + " ax2.plot(theta, theta * xi_minus[mask], label=label, **line_args[idx], lw=2.5)\n", + "# XI PLUS PLOT SETTINGS\n", + "\n", + "# Plot the scale cuts for different k_max\n", + "ax.axvline(x=5, color=\"gray\", linestyle=\"--\", alpha=0.7)\n", + "ax.axhline(y=0, color=\"black\", linestyle=\"--\", alpha=0.7)\n", + "\n", + "ymin = ax.get_ylim()[0]\n", + "ymax = ax.get_ylim()[1]\n", + "# Shadowing cut scaled\n", + "ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color=\"gray\", alpha=0.2)\n", + "ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color=\"gray\", alpha=0.2)\n", + "\n", + "ax.set_ylim(ymin, ymax)\n", + "\n", + "# Add labels directly under the tick\n", + "ax.text(\n", + " 4.5,\n", + " 0.47e-4,\n", + " r\"$k_\\mathrm{max} = 1 h$ Mpc$^{-1}$\",\n", + " ha=\"center\",\n", + " va=\"top\",\n", + " fontsize=20,\n", + " rotation=90,\n", + ")\n", + "\n", + "ax.set_ylabel(r\"$\\theta \\xi_\\pm$\", fontsize=26)\n", + "ax.set_xlabel(r\"$\\theta$ (arcmin)\", fontsize=26)\n", + "ax.set_xlim([theta.min() - 0.1, theta.max() + 20])\n", + "ax.set_title(r\"$\\xi_+(\\theta)$\", fontsize=26)\n", + "ax.set_xscale(\"log\")\n", + "ax.set_xticks(np.array([1, 10, 100]))\n", + "ax.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", + "ax.tick_params(axis=\"both\", which=\"major\", labelsize=24)\n", + "ax.tick_params(axis=\"both\", which=\"minor\", labelsize=20)\n", + "ax.yaxis.get_offset_text().set_fontsize(24)\n", + "ax.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", + "ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=20)\n", + "\n", + "# XI_MINUS PLOT SETTINGS\n", + "\n", + "# Plot the scale cuts for different k_max\n", + "ax2.axvline(x=50, color=\"gray\", linestyle=\"--\", alpha=0.7)\n", + "ax2.axhline(y=0, color=\"black\", linestyle=\"--\", alpha=0.7)\n", + "\n", + "ymin = ax2.get_ylim()[0]\n", + "ymax = ax2.get_ylim()[1]\n", + "# Shadowing cut scaled\n", + "ax2.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color=\"gray\", alpha=0.2)\n", + "ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color=\"gray\", alpha=0.2)\n", + "\n", + "ax2.set_ylim(ymin, ymax)\n", + "\n", + "# Add labels directly under the tick\n", + "ax2.text(\n", + " 45,\n", + " 1.15e-4,\n", + " r\"$k_\\mathrm{max} = 1 h$ Mpc$^{-1}$\",\n", + " ha=\"center\",\n", + " va=\"top\",\n", + " fontsize=20,\n", + " rotation=90,\n", + ")\n", + "\n", + "# ax2.set_ylabel(r'$\\theta \\xi_-$', fontsize=16)\n", + "ax2.set_xlabel(r\"$\\theta$ (arcmin)\", fontsize=26)\n", + "ax2.set_xlim([theta.min() - 0.1, theta.max() + 20])\n", + "ax2.set_xscale(\"log\")\n", + "ax2.set_title(r\"$\\xi_-(\\theta)$\", fontsize=26)\n", + "ax2.set_xticks(np.array([1, 10, 100]))\n", + "ax2.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", + "ax2.tick_params(axis=\"both\", which=\"major\", labelsize=24)\n", + "ax2.tick_params(axis=\"both\", which=\"minor\", labelsize=20)\n", + "ax2.yaxis.get_offset_text().set_fontsize(24)\n", + "ax2.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", + "ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=20)\n", + "\n", + "plt.savefig(\n", + " \"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/best_fit_xipm_SP_v1.4.6.3_B.pdf\",\n", + " bbox_inches=\"tight\",\n", + ")\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "root_to_plot = [fiducial_root_xi_chains]\n", + "labels = [r\"Best fit $\\tau_{0,2}(\\theta)$\"]\n", + "\n", + "bbox_to_anchor_xip = (0.285, 0.7)\n", + "bbox_to_anchor_xim = (0.3, 0.65)\n", + "tau0_data = data[\"TAU_0_PLUS\"].data\n", + "tau2_data = data[\"TAU_2_PLUS\"].data\n", + "cov_mat = data[\"COVMAT\"].data\n", + "\n", + "# Plot hyperparameter\n", + "\n", + "fig, [ax, ax2] = plt.subplots(1, 2, figsize=(20, 8))\n", + "\n", + "theta, tau0, tau2 = tau0_data[\"ANG\"], tau0_data[\"VALUE\"], tau2_data[\"VALUE\"]\n", + "ax.errorbar(\n", + " theta,\n", + " theta * tau0,\n", + " yerr=theta\n", + " * np.sqrt(\n", + " np.diag(\n", + " cov_mat[2 * len(theta) : 3 * len(theta), 2 * len(theta) : 3 * len(theta)]\n", + " )\n", + " ),\n", + " fmt=\"o\",\n", + " label=r\"UNIONS $\\tau_{0,+}$\",\n", + " color=\"black\",\n", + " capsize=2,\n", + ")\n", + "ax2.errorbar(\n", + " theta,\n", + " theta * tau2,\n", + " yerr=theta\n", + " * np.sqrt(\n", + " np.diag(\n", + " cov_mat[3 * len(theta) : 4 * len(theta), 3 * len(theta) : 4 * len(theta)]\n", + " )\n", + " ),\n", + " fmt=\"o\",\n", + " label=r\"UNIONS $\\tau_{2,+}$\",\n", + " color=\"black\",\n", + " capsize=2,\n", + ")\n", + "\n", + "for idx, (label, root) in enumerate(zip(labels, root_to_plot)):\n", + " # Read the results\n", + " theta = (\n", + " (\n", + " np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/tau_0_plus/theta.txt\".format(root)\n", + " )\n", + " )\n", + " * 180\n", + " / np.pi\n", + " * 60\n", + " )\n", + " tau0_plus = np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/tau_0_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " tau2_plus = np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/tau_2_plus/bin_1_1.txt\".format(root)\n", + " )\n", + "\n", + " mask = (theta > theta_min) & (theta < theta_max)\n", + " theta = theta[mask]\n", + " ax.plot(\n", + " theta,\n", + " theta * tau0_plus[mask],\n", + " label=r\"Best fit $\\tau_{0,+}(\\theta)$\",\n", + " c=\"orange\",\n", + " lw=2.5,\n", + " )\n", + " ax2.plot(\n", + " theta,\n", + " theta * tau2_plus[mask],\n", + " label=r\"Best fit $\\tau_{2,+}(\\theta)$\",\n", + " c=\"orange\",\n", + " lw=2.5,\n", + " )\n", + "\n", + "# XI PLUS PLOT SETTINGS\n", + "\n", + "# Plot the scale cuts for different k_max\n", + "ax.axhline(y=0, color=\"black\", linestyle=\"--\", alpha=0.7)\n", + "\n", + "ymin = ax.get_ylim()[0]\n", + "ymax = ax.get_ylim()[1]\n", + "\n", + "ax.set_ylim(ymin, ymax)\n", + "\n", + "ax.set_ylabel(r\"$\\theta\\tau_{0,2}$\", fontsize=26)\n", + "ax.set_xlabel(r\"$\\theta$ (arcmin)\", fontsize=26)\n", + "ax.set_xlim([theta.min() - 0.1, theta.max() + 20])\n", + "ax.set_title(r\"$\\tau_{0,+}(\\theta)$\", fontsize=26)\n", + "ax.set_xscale(\"log\")\n", + "ax.set_xticks(np.array([1, 10, 100]))\n", + "ax.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", + "ax.tick_params(axis=\"both\", which=\"major\", labelsize=24)\n", + "ax.tick_params(axis=\"both\", which=\"minor\", labelsize=20)\n", + "ax.yaxis.get_offset_text().set_fontsize(24)\n", + "ax.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", + "ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=20)\n", + "\n", + "# XI_MINUS PLOT SETTINGS\n", + "\n", + "# Plot the scale cuts for different k_max\n", + "ax2.axhline(y=0, color=\"black\", linestyle=\"--\", alpha=0.7)\n", + "\n", + "ymin = ax2.get_ylim()[0]\n", + "ymax = ax2.get_ylim()[1]\n", + "# Shadowing cut scaled\n", + "ax2.fill_betweenx(\n", + " y=[ymin, ymax],\n", + " x1=0,\n", + " x2=12,\n", + " color=\"gray\",\n", + " alpha=0.2,\n", + " label=r\"$B$-mode informed scale cut\",\n", + ")\n", + "ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color=\"gray\", alpha=0.2)\n", + "\n", + "ax2.set_ylim(ymin, ymax)\n", + "\n", + "# ax2.set_ylabel(r'$\\theta \\xi_-$', fontsize=16)\n", + "ax2.set_xlabel(r\"$\\theta$ (arcmin)\", fontsize=26)\n", + "ax2.set_xlim([theta.min() - 0.1, theta.max() + 20])\n", + "ax2.set_xscale(\"log\")\n", + "ax2.set_title(r\"$\\tau_{2,+}(\\theta)$\", fontsize=26)\n", + "ax2.set_xticks(np.array([1, 10, 100]))\n", + "ax2.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", + "ax2.tick_params(axis=\"both\", which=\"major\", labelsize=24)\n", + "ax2.tick_params(axis=\"both\", which=\"minor\", labelsize=20)\n", + "ax2.yaxis.get_offset_text().set_fontsize(24)\n", + "ax2.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", + "ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=20)\n", + "\n", + "plt.savefig(\n", + " \"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/notebooks/Plots/best_fit_tau_02_SP_v1.4.6.3_B.pdf\",\n", + " bbox_inches=\"tight\",\n", + ")\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "-all", + "main_language": "python", + "notebook_metadata_filter": "-all" + }, + "kernelspec": { + "display_name": "my_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/contours.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/contours.ipynb new file mode 100644 index 00000000..e95be1b5 --- /dev/null +++ b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/contours.ipynb @@ -0,0 +1,950 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# 2D contour plots\n", + "\n", + "This notebook produces the plots for all the 2D contours in the results section." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "import os.path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import seaborn as sns\n", + "from astropy.io import fits\n", + "from getdist import plots\n", + "\n", + "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", + "\n", + "plt.rcParams[\"text.usetex\"] = True\n", + "\n", + "sns.set_palette(\"husl\")\n", + "g = plots.get_subplot_plotter(width_inch=30)\n", + "g.settings.axes_fontsize = 70\n", + "g.settings.axes_labelsize = 80\n", + "g.settings.alpha_filled_add = 0.7\n", + "g.settings.legend_fontsize = 70\n", + "\n", + "\n", + "# SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE\n", + "\n", + "root_dir = \"/n09data/guerrini/output_chains/\"\n", + "path_datavectors = \"/home/guerrini/sp_validation/cosmo_inference/data/\"\n", + "path_output_chains = \"/n09data/guerrini/output_chains/\"\n", + "\n", + "data = fits.open(\n", + " os.path.join(\n", + " path_datavectors,\n", + " \"SP_v1.4.6.3_config/SP_v1.4.6.3_B/cosmosis_SP_v1.4.6.3_leak_corr_B_masked.fits\",\n", + " )\n", + ")\n", + "\n", + "roots_fid = {\n", + " \"SP_v1.4.6.3_leak_corr_B\": r\"UNIONS-3500 $C_\\ell$\",\n", + " \"SP_v1.4.6.3_B_fiducial_config\": r\"UNIONS-3500 $\\xi_\\pm$ (This work) \",\n", + " \"KiDS-Legacy_xipm\": r\"KiDS-Legacy $\\xi_\\pm$\",\n", + " \"HSC_Y3\": r\"HSC-Y3 $\\xi_\\pm$\",\n", + " \"Planck18\": r\"$\\textit{Planck}$ 2018\",\n", + "}\n", + "\n", + "roots_full = {\n", + " \"SP_v1.4.6.3_B_fiducial_config\": r\"UNIONS-3500 $\\xi_\\pm$ (This work) \",\n", + "}\n", + "\n", + "roots_ia = {\n", + " \"SP_v1.4.6.3_B_fiducial_config\": r\"Gaussian $A_{\\rm{IA}}$ prior\",\n", + " \"SP_v1.4.6.3_B_flat_ia_config\": r\"Flat $A_{\\rm{IA}}$ prior\",\n", + " \"SP_v1.4.6.3_B_no_ia_config\": r\"No IA\",\n", + "}\n", + "\n", + "roots_ext = {\n", + " \"SP_v1.4.6.3_B_fiducial_config\": r\"UNIONS-3500 $\\xi_\\pm$\",\n", + " \"SP_v1.4.6.3_B_planck_config\": r\"UNIONS-3500 $\\xi_\\pm$ + CMB\",\n", + " \"SP_v1.4.6.3_B_planck_desi_config\": r\"UNIONS-3500 $\\xi_\\pm$ + CMB + BAO\",\n", + " \"Planck18\": r\"$\\textit{Planck}$ 2018\",\n", + "}\n", + "\n", + "roots_dz = {\n", + " \"SP_v1.4.6.3_B_fiducial_config\": r\"Gaussian $\\Delta z$ prior\",\n", + " \"SP_v1.4.6.3_B_flat_delta_z_config\": r\"Flat $\\Delta z$ prior\",\n", + " \"SP_v1.4.6.3_B_no_delta_z_config\": r\"No $\\Delta z$ modelling\",\n", + "}\n", + "\n", + "roots_psf = {\n", + " \"SP_v1.4.6.3_B_flat_alpha_beta_config\": r\"Flat $\\alpha$ and $\\beta$ priors\",\n", + " \"SP_v1.4.6.3_B_fiducial_config\": r\"Gaussian $\\alpha$ and $\\beta$ priors\",\n", + " \"SP_v1.4.6.3_B_no_xi_sys_config\": r\"No $\\xi^{\\rm sys}$ included\",\n", + " \"SP_v1.4.6.3_B_no_leak_corr_config\": r\"No object-wise leakage correction\",\n", + "}\n", + "\n", + "roots_scale = {\n", + " \"SP_v1.4.6.3_B_fiducial_config\": r\"$\\xi_+$: $\\theta=[12,83]$\",\n", + " \"SP_v1.4.6.3_B_small_scales_config\": r\"$\\xi_+$: $\\theta=[5,83]$\",\n", + "}\n", + "\n", + "roots_nonlin = {\n", + " \"SP_v1.4.6.3_B_fiducial_config\": r\"Fiducial (\\texttt{HMCode2020}, $\\log(T_{\\rm AGN})$)\",\n", + " \"SP_v1.4.6.3_B_no_baryons_config\": r\"\\texttt{HMCode2020} no baryons\",\n", + " \"SP_v1.4.6.3_B_halofit_config\": r\"\\texttt{Halofit}\",\n", + "}\n", + "roots = roots_ext" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "## Retrieve the chains" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "# READ CHAIN\n", + "\n", + "chains = []\n", + "\n", + "for i, root in enumerate(list(roots.keys())):\n", + " burnin = 0\n", + " if \"SP\" not in root:\n", + " chain = g.samples_for_root(\n", + " root_dir + \"ext_data/{}/getdist_{}\".format(root, root),\n", + " cache=False,\n", + " settings={\n", + " \"ignore_rows\": burnin,\n", + " # 'smooth_scale_2D':0.2,\n", + " # 'smooth_scale_1D':0.2\n", + " },\n", + " )\n", + " p = chain.getParams()\n", + " if hasattr(p, \"S_8\") == False:\n", + " omega_m = chain.getParams().OMEGA_M\n", + " sigma_8 = chain.getParams().SIGMA_8\n", + "\n", + " s_8 = sigma_8 * (omega_m / 0.3) ** 0.5\n", + "\n", + " chain.addDerived(s_8, name=\"S_8\", label=r\"S_8\")\n", + "\n", + " p = chain.paramNames.parWithName(\"S_8\")\n", + "\n", + " elif \"config\" in root:\n", + " if os.path.isfile(root_dir + \"{}/getdist_{}.txt\".format(root, root)) == False:\n", + " samples = np.loadtxt(root_dir + \"{}/samples_{}.txt\".format(root, root))\n", + "\n", + " if \"nautilus\" in root:\n", + " weights = np.exp(samples[:, -3])\n", + " neglogL = samples[:, -2] - samples[:, -1]\n", + "\n", + " samples = np.column_stack((weights, neglogL, samples[:, 0:-3]))\n", + " elif \"mh\" in root:\n", + " samples = np.column_stack(\n", + " (\n", + " np.ones_like(samples[:, -1]),\n", + " np.log(samples[:, -1]) - np.log(samples[:, -2]),\n", + " samples[:, 0:-2],\n", + " )\n", + " )\n", + " burnin = 0.3\n", + " else:\n", + " samples = np.column_stack(\n", + " (samples[:, -1], samples[:, -3], samples[:, 0:-4])\n", + " )\n", + "\n", + " np.savetxt(root_dir + \"{}/getdist_{}.txt\".format(root, root), samples)\n", + "\n", + " chain = g.samples_for_root(\n", + " root_dir + \"{}/getdist_{}\".format(root, root),\n", + " cache=False,\n", + " settings={\n", + " \"ignore_rows\": burnin,\n", + " # 'smooth_scale_2D':0.2,\n", + " # 'smooth_scale_1D':0.2\n", + " },\n", + " )\n", + " else:\n", + " if (\n", + " os.path.isfile(\n", + " root_dir + \"{}/{}/getdist_{}_cell.txt\".format(root, root, root)\n", + " )\n", + " == False\n", + " ):\n", + " samples = np.loadtxt(\n", + " root_dir + \"{}/{}/samples_{}_cell.txt\".format(root, root, root)\n", + " )\n", + "\n", + " if \"nautilus\" in root:\n", + " weights = np.exp(samples[:, -3])\n", + " neglogL = samples[:, -2] - samples[:, -1]\n", + "\n", + " samples = np.column_stack((weights, neglogL, samples[:, 0:-3]))\n", + " elif \"mh\" in root:\n", + " samples = np.column_stack(\n", + " (\n", + " np.ones_like(samples[:, -1]),\n", + " np.log(samples[:, -1]) - np.log(samples[:, -2]),\n", + " samples[:, 0:-2],\n", + " )\n", + " )\n", + " burnin = 0.3\n", + " else:\n", + " samples = np.column_stack(\n", + " (samples[:, -1], samples[:, -3], samples[:, 0:-4])\n", + " )\n", + "\n", + " np.savetxt(\n", + " root_dir + \"{}/{}/getdist_{}_cell.txt\".format(root, root, root), samples\n", + " )\n", + "\n", + " chain = g.samples_for_root(\n", + " root_dir + \"{}/{}/getdist_{}_cell\".format(root, root, root),\n", + " cache=False,\n", + " settings={\n", + " \"ignore_rows\": burnin,\n", + " # 'smooth_scale_2D':0.2,\n", + " # 'smooth_scale_1D':0.2\n", + " },\n", + " )\n", + " p = chain.getParams()\n", + "\n", + " chains.append(chain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "name_list = [\n", + " \"OMEGA_M\",\n", + " \"ombh2\",\n", + " \"h0\",\n", + " \"n_s\",\n", + " \"SIGMA_8\",\n", + " \"S_8\",\n", + " \"logt_agn\",\n", + " \"a\",\n", + " \"m1\",\n", + " \"bias_1\",\n", + " \"alpha\",\n", + " \"beta\",\n", + " \"omch2\",\n", + "]\n", + "label_list = [\n", + " r\"\\Omega_{\\rm m}\",\n", + " r\"\\omega_{\\rm b}\",\n", + " r\"h\",\n", + " r\"n_{\\rm s}\",\n", + " r\"\\sigma_8\",\n", + " r\"S_8\",\n", + " r\"\\log T_{\\rm AGN}\",\n", + " r\"A_{\\rm IA}\",\n", + " r\"m_1\",\n", + " r\"\\Delta z\",\n", + " r\"\\alpha_{\\rm PSF}\",\n", + " r\"\\beta_{\\rm PSF}\",\n", + " r\"\\omega_{\\rm c}\",\n", + "]\n", + "\n", + "for chain in chains:\n", + " param_names = chain.getParamNames()\n", + " p = chain.getParams()\n", + " for name, label in zip(name_list, label_list):\n", + " if hasattr(p, name):\n", + " param_names.parWithName(name).label = label\n", + "\n", + "legend_labels = list(roots.values())" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "## Plot the chains" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, + "source": [ + "### FIDUCIAL PLOT" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "colours = [\n", + " \"royalblue\",\n", + " \"orange\",\n", + " \"crimson\",\n", + " \"forestgreen\",\n", + " \"indigo\",\n", + "]\n", + "\n", + "linestyle = [\"solid\", \"solid\", \"solid\", \"solid\", \"solid\"]\n", + "\n", + "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", + "\n", + "# FIDUCIAL PLOT\n", + "g.triangle_plot(\n", + " chains,\n", + " [\"SIGMA_8\", \"S_8\", \"OMEGA_M\"], #\n", + " legend_labels=legend_labels,\n", + " line_args=line_args,\n", + " contour_colors=colours,\n", + " label_order=[1, 0, 2, 3, 4],\n", + " filled=[True, True, False, False, True],\n", + ")\n", + "\n", + "g.export(\"../Plots/SP_v1.4.6.3_B_fiducial_config_contour_plot.pdf\")" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, + "source": [ + "### FULL PLOT" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "g.settings.axes_fontsize = 40\n", + "g.settings.axes_labelsize = 50\n", + "\n", + "colours = [\n", + " \"orange\",\n", + "]\n", + "\n", + "linestyle = [\n", + " \"solid\",\n", + "]\n", + "\n", + "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", + "\n", + "# FIDUCIAL PLOT\n", + "g.triangle_plot(\n", + " chains,\n", + " [\n", + " \"OMEGA_M\",\n", + " \"ombh2\",\n", + " \"h0\",\n", + " \"n_s\",\n", + " \"SIGMA_8\",\n", + " \"S_8\",\n", + " \"logt_agn\",\n", + " \"a\",\n", + " \"m1\",\n", + " \"bias_1\",\n", + " ],\n", + " legend_labels=legend_labels,\n", + " line_args=line_args,\n", + " contour_colors=colours,\n", + " filled=True,\n", + ")\n", + "\n", + "g.export(\"../Plots/SP_v1.4.6.3_B_fiducial_config_contour_plot_full.pdf\")" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": {}, + "source": [ + "### IA PLOT" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "colours = [\n", + " \"orange\",\n", + " \"royalblue\",\n", + " \"forestgreen\",\n", + "]\n", + "\n", + "linestyle = [\n", + " \"solid\",\n", + " \"solid\",\n", + " \"solid\",\n", + "]\n", + "\n", + "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", + "\n", + "g.triangle_plot(\n", + " chains,\n", + " [\"S_8\", \"OMEGA_M\", \"a\"], #\n", + " legend_labels=legend_labels,\n", + " line_args=line_args,\n", + " contour_args={\"alpha\": 0.6},\n", + " contour_colors=colours,\n", + " filled=[True, False, True],\n", + ")\n", + "\n", + "g.export(\"../Plots/SP_v1.4.6.3_B_fiducial_config_contour_plot_ia.pdf\")" + ] + }, + { + "cell_type": "markdown", + "id": "12", + "metadata": {}, + "source": [ + "### PSF PLOT" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "colours = [\n", + " \"royalblue\",\n", + " \"orange\",\n", + " \"hotpink\",\n", + " \"slategray\",\n", + "]\n", + "\n", + "linestyle = [\n", + " \"solid\",\n", + " \"solid\",\n", + " \"solid\",\n", + " \"solid\",\n", + "]\n", + "\n", + "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", + "\n", + "g.triangle_plot(\n", + " chains,\n", + " [\"S_8\", \"OMEGA_M\", \"alpha\", \"beta\"], #\n", + " legend_labels=legend_labels,\n", + " line_args=line_args,\n", + " contour_args=[{\"alpha\": 1}, {\"alpha\": 0.6}, {\"alpha\": 0.8}, {\"alpha\": 0.8}],\n", + " contour_colors=colours,\n", + " legend_loc=\"upper right\",\n", + " label_order=[1, 0, 2, 3],\n", + " filled=[False, True, True, True],\n", + ")\n", + "\n", + "g.subplots[3, 2].scatter(\n", + " 0.005, 0.81, color=\"k\", marker=\"X\", s=400, label=\"Fiducial config best-fit\"\n", + ")\n", + "g.subplots[3, 2].scatter(\n", + " 0.022, 0.798, color=\"k\", marker=\"P\", s=400, label=\"Fiducial config best-fit\"\n", + ")\n", + "\n", + "g.export(\"../Plots/SP_v1.4.6.3_B_fiducial_config_contour_plot_psf.pdf\")" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "### DELTA Z PLOT" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "colours = [\n", + " \"orange\",\n", + " \"royalblue\",\n", + " \"indigo\",\n", + "]\n", + "\n", + "linestyle = [\n", + " \"solid\",\n", + " \"solid\",\n", + " \"solid\",\n", + "]\n", + "\n", + "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", + "g.triangle_plot(\n", + " chains,\n", + " [\"S_8\", \"OMEGA_M\", \"bias_1\"], #\n", + " legend_labels=legend_labels,\n", + " line_args=line_args,\n", + " contour_args=[{\"alpha\": 1.0}, {\"alpha\": 0.9}, {\"alpha\": 0.5}],\n", + " contour_colors=colours,\n", + " filled=[True, False, True],\n", + ")\n", + "\n", + "g.export(\"../Plots/SP_v1.4.6.3_B_fiducial_config_contour_plot_dz.pdf\")" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "### EXTERNAL DATA" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "colours = [\n", + " \"orange\",\n", + " \"royalblue\",\n", + " \"crimson\",\n", + " \"forestgreen\",\n", + "]\n", + "\n", + "linestyle = [\n", + " \"solid\",\n", + " \"solid\",\n", + " \"solid\",\n", + " \"solid\",\n", + " \"solid\",\n", + "]\n", + "\n", + "line_args = [dict(color=col, ls=ls) for col, ls in zip(colours, linestyle)]\n", + "\n", + "g = plots.get_subplot_plotter(width_inch=10)\n", + "g.settings.axes_fontsize = 25\n", + "g.settings.axes_labelsize = 25\n", + "g.settings.legend_fontsize = 22\n", + "\n", + "g.plot_2d(\n", + " chains,\n", + " [\"S_8\", \"OMEGA_M\", \"SIGMA_8\"], #\n", + " line_args=line_args,\n", + " contour_colors=colours,\n", + " legend_labels=legend_labels,\n", + " alphas=[0.7, 1.0, 1.0, 1.0],\n", + " filled=[True, True, True, False],\n", + ")\n", + "\n", + "g.add_y_bands(0.2975, 0.0086, alpha2=0, color=\"k\", label=\"BAO\")\n", + "g.add_legend(legend_labels, legend_loc=\"upper right\")\n", + "\n", + "g.export(\"../Plots/SP_v1.4.6.3_B_fiducial_config_contour_plot_ext.pdf\")" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "### Small scales" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "colours = [\n", + " \"orange\",\n", + " \"dodgerblue\",\n", + "]\n", + "\n", + "linestyle = [\n", + " \"solid\",\n", + " \"solid\",\n", + "]\n", + "\n", + "line_args = [dict(color=col, ls=ls) for col, ls in zip(colours, linestyle)]\n", + "\n", + "g = plots.get_subplot_plotter(width_inch=9)\n", + "g.settings.axes_fontsize = 25\n", + "g.settings.axes_labelsize = 25\n", + "g.settings.alpha_filled_add = 0.7\n", + "g.settings.legend_fontsize = 30\n", + "\n", + "g.plot_2d(\n", + " chains,\n", + " [\"S_8\", \"OMEGA_M\"], #\n", + " line_args=line_args,\n", + " contour_args=[{\"alpha\": 0.7}, {\"alpha\": 1.0}],\n", + " contour_colors=colours,\n", + " filled=[True, True],\n", + ")\n", + "g.add_legend(legend_labels, legend_loc=\"upper right\")\n", + "\n", + "g.export(\"../Plots/SP_v1.4.6.3_B_fiducial_config_contour_plot_scales.pdf\")" + ] + }, + { + "cell_type": "markdown", + "id": "20", + "metadata": {}, + "source": [ + "### BBN Prior" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "from getdist.gaussian_mixtures import Gaussian1D\n", + "\n", + "colours = [\n", + " \"orange\",\n", + " \"royalblue\",\n", + "]\n", + "\n", + "linestyle = [\n", + " \"solid\",\n", + " \"solid\",\n", + "]\n", + "\n", + "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", + "\n", + "# BBN PRIOR\n", + "bbn_prior = Gaussian1D(\n", + " mean=0.02218,\n", + " sigma=0.00055,\n", + " name=\"ombh2\",\n", + " labels=[r\"\\omega_{\\rm b}\"],\n", + " label=\"BBN prior\",\n", + ")\n", + "bbn_chain = bbn_prior.MCSamples(3000, label=\"BBN prior\")\n", + "\n", + "g.triangle_plot(\n", + " chains + [bbn_chain],\n", + " name_list,\n", + " legend_labels=legend_labels,\n", + " line_args=line_args,\n", + " contour_colors=colours,\n", + " filled=[True, False],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, + "source": [ + "## Plot the best-fit $\\xi_\\pm$" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "xi_p_data = data[\"XI_PLUS\"].data\n", + "xi_m_data = data[\"XI_MINUS\"].data\n", + "cov_mat = data[\"COVMAT\"].data\n", + "\n", + "labels = roots_scale.values()\n", + "\n", + "bbox_to_anchor_xip = (0.685, 0.09)\n", + "bbox_to_anchor_xim = (0.3, 0.65)\n", + "theta_min = 1.0\n", + "theta_max = 250.0\n", + "loc_legend = \"lower center\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], + "source": [ + "colours = [\n", + " \"orange\",\n", + " \"dodgerblue\",\n", + "]\n", + "\n", + "linestyle = [\n", + " \"solid\",\n", + " \"solid\",\n", + "]\n", + "\n", + "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", + "\n", + "labels = roots_scale.values()\n", + "\n", + "fig, ax = plt.subplots(1, 1, figsize=(11, 7))\n", + "\n", + "theta, xi_p, xi_m = xi_p_data[\"ANG\"], xi_p_data[\"VALUE\"], xi_m_data[\"VALUE\"]\n", + "ax.errorbar(\n", + " theta,\n", + " theta * xi_p,\n", + " yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])),\n", + " fmt=\"o\",\n", + " color=\"black\",\n", + " capsize=2,\n", + ")\n", + "\n", + "for idx, (label, root) in enumerate(zip(labels, roots_scale)):\n", + " # Read the results\n", + " theta = (\n", + " (\n", + " np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " )\n", + " * 180\n", + " / np.pi\n", + " * 60\n", + " )\n", + " xi_plus = np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_minus = np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_sys_plus = np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + " theta_xi_sys = (\n", + " np.loadtxt(path_output_chains + \"{}/best_fit/xi_sys/theta.txt\".format(root))\n", + " * 180\n", + " / np.pi\n", + " * 60\n", + " )\n", + "\n", + " xi_sys_plus = np.interp(theta, theta_xi_sys, xi_sys_plus)\n", + " xi_sys_minus = np.interp(theta, theta_xi_sys, xi_sys_minus)\n", + " xi_plus += xi_sys_plus\n", + " xi_minus += xi_sys_minus\n", + "\n", + " mask = (theta > theta_min) & (theta < theta_max)\n", + " theta = theta[mask]\n", + " ax.plot(theta, theta * xi_plus[mask], label=label, **line_args[idx])\n", + "\n", + "ymin = ax.get_ylim()[0]\n", + "ymax = ax.get_ylim()[1]\n", + "\n", + "ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color=\"gray\", alpha=0.2)\n", + "ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=5, color=\"gray\", alpha=0.7)\n", + "ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=300, color=\"gray\", alpha=0.2)\n", + "\n", + "ax.set_ylim(ymin, ymax)\n", + "\n", + "ax.set_ylabel(r\"$\\theta \\xi_\\pm$\", fontsize=26)\n", + "ax.set_xlabel(r\"$\\theta$ (arcmin)\", fontsize=26)\n", + "ax.set_xlim([theta.min() - 0.1, theta.max() + 20])\n", + "ax.set_title(r\"$\\xi_+(\\theta)$\", fontsize=26)\n", + "ax.set_xscale(\"log\")\n", + "ax.set_xticks(np.array([1, 10, 100]))\n", + "ax.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", + "ax.tick_params(axis=\"both\", which=\"major\", labelsize=24)\n", + "ax.tick_params(axis=\"both\", which=\"minor\", labelsize=20)\n", + "ax.yaxis.get_offset_text().set_fontsize(24)\n", + "ax.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", + "ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=20)\n", + "\n", + "\n", + "plt.savefig(\"./../Plots/scale_cut_xipm_SP_v1.4.6.3_B.pdf\", bbox_inches=\"tight\")\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25", + "metadata": {}, + "outputs": [], + "source": [ + "labels = roots_nonlin.values()\n", + "\n", + "colours = [\"orange\", \"hotpink\", \"teal\"]\n", + "\n", + "linestyle = [\"solid\", \"solid\", \"dashed\"]\n", + "\n", + "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", + "\n", + "fig, [ax, ax2] = plt.subplots(2, 1, figsize=(11, 14))\n", + "\n", + "theta, xi_p, xi_m = xi_p_data[\"ANG\"], xi_p_data[\"VALUE\"], xi_m_data[\"VALUE\"]\n", + "ax.errorbar(\n", + " theta,\n", + " theta * xi_p,\n", + " yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])),\n", + " fmt=\"o\",\n", + " color=\"black\",\n", + " capsize=2,\n", + ")\n", + "ax2.errorbar(\n", + " theta,\n", + " theta * xi_m,\n", + " yerr=theta\n", + " * np.sqrt(\n", + " np.diag(cov_mat[len(theta) : 2 * len(theta), len(theta) : 2 * len(theta)])\n", + " ),\n", + " fmt=\"o\",\n", + " color=\"black\",\n", + " capsize=2,\n", + ")\n", + "\n", + "for idx, (label, root) in enumerate(zip(labels, roots_nonlin)):\n", + " # Read the results\n", + " theta = (\n", + " (\n", + " np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " )\n", + " * 180\n", + " / np.pi\n", + " * 60\n", + " )\n", + " xi_plus = np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_minus = np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + " xi_sys_plus = np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " path_output_chains + \"{}/best_fit/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + " theta_xi_sys = (\n", + " np.loadtxt(path_output_chains + \"{}/best_fit/xi_sys/theta.txt\".format(root))\n", + " * 180\n", + " / np.pi\n", + " * 60\n", + " )\n", + "\n", + " xi_sys_plus = np.interp(theta, theta_xi_sys, xi_sys_plus)\n", + " xi_sys_minus = np.interp(theta, theta_xi_sys, xi_sys_minus)\n", + " xi_plus += xi_sys_plus\n", + " xi_minus += xi_sys_minus\n", + "\n", + " mask = (theta > theta_min) & (theta < theta_max)\n", + " theta = theta[mask]\n", + " ax.plot(theta, theta * xi_plus[mask], label=label, **line_args[idx])\n", + " ax2.plot(theta, theta * xi_minus[mask], label=label, **line_args[idx])\n", + "\n", + "ymin = ax.get_ylim()[0]\n", + "ymax = ax.get_ylim()[1]\n", + "ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color=\"gray\", alpha=0.2)\n", + "ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=300, color=\"gray\", alpha=0.2)\n", + "\n", + "ax.set_ylim(ymin, ymax)\n", + "\n", + "ax.set_ylabel(r\"$\\theta \\xi_\\pm$\", fontsize=26)\n", + "ax.set_xlabel(r\"$\\theta$ (arcmin)\", fontsize=26)\n", + "ax.set_xlim([theta.min() - 0.1, theta.max() + 20])\n", + "ax.set_title(r\"$\\xi_+(\\theta)$\", fontsize=26)\n", + "ax.set_xscale(\"log\")\n", + "ax.set_xticks(np.array([1, 10, 100]))\n", + "ax.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", + "ax.tick_params(axis=\"both\", which=\"major\", labelsize=24)\n", + "ax.tick_params(axis=\"both\", which=\"minor\", labelsize=20)\n", + "ax.yaxis.get_offset_text().set_fontsize(24)\n", + "ax.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", + "\n", + "\n", + "ymin = ax2.get_ylim()[0]\n", + "ymax = ax2.get_ylim()[1]\n", + "ax2.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color=\"gray\", alpha=0.2)\n", + "ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=3000, color=\"gray\", alpha=0.2)\n", + "\n", + "ax2.set_ylim(ymin, ymax)\n", + "ax2.set_xlabel(r\"$\\theta$ (arcmin)\", fontsize=26)\n", + "ax2.set_xlim([theta.min() - 0.1, theta.max()])\n", + "ax2.set_xscale(\"log\")\n", + "ax2.set_title(r\"$\\xi_-(\\vartheta)$\", fontsize=26)\n", + "ax2.set_xticks(np.array([1, 10, 100]))\n", + "ax2.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", + "ax2.tick_params(axis=\"both\", which=\"major\", labelsize=24)\n", + "ax2.tick_params(axis=\"both\", which=\"minor\", labelsize=20)\n", + "ax2.yaxis.get_offset_text().set_fontsize(24)\n", + "ax2.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", + "ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=20)\n", + "\n", + "plt.savefig(\"./../Plots/nonlin_xipm_SP_v1.4.6.3_B.pdf\", bbox_inches=\"tight\")\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "my_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2.ipynb new file mode 100644 index 00000000..f124e4cd --- /dev/null +++ b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2.ipynb @@ -0,0 +1,690 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import configparser\n", + "import os\n", + "import re\n", + "import subprocess\n", + "import sys\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import scipy.stats as stats\n", + "from astropy.io import fits\n", + "from getdist import plots\n", + "from IPython.display import Markdown, display\n", + "from scipy.interpolate import interp1d\n", + "\n", + "sys.path.append(\"/home/guerrini/sp_validation/cosmo_inference/scripts\")\n", + "\n", + "import chain_postprocessing\n", + "\n", + "%matplotlib inline\n", + "\n", + "plt.rc(\"mathtext\", fontset=\"stix\")\n", + "plt.rc(\"font\", family=\"sans-serif\")\n", + "\n", + "g = plots.get_subplot_plotter(width_inch=30)\n", + "g.settings.axes_fontsize = 30\n", + "g.settings.axes_labelsize = 30\n", + "g.settings.alpha_filled_add = 0.7\n", + "g.settings.legend_fontsize = 40\n", + "\n", + "# #SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE\n", + "root_dir = \"/n09data/guerrini/output_chains/\"\n", + "blind = \"B\"\n", + "\n", + "roots = [\n", + " f\"SP_v1.4.6.3_{blind}_fiducial_config\",\n", + " f\"SP_v1.4.6.3_{blind}_small_scales_config\",\n", + " f\"SP_v1.4.6.3_{blind}_flat_alpha_beta_config\",\n", + " f\"SP_v1.4.6.3_{blind}_no_xi_sys_config\",\n", + " f\"SP_v1.4.6.3_{blind}_no_leak_corr_config\",\n", + " f\"SP_v1.4.6.3_{blind}_flat_delta_z_config\",\n", + " f\"SP_v1.4.6.3_{blind}_no_delta_z_config\",\n", + " f\"SP_v1.4.6.3_{blind}_flat_ia_config\",\n", + " f\"SP_v1.4.6.3_{blind}_no_ia_config\",\n", + " f\"SP_v1.4.6.3_{blind}_no_m_bias_config\",\n", + " f\"SP_v1.4.6.3_{blind}_unmasked_covmat_config\",\n", + " f\"SP_v1.4.6.3_{blind}_halofit_config\",\n", + " f\"SP_v1.4.6.3_{blind}_no_baryons_config\",\n", + " f\"SP_v1.4.6.3_{blind}_nautilus_config\",\n", + " f\"SP_v1.4.6.3_{blind}_planck_config\",\n", + " f\"SP_v1.4.6.3_{blind}_planck_desi_config\",\n", + "]\n", + "\n", + "catalog_versions = [\n", + " f\"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}\",\n", + "]\n", + "\n", + "catalog_sub_versions = [\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", + " f\"SP_v1.4.6.3_{blind}_masked\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", + " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", + "]\n", + "output_folder = \"/n09data/guerrini/output_chains/\"\n", + "\n", + "path_ini_files = \"/home/guerrini/sp_validation/cosmo_inference/cosmosis_config/\"\n", + "\n", + "\n", + "ini_roots = [\n", + " f\"blind_{blind}/fiducial\",\n", + " f\"blind_{blind}/small_scales\",\n", + " f\"blind_{blind}/flat_alpha_beta\",\n", + " f\"blind_{blind}/no_xi_sys\",\n", + " f\"blind_{blind}/no_leak_corr\",\n", + " f\"blind_{blind}/flat_delta_z\",\n", + " f\"blind_{blind}/no_delta_z\",\n", + " f\"blind_{blind}/flat_ia\",\n", + " f\"blind_{blind}/no_ia\",\n", + " f\"blind_{blind}/no_m_bias\",\n", + " f\"blind_{blind}/unmasked_covmat\",\n", + " f\"blind_{blind}/halofit\",\n", + " f\"blind_{blind}/no_baryons\",\n", + " f\"blind_{blind}/nautilus\",\n", + " f\"blind_{blind}/planck\",\n", + " f\"blind_{blind}/planck_desi\",\n", + "]\n", + "\n", + "properties = {}\n", + "\n", + "for i, root in enumerate(roots):\n", + " print(root)\n", + " config = configparser.ConfigParser()\n", + " config.optionxform = str # Preserve case sensitivity of option names\n", + " config.read(\n", + " path_ini_files\n", + " + \"config_space_v1.4.6.3_fiducial/pipeline/\"\n", + " + ini_roots[i]\n", + " + \".ini\"\n", + " )\n", + " add_xi_sys = config[\"2pt_like\"][\"add_xi_sys\"]\n", + " lower_bound_xi_plus, upper_bound_xi_plus = map(\n", + " float, config[\"2pt_like\"][\"angle_range_XI_PLUS_1_1\"].split()\n", + " )\n", + " lower_bound_xi_minus, upper_bound_xi_minus = map(\n", + " float, config[\"2pt_like\"][\"angle_range_XI_MINUS_1_1\"].split()\n", + " )\n", + "\n", + " properties[root] = {\n", + " \"add_xi_sys\": add_xi_sys,\n", + " \"lower_bound_xi_plus\": lower_bound_xi_plus,\n", + " \"upper_bound_xi_plus\": upper_bound_xi_plus,\n", + " \"lower_bound_xi_minus\": lower_bound_xi_minus,\n", + " \"upper_bound_xi_minus\": upper_bound_xi_minus,\n", + " }" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Retrieve the chains" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# READ CHAIN\n", + "\n", + "chains = []\n", + "\n", + "for i, root in enumerate(roots):\n", + " burnin = 0\n", + "\n", + " if os.path.isfile(root_dir + \"{}/getdist_{}.txt\".format(root, root)) == False:\n", + " samples = np.loadtxt(root_dir + \"{}/samples_{}.txt\".format(root, root))\n", + "\n", + " if \"nautilus\" in root:\n", + " samples = np.column_stack(\n", + " (\n", + " np.exp(samples[:, -3]),\n", + " samples[:, -1] - samples[:, -2],\n", + " samples[:, 0:-3],\n", + " )\n", + " )\n", + " elif \"mh\" in root:\n", + " samples = np.column_stack(\n", + " (\n", + " np.ones_like(samples[:, -1]),\n", + " np.log(samples[:, -1]) - np.log(samples[:, -2]),\n", + " samples[:, 0:-2],\n", + " )\n", + " )\n", + " burnin = 0.3\n", + " else:\n", + " samples = np.column_stack(\n", + " (samples[:, -1], samples[:, -3], samples[:, 0:-4])\n", + " )\n", + "\n", + " np.savetxt(root_dir + \"{}/getdist_{}.txt\".format(root, root), samples)\n", + "\n", + " chain = g.samples_for_root(\n", + " root_dir + \"{}/getdist_{}\".format(root, root),\n", + " cache=False,\n", + " settings={\n", + " \"ignore_rows\": burnin,\n", + " \"smooth_scale_2D\": 0.5,\n", + " \"smooth_scale_1D\": 0.5,\n", + " },\n", + " )\n", + " p = chain.getParams()\n", + "\n", + " chains.append(chain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "param_list = [\n", + " \"OMEGA_M\",\n", + " \"ombh2\",\n", + " \"h0\",\n", + " \"n_s\",\n", + " \"SIGMA_8\",\n", + " \"s_8_input\",\n", + " \"logt_agn\",\n", + " \"a\",\n", + " \"m1\",\n", + " \"bias_1\",\n", + " \"alpha\",\n", + " \"beta\",\n", + " \"omch2\",\n", + " \"m\",\n", + " \"a_planck\",\n", + "]\n", + "label_list = [\n", + " r\"\\Omega_m\",\n", + " r\"\\omega_b\",\n", + " \"h_0\",\n", + " \"n_s\",\n", + " r\"\\sigma_8\",\n", + " \"S_8\",\n", + " \"log T_{AGN}\",\n", + " \"A_{IA}\",\n", + " \"m_1\",\n", + " r\"\\Delta z_1\",\n", + " \"\\\\alpha_{PSF}\",\n", + " \"\\\\beta_{PSF}\",\n", + " r\"\\omega_c\",\n", + " \"M\",\n", + " \"A_{\\rm Planck}\",\n", + "]\n", + "\n", + "for chain in chains:\n", + " param_names = chain.getParamNames()\n", + " for name, label in zip(param_list, label_list):\n", + " if param_names.parWithName(name) is not None:\n", + " param_names.parWithName(name).label = label" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Extract the best fit parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "best_fit = {}\n", + "\n", + "for root, chain in zip(roots, chains):\n", + " print(root)\n", + " p = chain.getParams()\n", + "\n", + " best_fit[root] = chain_postprocessing.extract_best_fit_params(\n", + " chain, best_fit_method=\"2Dkde\"\n", + " )\n", + "\n", + " for param_name in best_fit[root].keys():\n", + " high_68, low_68, high_95, low_95 = chain_postprocessing.compute_limits(\n", + " chain, param_name\n", + " )\n", + " if param_name == \"S_8\":\n", + " print(f\"{best_fit[root][param_name]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run `Cosmosis` in test mode to get the data vectors" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if not os.path.exists(path_ini_files + \"/values_empty.ini\"):\n", + " content = \"\"\"[cosmological_parameters]\n", + "\n", + "tau = 0.0544\n", + "w = -1.0\n", + "mnu = 0.06\n", + "omega_k = 0.0\n", + "wa = 0.0\n", + "\n", + "[halo_model_parameters]\n", + "\n", + "[intrinsic_alignment_parameters]\n", + "\n", + "[shear_calibration_parameters]\n", + "\n", + "[nofz_shifts]\n", + "\n", + "[psf_leakage_parameters]\n", + "\"\"\"\n", + "\n", + " with open(path_ini_files + \"/values_empty.ini\", \"w\") as f:\n", + " f.write(content)\n", + " f.close()\n", + "\n", + " print(\"File created successfully\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "section_map = {\n", + " \"omch2\": \"cosmological_parameters\",\n", + " \"ombh2\": \"cosmological_parameters\",\n", + " \"h0\": \"cosmological_parameters\",\n", + " \"n_s\": \"cosmological_parameters\",\n", + " \"tau\": \"cosmological_parameters\",\n", + " \"s_8_input\": \"cosmological_parameters\",\n", + " \"logt_agn\": \"halo_model_parameters\",\n", + " \"a\": \"intrinsic_alignment_parameters\",\n", + " \"m1\": \"shear_calibration_parameters\",\n", + " \"bias_1\": \"nofz_shifts\",\n", + " \"alpha\": \"psf_leakage_parameters\",\n", + " \"beta\": \"psf_leakage_parameters\",\n", + " \"m\": \"supernova_params\",\n", + " \"a_planck\": \"planck\",\n", + "}\n", + "\n", + "best_fit[\"SP_v1.4.6.3_B_no_ia_config\"][\"a\"] = 0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "env = os.environ.copy()\n", + "env[\"LD_LIBRARY_PATH\"] = (\n", + " \"/home/guerrini/.conda/envs/sp_validation/lib/python3.9/site-packages/cosmosis/datablock:\"\n", + " + env.get(\"LD_LIBRARY_PATH\", \"\")\n", + ")\n", + "\n", + "for i, root in enumerate(roots):\n", + " print(root)\n", + " config = configparser.ConfigParser()\n", + " config.optionxform = str # Preserve case sensitivity of option names\n", + "\n", + " for param, section in section_map.items():\n", + " # Check if this parameter exists for the current root\n", + " if param in best_fit[root]:\n", + " value = best_fit[root][param]\n", + "\n", + " if section not in config:\n", + " config.add_section(section)\n", + "\n", + " config[section][param] = str(value)\n", + "\n", + " with open(path_ini_files + \"/values_empty.ini\", \"w\") as configfile:\n", + " config.write(configfile)\n", + "\n", + " # Modify the ini file to run in test mode at the best fit\n", + " config = configparser.ConfigParser()\n", + " config.optionxform = str # Preserve case sensitivity of option names\n", + "\n", + " ini_file = path_ini_files + \"config_space_v1.4.6.3_fiducial/pipeline/{}.ini\".format(\n", + " ini_roots[i]\n", + " )\n", + " config.read(ini_file)\n", + "\n", + " sampler = config[\"runtime\"][\"sampler\"]\n", + " config[\"runtime\"][\"sampler\"] = \"test\"\n", + " values = config[\"pipeline\"][\"values\"]\n", + " config[\"pipeline\"][\"values\"] = path_ini_files + \"/values_empty.ini\"\n", + " config[\"DEFAULT\"][\"FITS_FILE\"] = (\n", + " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_versions[0]}/cosmosis_{catalog_sub_versions[i]}.fits\"\n", + " )\n", + " config[\"test\"][\"save_dir\"] = root_dir + \"{}/best_fit\".format(root)\n", + "\n", + " with open(ini_file, \"w\") as configfile:\n", + " config.write(configfile)\n", + "\n", + " # Run cosmosis\n", + " result = subprocess.run(\n", + " [\"cosmosis\", ini_file], env=env, capture_output=True, text=True\n", + " )\n", + " print(f\"STDOUT:\\n{result.stdout}\")\n", + " print(f\"STDERR:\\n{result.stderr}\")\n", + "\n", + " # Modify the ini file to the previous one\n", + " config[\"pipeline\"][\"values\"] = values\n", + " config[\"runtime\"][\"sampler\"] = sampler\n", + "\n", + " with open(ini_file, \"w\") as configfile:\n", + " config.write(configfile)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute the $\\chi^2$" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metrics = {}\n", + "\n", + "for idx, root in enumerate(roots):\n", + " print(root)\n", + " match = re.search(r\"corr_([A-Za-z])\", root)\n", + " if match:\n", + " blind = match.group(1)\n", + "\n", + " add_xi_sys = properties[root][\"add_xi_sys\"]\n", + " print(f\"add_xi_sys: {add_xi_sys}\")\n", + " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", + " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", + " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", + " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", + "\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder + \"{}/best_fit/shear_xi_plus/theta.txt\".format(root)\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " shear_xi_plus = np.loadtxt(\n", + " output_folder + \"{}/best_fit/shear_xi_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " shear_xi_minus = np.loadtxt(\n", + " output_folder + \"{}/best_fit/shear_xi_minus/bin_1_1.txt\".format(root)\n", + " )\n", + "\n", + " if add_xi_sys == \"T\":\n", + " xi_sys_plus = np.loadtxt(\n", + " output_folder + \"{}/best_fit/xi_sys/shear_xi_plus.txt\".format(root)\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " output_folder + \"{}/best_fit/xi_sys/shear_xi_minus.txt\".format(root)\n", + " )\n", + "\n", + " theta_tau = np.loadtxt(\n", + " output_folder + \"{}/best_fit/tau_0_plus/theta.txt\".format(root)\n", + " )\n", + " theta_tau_arcmin = theta_tau * 180 * 60 / np.pi\n", + " tau_0_model = np.loadtxt(\n", + " output_folder + \"{}/best_fit/tau_0_plus/bin_1_1.txt\".format(root)\n", + " )\n", + " tau_2_model = np.loadtxt(\n", + " output_folder + \"{}/best_fit/tau_2_plus/bin_1_1.txt\".format(root)\n", + " )\n", + "\n", + " data = fits.open(\n", + " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_versions[0]}/cosmosis_{catalog_sub_versions[idx]}.fits\"\n", + " )\n", + "\n", + " tau_0_data = data[\"TAU_0_PLUS\"].data[\"VALUE\"]\n", + " tau_2_data = data[\"TAU_2_PLUS\"].data[\"VALUE\"]\n", + "\n", + " theta_data = data[\"XI_PLUS\"].data[\"ANG\"]\n", + " xi_plus_data = data[\"XI_PLUS\"].data[\"VALUE\"]\n", + " xi_minus_data = data[\"XI_MINUS\"].data[\"VALUE\"]\n", + "\n", + " # Load the covariance\n", + " cov = data[\"COVMAT\"].data\n", + " cov_xi = cov[0 : 2 * len(xi_plus_data), 0 : 2 * len(xi_plus_data)]\n", + " cov_tau = cov[2 * len(xi_plus_data) :, 2 * len(xi_plus_data) :]\n", + "\n", + " # interpolate the model\n", + " interp_xi_plus = interp1d(\n", + " theta_arcmin, shear_xi_plus, kind=\"cubic\", fill_value=\"extrapolate\"\n", + " )\n", + " interp_xi_minus = interp1d(\n", + " theta_arcmin, shear_xi_minus, kind=\"cubic\", fill_value=\"extrapolate\"\n", + " )\n", + "\n", + " xi_plus_model = interp_xi_plus(theta_data)\n", + " if add_xi_sys:\n", + " xi_plus_model += xi_sys_plus\n", + " xi_minus_model = interp_xi_minus(theta_data)\n", + " if add_xi_sys:\n", + " xi_minus_model += xi_sys_minus\n", + "\n", + " # Concatenate the data vector\n", + " xi_data = np.concatenate((xi_plus_data, xi_minus_data))\n", + " xi_model = np.concatenate((xi_plus_model, xi_minus_model))\n", + "\n", + " tau_data = np.concatenate((tau_0_data, tau_2_data))\n", + " tau_model = np.concatenate((tau_0_model, tau_2_model))\n", + "\n", + " # Apply scale cuts\n", + " mask_xi_plus = (theta_data > lower_bound_xi_plus) & (\n", + " theta_data < upper_bound_xi_plus\n", + " )\n", + " mask_xi_minus = (theta_data > lower_bound_xi_minus) & (\n", + " theta_data < upper_bound_xi_minus\n", + " )\n", + " mask = np.concatenate((mask_xi_plus, mask_xi_minus))\n", + "\n", + " xi_data = xi_data[mask]\n", + " xi_model = xi_model[mask]\n", + " cov_xi = cov_xi[mask][:, mask]\n", + "\n", + " cov_xi_plus = cov[0 : len(xi_plus_data), 0 : len(xi_plus_data)]\n", + " cov_xi_plus = cov_xi_plus[mask_xi_plus][:, mask_xi_plus]\n", + " cov_xi_minus = cov[\n", + " len(xi_plus_data) : 2 * len(xi_minus_data),\n", + " len(xi_plus_data) : 2 * len(xi_minus_data),\n", + " ]\n", + " cov_xi_minus = cov_xi_minus[mask_xi_minus][:, mask_xi_minus]\n", + "\n", + " xi_plus_chi2 = np.dot(\n", + " (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]),\n", + " np.dot(\n", + " np.linalg.inv(cov_xi_plus),\n", + " (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]),\n", + " ),\n", + " )\n", + " xi_minus_chi2 = np.dot(\n", + " (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]),\n", + " np.dot(\n", + " np.linalg.inv(cov_xi_minus),\n", + " (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]),\n", + " ),\n", + " )\n", + " xi_chi2 = np.dot(\n", + " (xi_model - xi_data), np.dot(np.linalg.inv(cov_xi), (xi_model - xi_data))\n", + " )\n", + " tau_chi2 = np.dot(\n", + " (tau_model - tau_data), np.dot(np.linalg.inv(cov_tau), (tau_model - tau_data))\n", + " )\n", + " n_dof_xi_plus = np.sum(mask_xi_plus)\n", + " n_dof_xi_minus = np.sum(mask_xi_minus)\n", + " n_dof_tau = len(tau_0_data) + len(tau_2_data)\n", + " p_value_xi_plus = 1 - stats.chi2.cdf(xi_plus_chi2, n_dof_xi_plus)\n", + " p_value_xi_minus = 1 - stats.chi2.cdf(xi_minus_chi2, n_dof_xi_minus)\n", + " p_value_xi = 1 - stats.chi2.cdf(xi_chi2, n_dof_xi_plus + n_dof_xi_minus)\n", + " p_value_tau = 1 - stats.chi2.cdf(tau_chi2, n_dof_tau)\n", + " chi2_tot = xi_plus_chi2 + xi_minus_chi2 + tau_chi2\n", + " n_dof_tot = n_dof_xi_plus + n_dof_xi_minus + n_dof_tau\n", + " p_value_tot = 1 - stats.chi2.cdf(chi2_tot, n_dof_tot)\n", + "\n", + " metrics[root] = {\n", + " \"chi2_xi_plus\": xi_plus_chi2,\n", + " \"n_dof_xi_plus\": n_dof_xi_plus,\n", + " \"p_value_xi_plus\": p_value_xi_plus,\n", + " \"chi2_xi_minus\": xi_minus_chi2,\n", + " \"n_dof_xi_minus\": n_dof_xi_minus,\n", + " \"p_value_xi_minus\": p_value_xi_minus,\n", + " \"chi2_xi\": xi_chi2,\n", + " \"p_value_xi\": p_value_xi,\n", + " \"chi2_tau\": tau_chi2,\n", + " \"n_dof_tau\": n_dof_tau,\n", + " \"p_value_tau\": p_value_tau,\n", + " \"chi2_tot\": chi2_tot,\n", + " \"n_dof_tot\": n_dof_tot,\n", + " \"p_value_tot\": p_value_tot,\n", + " }\n", + " print(\"Done!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def get_latex_table(metrics):\n", + " latex_lines = [\n", + " r\"\\begin{tabular}{lccc|ccc|ccc}\",\n", + " r\"\\hline\",\n", + " r\"Root & $\\chi^2_{\\xi^+}$/dof & $p_{\\xi^+}$ & $\\chi^2_{\\xi^-}$/dof & $p_{\\xi^+}$ & $\\chi^2_{\\xi}$/dof & $p_{\\xi}$ &\"\n", + " r\"$\\chi^2_\\tau$/dof & $p_\\tau$ & $\\chi^2_{\\text{tot}}$/dof & $p_{\\text{tot}}$ \\\\\",\n", + " r\"\\hline\",\n", + " ]\n", + "\n", + " for root, vals in metrics.items():\n", + " escaped = root.replace(\"_\", r\"\\_\")\n", + " line = (\n", + " f\"{escaped} & \"\n", + " f\"{vals['chi2_xi_plus']:.2f}/{vals['n_dof_xi_plus']} & {vals['p_value_xi_plus']:.3g} & \"\n", + " f\"{vals['chi2_xi_minus']:.2f}/{vals['n_dof_xi_minus']} & {vals['p_value_xi_minus']:.3g} & \"\n", + " f\"{vals['chi2_xi']:.2f}/{vals['n_dof_xi_plus'] + vals['n_dof_xi_minus']} & {vals['p_value_xi']:.3g} &\"\n", + " f\"{vals['chi2_tau']:.2f}/{vals['n_dof_tau']} & {vals['p_value_tau']:.3g} & \"\n", + " f\"{vals['chi2_tot']:.2f}/{vals['n_dof_tot']} & {vals['p_value_tot']:.3g} \\\\\\\\\"\n", + " )\n", + " latex_lines.append(line)\n", + "\n", + " latex_lines.append(r\"\\hline\")\n", + " latex_lines.append(r\"\\end{tabular}\")\n", + "\n", + " # Print LaTeX table\n", + " print(\"\\n\".join(latex_lines))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "get_latex_table(metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def display_markdown(metrics):\n", + " # Build Markdown table\n", + " header = (\n", + " \"| Root | $\\\\chi^2$ (ξ⁺) / dof | p-val (ξ⁺) |$\\\\chi^2$ (ξ-) / dof | p-val (ξ-) | $\\\\chi^2$ (ξ) / dof | p-val (ξ) | $\\\\chi^2$ (τ) / dof | p-val (τ) | $\\\\chi^2$ (tot) / dof | p-val (tot) |\\n\"\n", + " \"|------|----------------|------------|----------------|------------|------------|---------------|------------|------------|------------------|--------------|\\n\"\n", + " )\n", + "\n", + " rows = []\n", + " for root, vals in metrics.items():\n", + " row = f\"| `{root}` \"\n", + " row += f\"| {vals['chi2_xi_plus']:.2f} / {vals['n_dof_xi_plus']} \"\n", + " row += f\"| {vals['p_value_xi_plus']:.5f} \"\n", + " row += f\"| {vals['chi2_xi_minus']:.2f} / {vals['n_dof_xi_minus']} \"\n", + " row += f\"| {vals['p_value_xi_minus']:.5f} \"\n", + " row += f\"| {vals['chi2_xi']:.2f} / {vals['n_dof_xi_minus'] + vals['n_dof_xi_plus']} \"\n", + " row += f\"| {vals['p_value_xi']:.5f} \"\n", + " row += f\"| {vals['chi2_tau']:.2f} / {vals['n_dof_tau']} \"\n", + " row += f\"| {vals['p_value_tau']:.5f} \"\n", + " row += f\"| {vals['chi2_tot']:.2f} / {vals['n_dof_tot']} \"\n", + " row += f\"| {vals['p_value_tot']:.5f} |\"\n", + " rows.append(row)\n", + "\n", + " # Display in Jupyter\n", + " display(Markdown(header + \"\\n\".join(rows)))\n", + " return header + \"\\n\".join(rows)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "markdown_source = display_markdown(metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "my_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2_glass_mock.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2_glass_mock.ipynb new file mode 100644 index 00000000..ddd66cd0 --- /dev/null +++ b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2_glass_mock.ipynb @@ -0,0 +1,565 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import configparser\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "# Make the plot\n", + "import seaborn as sns\n", + "from astropy.io import fits\n", + "from getdist import plots\n", + "from scipy.interpolate import interp1d\n", + "from scipy.stats import chi2\n", + "\n", + "sys.path.append(\"/home/guerrini/sp_validation/cosmo_inference/scripts\")\n", + "\n", + "import chain_postprocessing\n", + "\n", + "%matplotlib inline\n", + "\n", + "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", + "\n", + "plt.rcParams[\"axes.labelsize\"] = 18\n", + "plt.rcParams[\"xtick.labelsize\"] = 18\n", + "plt.rcParams[\"ytick.labelsize\"] = 18\n", + "\n", + "plt.rcParams[\"text.usetex\"] = True\n", + "\n", + "g = plots.get_subplot_plotter(width_inch=30)\n", + "g.settings.axes_fontsize = 30\n", + "g.settings.axes_labelsize = 30\n", + "g.settings.alpha_filled_add = 0.7\n", + "g.settings.legend_fontsize = 40\n", + "\n", + "# #SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE\n", + "\n", + "root_dir = \"/n09data/guerrini/glass_mock_chains/\"\n", + "\n", + "# Version of the glass mock chain run\n", + "chain_version = \"v6\"\n", + "\n", + "# Path to the glass mock data vectors\n", + "root_glass_dv = (\n", + " f\"/home/guerrini/sp_validation/cosmo_inference/data/glass_mocks/{chain_version}/\"\n", + ")\n", + "\n", + "# Choose the best-fit method\n", + "best_fit_method = \"2Dkde\"\n", + "\n", + "# Create the list of mocks\n", + "max_sim = 350\n", + "failed_simulations = [82, 83, 281, 282, 283, 284, 285, 286, 287]\n", + "roots = [f\"glass_mock_{chain_version}_{str(i).zfill(5)}\" for i in range(1, max_sim + 1)]\n", + "roots = [root for root in roots if int(root.split(\"_\")[-1]) not in failed_simulations]\n", + "\n", + "catalog_versions = [\n", + " \"SP_v1.4.6.3_config/SP_v1.4.6.3_A\",\n", + "]\n", + "\n", + "output_folder_chains = \"/n23data1/n06data/lgoh/scratch/temp/\"\n", + "path_ini_files = \"/home/guerrini/sp_validation/cosmo_inference/cosmosis_config/\"\n", + "output_fig_path = (\n", + " \"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/notebooks/Plots/\"\n", + ")\n", + "\n", + "ini_root = \"blind_A/fiducial\"\n", + "\n", + "lower_bound_xi = 12\n", + "upper_bound_xi = 83" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Retrieve the chains" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# READ CHAIN\n", + "\n", + "chains = []\n", + "best_fit = {}\n", + "\n", + "for i, root in enumerate(roots):\n", + " burnin = 0\n", + "\n", + " if os.path.isfile(f\"{root_dir}/{root}/{root}/getdist_{root}.txt\") == True:\n", + " chain = g.samples_for_root(\n", + " f\"{root_dir}/{root}/{root}/getdist_{root}\",\n", + " cache=False,\n", + " settings={\n", + " \"ignore_rows\": burnin,\n", + " \"smooth_scale_2D\": 0.5,\n", + " \"smooth_scale_1D\": 0.5,\n", + " },\n", + " )\n", + " p = chain.getParams()\n", + "\n", + " best_fit[root] = chain_postprocessing.extract_best_fit_params(\n", + " chain, best_fit_method=\"2Dkde\"\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "param_list = [\n", + " \"OMEGA_M\",\n", + " \"ombh2\",\n", + " \"h0\",\n", + " \"n_s\",\n", + " \"SIGMA_8\",\n", + " \"s_8_input\",\n", + " \"logt_agn\",\n", + " \"a\",\n", + " \"m1\",\n", + " \"bias_1\",\n", + " \"alpha\",\n", + " \"beta\",\n", + " \"omch2\",\n", + " \"m\",\n", + " \"a_planck\",\n", + "]\n", + "label_list = [\n", + " r\"\\Omega_m\",\n", + " r\"\\omega_b\",\n", + " \"h_0\",\n", + " \"n_s\",\n", + " r\"\\sigma_8\",\n", + " \"S_8\",\n", + " \"log T_{AGN}\",\n", + " \"A_{IA}\",\n", + " \"m_1\",\n", + " r\"\\Delta z_1\",\n", + " \"\\\\alpha_{PSF}\",\n", + " \"\\\\beta_{PSF}\",\n", + " r\"\\omega_c\",\n", + " \"M\",\n", + " \"A_{\\rm Planck}\",\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run `Cosmosis` in test mode to get the data vectors" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if not os.path.exists(path_ini_files + \"/values_empty.ini\"):\n", + " content = \"\"\"[cosmological_parameters]\n", + "\n", + "tau = 0.0544\n", + "w = -1.0\n", + "mnu = 0.06\n", + "omega_k = 0.0\n", + "wa = 0.0\n", + "\n", + "[halo_model_parameters]\n", + "\n", + "[intrinsic_alignment_parameters]\n", + "\n", + "[shear_calibration_parameters]\n", + "\n", + "[nofz_shifts]\n", + "\n", + "[psf_leakage_parameters]\n", + "\"\"\"\n", + "\n", + " with open(path_ini_files + \"/values_empty.ini\", \"w\") as f:\n", + " f.write(content)\n", + " f.close()\n", + "\n", + " print(\"File created successfully\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "section_map = {\n", + " \"omch2\": \"cosmological_parameters\",\n", + " \"ombh2\": \"cosmological_parameters\",\n", + " \"h0\": \"cosmological_parameters\",\n", + " \"n_s\": \"cosmological_parameters\",\n", + " \"s_8_input\": \"cosmological_parameters\",\n", + " \"logt_agn\": \"halo_model_parameters\",\n", + " \"a\": \"intrinsic_alignment_parameters\",\n", + " \"m1\": \"shear_calibration_parameters\",\n", + " \"bias_1\": \"nofz_shifts\",\n", + " \"alpha\": \"psf_leakage_parameters\",\n", + " \"beta\": \"psf_leakage_parameters\",\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "env = os.environ.copy()\n", + "env[\"LD_LIBRARY_PATH\"] = (\n", + " \"/home/guerrini/.conda/envs/sp_validation/lib/python3.9/site-packages/cosmosis/datablock:\"\n", + " + env.get(\"LD_LIBRARY_PATH\", \"\")\n", + ")\n", + "for i, root in enumerate(roots):\n", + " print(root)\n", + " config = configparser.ConfigParser()\n", + " config.optionxform = str # Preserve case sensitivity of option names\n", + "\n", + " for param, section in section_map.items():\n", + " # Check if this parameter exists for the current root\n", + " if param in best_fit[root]:\n", + " value = best_fit[root][param]\n", + "\n", + " if section not in config:\n", + " config.add_section(section)\n", + "\n", + " config[section][param] = str(value)\n", + "\n", + " with open(path_ini_files + \"/values_empty.ini\", \"w\") as configfile:\n", + " config.write(configfile)\n", + "\n", + " # Modify the ini file to run in test mode at the best fit\n", + " config = configparser.ConfigParser()\n", + " config.optionxform = str # Preserve case sensitivity of option names\n", + "\n", + " ini_file = (\n", + " path_ini_files + f\"config_space_v1.4.6.3_fiducial/pipeline/{ini_root}.ini\"\n", + " )\n", + " config.read(ini_file)\n", + "\n", + " sampler = config[\"runtime\"][\"sampler\"]\n", + " config[\"runtime\"][\"sampler\"] = \"test\"\n", + " values = config[\"pipeline\"][\"values\"]\n", + " config[\"pipeline\"][\"values\"] = path_ini_files + \"/values_empty.ini\"\n", + " config[\"DEFAULT\"][\"FITS_FILE\"] = (\n", + " f\"{root_glass_dv}/glass_mock_{root[-5:]}/cosmosis_glass_mock_v6_{root[-5:]}.fits\"\n", + " )\n", + " config[\"test\"][\"save_dir\"] = output_folder_chains + f\"{root}/best_fit_config\"\n", + "\n", + " with open(ini_file, \"w\") as configfile:\n", + " config.write(configfile)\n", + "\n", + " # Run cosmosis\n", + " result = subprocess.run(\n", + " [\"cosmosis\", ini_file], env=env, capture_output=True, text=True\n", + " )\n", + " # print(f\"STDOUT:\\n{result.stdout}\")\n", + " # print(f\"STDERR:\\n{result.stderr}\")\n", + "\n", + " # Modify the ini file to the previous one\n", + " config[\"pipeline\"][\"values\"] = values\n", + " config[\"runtime\"][\"sampler\"] = sampler\n", + "\n", + " with open(ini_file, \"w\") as configfile:\n", + " config.write(configfile)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "xi_plus_chi2s = np.array([])\n", + "xi_minus_chi2s = np.array([])\n", + "xi_chi2s = np.array([])\n", + "tau_chi2s = np.array([])\n", + "chi2_tots = np.array([])\n", + "\n", + "\n", + "for idx, root in enumerate(roots):\n", + " print(root)\n", + "\n", + " data = fits.open(\n", + " f\"{root_glass_dv}/glass_mock_{root[-5:]}/cosmosis_glass_mock_v6_{root[-5:]}.fits\"\n", + " )\n", + "\n", + " tau_0_data = data[\"TAU_0_PLUS\"].data[\"VALUE\"]\n", + " tau_2_data = data[\"TAU_2_PLUS\"].data[\"VALUE\"]\n", + "\n", + " theta_data = data[\"XI_PLUS\"].data[\"ANG\"]\n", + " xi_plus_data = data[\"XI_PLUS\"].data[\"VALUE\"]\n", + " xi_minus_data = data[\"XI_MINUS\"].data[\"VALUE\"]\n", + " xi_data = np.concatenate((xi_plus_data, xi_minus_data))\n", + "\n", + " tau_data = np.concatenate((tau_0_data, tau_2_data))\n", + "\n", + " # Apply scale cuts\n", + " mask_xi_plus = (theta_data > lower_bound_xi) & (theta_data < upper_bound_xi)\n", + " mask_xi_minus = (theta_data > lower_bound_xi) & (theta_data < upper_bound_xi)\n", + " mask = np.concatenate((mask_xi_plus, mask_xi_minus))\n", + " # Load the covariance\n", + " cov = data[\"COVMAT\"].data\n", + " cov_xi = cov[0 : 2 * len(xi_plus_data), 0 : 2 * len(xi_plus_data)]\n", + " cov_tau = cov[\n", + " 2 * len(xi_plus_data) : 4 * len(xi_plus_data),\n", + " 2 * len(xi_plus_data) : 4 * len(xi_plus_data),\n", + " ]\n", + " xi_data = xi_data[mask]\n", + " cov_xi = cov_xi[mask][:, mask]\n", + "\n", + " cov_xi_plus = cov[0 : len(xi_plus_data), 0 : len(xi_plus_data)]\n", + " cov_xi_plus = cov_xi_plus[mask_xi_plus][:, mask_xi_plus]\n", + " cov_xi_minus = cov[\n", + " len(xi_plus_data) : 2 * len(xi_minus_data),\n", + " len(xi_plus_data) : 2 * len(xi_minus_data),\n", + " ]\n", + " cov_xi_minus = cov_xi_minus[mask_xi_minus][:, mask_xi_minus]\n", + "\n", + " # Read the results\n", + " theta = np.loadtxt(\n", + " output_folder_chains + f\"{root}/best_fit_config/shear_xi_plus/theta.txt\"\n", + " )\n", + " theta_arcmin = theta * 180 * 60 / np.pi\n", + " shear_xi_plus = np.loadtxt(\n", + " output_folder_chains + f\"{root}/best_fit_config/shear_xi_plus/bin_1_1.txt\"\n", + " )\n", + " shear_xi_minus = np.loadtxt(\n", + " output_folder_chains + f\"{root}/best_fit_config/shear_xi_minus/bin_1_1.txt\"\n", + " )\n", + "\n", + " xi_sys_plus = np.loadtxt(\n", + " output_folder_chains + f\"{root}/best_fit_config/xi_sys/shear_xi_plus.txt\"\n", + " )\n", + " xi_sys_minus = np.loadtxt(\n", + " output_folder_chains + f\"{root}/best_fit_config/xi_sys/shear_xi_minus.txt\"\n", + " )\n", + "\n", + " theta_tau = np.loadtxt(\n", + " output_folder_chains + f\"{root}/best_fit_config/tau_0_plus/theta.txt\"\n", + " )\n", + " theta_tau_arcmin = theta_tau * 180 * 60 / np.pi\n", + " tau_0_model = np.loadtxt(\n", + " output_folder_chains + f\"{root}/best_fit_config/tau_0_plus/bin_1_1.txt\"\n", + " )\n", + " tau_2_model = np.loadtxt(\n", + " output_folder_chains + f\"{root}/best_fit_config/tau_2_plus/bin_1_1.txt\"\n", + " )\n", + "\n", + " # interpolate the model\n", + " interp_xi_plus = interp1d(\n", + " theta_arcmin, shear_xi_plus, kind=\"cubic\", fill_value=\"extrapolate\"\n", + " )\n", + " interp_xi_minus = interp1d(\n", + " theta_arcmin, shear_xi_minus, kind=\"cubic\", fill_value=\"extrapolate\"\n", + " )\n", + "\n", + " xi_plus_model = interp_xi_plus(theta_data)\n", + " xi_plus_model += xi_sys_plus\n", + " xi_minus_model = interp_xi_minus(theta_data)\n", + " xi_minus_model += xi_sys_minus\n", + "\n", + " xi_model = np.concatenate((xi_plus_model, xi_minus_model))\n", + " tau_model = np.concatenate((tau_0_model, tau_2_model))\n", + " xi_model = xi_model[mask]\n", + "\n", + " xi_plus_chi2 = np.dot(\n", + " (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]),\n", + " np.dot(\n", + " np.linalg.inv(cov_xi_plus),\n", + " (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]),\n", + " ),\n", + " )\n", + " xi_minus_chi2 = np.dot(\n", + " (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]),\n", + " np.dot(\n", + " np.linalg.inv(cov_xi_minus),\n", + " (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]),\n", + " ),\n", + " )\n", + " xi_chi2 = np.dot(\n", + " (xi_model - xi_data), np.dot(np.linalg.inv(cov_xi), (xi_model - xi_data))\n", + " )\n", + " tau_chi2 = np.dot(\n", + " (tau_model - tau_data), np.dot(np.linalg.inv(cov_tau), (tau_model - tau_data))\n", + " )\n", + " chi2_tot = xi_plus_chi2 + xi_minus_chi2 + tau_chi2\n", + "\n", + " xi_plus_chi2s = np.append(xi_plus_chi2s, xi_plus_chi2)\n", + " xi_minus_chi2s = np.append(xi_minus_chi2s, xi_minus_chi2)\n", + " xi_chi2s = np.append(xi_chi2s, xi_chi2)\n", + " tau_chi2s = np.append(tau_chi2s, tau_chi2)\n", + " chi2_tots = np.append(chi2_tots, chi2_tot)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig, [ax1, ax2] = plt.subplots(2, 1, figsize=(7, 10))\n", + "chi2_fiducial = -2 * -37.560916821678894\n", + "dof, loc, scale = chi2.fit(chi2_tots, floc=0)\n", + "\n", + "print(f\"Best-fit dof: {dof:.3e}\")\n", + "counts, bin_edges = np.histogram(chi2_tots, bins=25, density=True)\n", + "\n", + "sns.histplot(\n", + " chi2_tots,\n", + " ax=ax1,\n", + " kde=False,\n", + " bins=bin_edges,\n", + " stat=\"density\",\n", + " label=r\"$\\chi^2$ for \\texttt{GLASS} mocks best-fits\",\n", + " color=\"green\",\n", + " alpha=0.3,\n", + ")\n", + "\n", + "# Compute the p-value\n", + "\n", + "# 1. Get in which bin the chi2 of the fiducial falls\n", + "bin_index = np.digitize(chi2_fiducial, bin_edges)\n", + "\n", + "# 2. Compute the p-value as the integral of the tail of the histogram\n", + "p_value = np.sum(counts[bin_index:]) * np.diff(bin_edges)[0]\n", + "\n", + "print(f\"P-value: {p_value}\")\n", + "\n", + "ax1.axvline(chi2_fiducial, color=\"red\", label=r\"$\\chi^2$ of the fiducial\", lw=2)\n", + "\n", + "mantissa, exponent = np.frexp(p_value)\n", + "pte_string = rf\"${{\\rm PTE}} = {p_value:.4f}$\"\n", + "print(f\"mantissa: {mantissa}, exponent: {exponent}\")\n", + "x_text = 78\n", + "y_text = max(counts) * 0.95\n", + "ax1.text(\n", + " x_text,\n", + " y_text,\n", + " pte_string,\n", + " fontsize=15,\n", + " bbox=dict(facecolor=\"wheat\", alpha=0.8, edgecolor=\"black\"),\n", + ")\n", + "\n", + "chi2_string = rf\"${{\\rm Eff. dof}}= {dof:.1f}$\"\n", + "y_text = max(counts) * 0.85\n", + "ax1.text(\n", + " x_text,\n", + " y_text,\n", + " chi2_string,\n", + " fontsize=15,\n", + " bbox=dict(facecolor=\"wheat\", alpha=0.8, edgecolor=\"black\"),\n", + ")\n", + "\n", + "ax1.set_xlabel(r\"$\\chi^2_{\\rm tot}$\")\n", + "ax1.set_ylabel(\"Density\")\n", + "\n", + "chi2_fiducial = 9.5\n", + "dof, loc, scale = chi2.fit(xi_chi2s, floc=0)\n", + "\n", + "print(f\"Best-fit dof: {dof:.3e}\")\n", + "counts, bin_edges = np.histogram(xi_chi2s, bins=25, density=True)\n", + "\n", + "sns.histplot(\n", + " xi_chi2s,\n", + " ax=ax2,\n", + " kde=False,\n", + " bins=bin_edges,\n", + " stat=\"density\",\n", + " label=r\"$\\chi^2$ for \\texttt{GLASS} mocks best-fits\",\n", + " color=\"pink\",\n", + " alpha=0.5,\n", + ")\n", + "\n", + "# Compute the p-value\n", + "\n", + "# 1. Get in which bin the chi2 of the fiducial falls\n", + "bin_index = np.digitize(chi2_fiducial, bin_edges)\n", + "\n", + "# 2. Compute the p-value as the integral of the tail of the histogram\n", + "p_value = np.sum(counts[bin_index:]) * np.diff(bin_edges)[0]\n", + "\n", + "print(f\"P-value: {p_value}\")\n", + "\n", + "ax2.axvline(chi2_fiducial, color=\"red\", label=r\"$\\chi^2$ of the fiducial\", lw=2)\n", + "\n", + "mantissa, exponent = np.frexp(p_value)\n", + "print(f\"mantissa: {mantissa}, exponent: {exponent}\")\n", + "pte_string = rf\"${{\\rm PTE}} = {p_value:.4f}$\"\n", + "# rf\"${{\\rm PTE}} = {mantissa:.2f} \\times 10^{{{exponent}}}$\" if exponent != 0 else\n", + "x_text = 17.5\n", + "y_text = max(counts) * 0.95\n", + "ax2.text(\n", + " x_text,\n", + " y_text,\n", + " pte_string,\n", + " fontsize=15,\n", + " bbox=dict(facecolor=\"wheat\", alpha=0.8, edgecolor=\"black\"),\n", + ")\n", + "\n", + "chi2_string = rf\"${{\\rm Eff. dof}}= {dof:.1f}$\"\n", + "y_text = max(counts) * 0.85\n", + "ax2.text(\n", + " x_text,\n", + " y_text,\n", + " chi2_string,\n", + " fontsize=15,\n", + " bbox=dict(facecolor=\"wheat\", alpha=0.8, edgecolor=\"black\"),\n", + ")\n", + "\n", + "ax2.set_xlabel(r\"$\\chi^2 (\\xi_\\pm)$\")\n", + "ax2.set_ylabel(\"Density\")\n", + "fig.savefig(f\"{output_fig_path}/chi2_glass_mocks_p_value_xi_tau.pdf\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "my_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_prior_psf_leakage.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_prior_psf_leakage.ipynb new file mode 100644 index 00000000..ad6f5fd1 --- /dev/null +++ b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_prior_psf_leakage.ipynb @@ -0,0 +1,261 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Covariance matrix and PSF leakage\n", + "\n", + "This notebook plots the combined covariance matrix, and samples and plots the 2D marginalised posteriors of the PSF leakage parameters $\\alpha$ and $\\beta$." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "if not os.path.exists(\"./Plots\"):\n", + " os.makedirs(\"./Plots\")\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import seaborn as sns\n", + "from astropy.io import fits\n", + "from getdist import MCSamples, plots\n", + "from shear_psf_leakage.rho_tau_stat import PSFErrorFit, RhoStat, TauStat\n", + "\n", + "# Use paper style and seaborn with husl palette\n", + "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", + "# Set default palette - will be updated per plot as needed\n", + "sns.set_palette(\"husl\")\n", + "%matplotlib inline\n", + "\n", + "g = plots.get_subplot_plotter(width_inch=30)\n", + "g.settings.axes_fontsize = 30\n", + "g.settings.axes_labelsize = 30\n", + "g.settings.alpha_filled_add = 0.7\n", + "g.settings.legend_fontsize = 25\n", + "\n", + "ver = \"v1.4.6.3\"\n", + "blind = \"B\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "data_path = f\"/home/guerrini/sp_validation/cosmo_inference/data/SP_{ver}_config/\"\n", + "\n", + "path_cosmo_val = \"/home/guerrini/sp_validation/cosmo_val/output/\"\n", + "\n", + "roots = [f\"SP_{ver}_{blind}\", f\"SP_{ver}_leak_corr_{blind}\"]\n", + "\n", + "labels = [f\"SP_{ver}_{blind}\", f\"SP_{ver}_leak_corr_{blind}\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "data_vectors = []\n", + "\n", + "for root in roots:\n", + " data_vectors.append(\n", + " fits.open(data_path + f\"SP_{ver}_{blind}/cosmosis_{root}_masked.fits\")\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "def cov_to_corr(cov):\n", + " \"\"\"Convert a covariance matrix to a correlation matrix.\"\"\"\n", + " d = np.sqrt(np.diag(cov))\n", + " corr = cov / np.outer(d, d)\n", + " corr[cov == 0] = 0\n", + " return corr" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# Print the covariance matrix for each root\n", + "for i, root in enumerate(roots):\n", + " print(f\"Covariance matrix for {labels[i]}:\")\n", + " cov = data_vectors[i][\"COVMAT\"].data\n", + "\n", + " n_bins = cov.shape[0] // 4\n", + "\n", + " fig, ax = plt.subplots(figsize=(10, 8))\n", + "\n", + " im = ax.imshow(cov_to_corr(cov), vmin=-1, vmax=1, cmap=\"seismic\")\n", + " ax.set_aspect(\"equal\")\n", + " ax.set_yticks(np.array([10, 30, 50, 70]))\n", + " ax.set_yticklabels(\n", + " [\n", + " r\"$\\xi_+(\\vartheta)$\",\n", + " r\"$\\xi_-(\\vartheta)$\",\n", + " r\"$\\tau_0(\\vartheta)$\",\n", + " r\"$\\tau_2(\\vartheta)$\",\n", + " ]\n", + " )\n", + " ax.set_xticks(np.array([10, 30, 50, 70]))\n", + " ax.set_xticklabels(\n", + " [\n", + " r\"$\\xi_+(\\vartheta)$\",\n", + " r\"$\\xi_-(\\vartheta)$\",\n", + " r\"$\\tau_0(\\vartheta)$\",\n", + " r\"$\\tau_2(\\vartheta)$\",\n", + " ],\n", + " rotation=45,\n", + " )\n", + " fig.colorbar(im, ax=ax)\n", + "\n", + " plt.savefig(f\"./Plots/cov_matrix_{root}.png\", bbox_inches=\"tight\", dpi=300)\n", + " plt.show()\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# Create dummy rho and tau stat handler.\n", + "\n", + "# Inference of the xi_sys parameters\n", + "sep_units = \"arcmin\"\n", + "coord_units = \"degrees\"\n", + "theta_min = 1.0\n", + "theta_max = 250\n", + "nbins = 20\n", + "\n", + "\n", + "TreeCorrConfig_xi = {\n", + " \"ra_units\": coord_units,\n", + " \"dec_units\": coord_units,\n", + " \"min_sep\": theta_min,\n", + " \"max_sep\": theta_max,\n", + " \"sep_units\": sep_units,\n", + " \"nbins\": nbins,\n", + " \"var_method\": \"jackknife\",\n", + "}\n", + "\n", + "rho_stats_handler = RhoStat(output=\".\", treecorr_config=TreeCorrConfig_xi, verbose=True)\n", + "\n", + "tau_stats_handler = TauStat(\n", + " catalogs=rho_stats_handler.catalogs,\n", + " output=\".\",\n", + " treecorr_config=TreeCorrConfig_xi,\n", + " verbose=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "# Create a PSFErrorFit instance\n", + "psf_fitter = PSFErrorFit(\n", + " rho_stats_handler,\n", + " tau_stats_handler,\n", + " path_cosmo_val + \"rho_tau_stats/\",\n", + " use_eta=False,\n", + ")\n", + "\n", + "g = plots.get_subplot_plotter(width_inch=30)\n", + "\n", + "g.settings.axes_fontsize = 30\n", + "g.settings.axes_labelsize = 30\n", + "g.settings.alpha_filled_add = 0.7\n", + "g.settings.legend_fontsize = 40\n", + "\n", + "chains = []\n", + "\n", + "# Load rho-, tau-statistics, and cov_tau from the data_vector\n", + "for i, root in enumerate(roots):\n", + " print(\"Sampling PSF parameters for \", labels[i])\n", + " path_rho = f\"rho_stats_{root}.fits\"\n", + " path_tau = f\"tau_stats_{root}.fits\"\n", + " path_cov_rho = f\"cov_rho_{root}.npy\"\n", + " path_cov_tau = f\"cov_tau_{root}_th.npy\"\n", + " psf_fitter.load_rho_stat(path_rho)\n", + " psf_fitter.load_tau_stat(path_tau)\n", + " psf_fitter.load_covariance(path_cov_rho, cov_type=\"rho\")\n", + " psf_fitter.load_covariance(path_cov_tau, cov_type=\"tau\")\n", + " samples_lq, _, _ = psf_fitter.get_least_squares_params_samples(\n", + " npatch=None, apply_debias=False\n", + " )\n", + "\n", + " samples_gd = MCSamples(\n", + " samples=samples_lq, names=[r\"\\alpha\", r\"\\beta\"], labels=[r\"\\alpha\", r\"\\beta\"]\n", + " )\n", + "\n", + " chains.append(samples_gd)\n", + "\n", + "g.triangle_plot(\n", + " chains,\n", + " filled=True,\n", + " legend_labels=labels,\n", + " legend_loc=\"upper right\",\n", + ")\n", + "\n", + "# plt.savefig(f\"./Plots/psf_leakage_params.png\", bbox_inches='tight', dpi=300)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "my_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/glass_mock_hist.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/glass_mock_hist.ipynb new file mode 100644 index 00000000..32f89a18 --- /dev/null +++ b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/glass_mock_hist.ipynb @@ -0,0 +1,586 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "import IPython\n", + "\n", + "ipython = IPython.get_ipython()\n", + "\n", + "if ipython is not None:\n", + " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", + " ipython.run_line_magic(\"autoreload\", \"2\")\n", + "\n", + "import os\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import seaborn as sns\n", + "from getdist import plots\n", + "from tqdm import tqdm\n", + "\n", + "g = plots.get_subplot_plotter(width_inch=7)\n", + "g.settings.axes_fontsize = 15\n", + "g.settings.axes_labelsize = 15\n", + "g.settings.alpha_filled_add = 0.7\n", + "g.settings.legend_fontsize = 15\n", + "\n", + "if os.path.exists(\"/home/guerrini/matplotlib_config/paper.mplstyle\"):\n", + " plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", + "\n", + "# Set default palette - will be updated per plot as needed\n", + "sns.set_palette(\"husl\")\n", + "\n", + "if ipython is not None:\n", + " ipython.run_line_magic(\"matplotlib\", \"inline\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": { + "lines_to_next_cell": 1 + }, + "outputs": [], + "source": [ + "root_dir = \"/n09data/guerrini/glass_mock_chains/\"\n", + "chain_version = \"v6\"\n", + "num_sims = 350\n", + "\n", + "roots = [f\"glass_mock_{chain_version}_{i + 1:05d}\" for i in range(num_sims)]\n", + "\n", + "\n", + "# # %%\n", + "def load_samples_and_write_paramames(root_dir, root, chain_type=\"configuration\"):\n", + " assert chain_type in [\"configuration\", \"harmonic\"], (\n", + " \"chain_type must be 'configuration' or 'harmonic'\"\n", + " )\n", + "\n", + " if chain_type == \"configuration\":\n", + " path_samples = root_dir + \"{}/{}/samples_{}.txt\".format(\"/\" + root, root, root)\n", + " path_paramnames = root_dir + \"{}/{}/getdist_{}.paramnames\".format(\n", + " \"/\" + root, root, root\n", + " )\n", + " else:\n", + " path_samples = root_dir + \"{}/{}/samples_{}_cell.txt\".format(\n", + " \"/\" + root, root, root\n", + " )\n", + " path_paramnames = root_dir + \"{}/{}/getdist_{}_cell.paramnames\".format(\n", + " \"/\" + root, root, root\n", + " )\n", + "\n", + " with open(path_samples, \"r\") as file:\n", + " params = file.readline()[1:].split(\"\\t\")[:-4]\n", + " file.close()\n", + "\n", + " with open(path_paramnames, \"w\") as file:\n", + " for i in range(len(params)):\n", + " if len(params[i].split(\"--\")) > 1:\n", + " file.write(params[i].split(\"--\")[1] + \"\\n\")\n", + " else:\n", + " file.write(params[i].split(\"--\")[0] + \"\\n\")\n", + " file.close()\n", + "\n", + "\n", + "def write_samples_getdist_format(root_dir, root, chain_type=\"configuration\"):\n", + " assert chain_type in [\"configuration\", \"harmonic\"], (\n", + " \"chain_type must be 'configuration' or 'harmonic'\"\n", + " )\n", + "\n", + " if chain_type == \"configuration\":\n", + " path_samples = root_dir + \"{}/{}/samples_{}.txt\".format(\"/\" + root, root, root)\n", + " path_gd_samples = root_dir + \"{}/{}/getdist_{}.txt\".format(\n", + " \"/\" + root, root, root\n", + " )\n", + " path_gd = root_dir + \"{}/{}/getdist_{}\".format(root, root, root)\n", + " else:\n", + " path_samples = root_dir + \"{}/{}/samples_{}_cell.txt\".format(\n", + " \"/\" + root, root, root\n", + " )\n", + " path_gd_samples = root_dir + \"{}/{}/getdist_{}_cell.txt\".format(\n", + " \"/\" + root, root, root\n", + " )\n", + " path_gd = root_dir + \"{}/{}/getdist_{}_cell\".format(root, root, root)\n", + "\n", + " samples = np.loadtxt(\n", + " path_samples,\n", + " )\n", + " if \"nautilus\" in root:\n", + " samples = np.column_stack(\n", + " (np.exp(samples[:, -3]), samples[:, -1] - samples[:, -2], samples[:, 0:-3])\n", + " )\n", + " else:\n", + " samples = np.column_stack((samples[:, -1], samples[:, -2], samples[:, 0:-4]))\n", + " np.savetxt(path_gd_samples, samples)\n", + "\n", + " chain = g.samples_for_root(\n", + " path_gd,\n", + " cache=False,\n", + " settings={\"ignore_rows\": 0.0, \"smooth_scale_2D\": 0.5, \"smooth_scale_1D\": 0.5},\n", + " )\n", + "\n", + " return chain\n", + "\n", + "\n", + "def extract_param_chain(chain, param_names):\n", + " margestats = chain.getMargeStats()\n", + " likestats = chain.getLikeStats()\n", + "\n", + " param_values = {}\n", + " for param_name in param_names:\n", + " if param_name not in chain.getParamNames().list():\n", + " raise ValueError(f\"Parameter {param_name} not found in chain.\")\n", + "\n", + " param_stats = margestats.parWithName(param_name)\n", + " param_values[param_name] = {\n", + " \"mean\": param_stats.mean,\n", + " \"1sigma_minus\": param_stats.mean - param_stats.limits[0].lower,\n", + " \"1sigma_plus\": param_stats.limits[0].upper - param_stats.mean,\n", + " \"2sigma_minus\": param_stats.mean - param_stats.limits[1].lower,\n", + " \"2sigma_plus\": param_stats.limits[1].upper - param_stats.mean,\n", + " }\n", + "\n", + " param_stats = likestats.parWithName(param_name)\n", + " param_names_getdist = chain.getParamNames()\n", + " par = param_names_getdist.parWithName(param_name)\n", + " kde = chain.get1DDensity(par, num_bins=1000)\n", + " kde_map = kde.x[np.argmax(kde.P)]\n", + " param_values[param_name].update(\n", + " {\n", + " \"MAP\": kde_map,\n", + " }\n", + " )\n", + "\n", + " par = chain.getParamNames().parWithName(\"S_8\")\n", + " par_om = chain.getParamNames().parWithName(\"OMEGA_M\")\n", + " kde = chain.get2DDensity(par, par_om, fine_bins_2D=1000)\n", + " s8_kde_map = kde.x[np.unravel_index(np.argmax(kde.P), kde.P.shape)[1]]\n", + " om_kde_map = kde.y[np.unravel_index(np.argmax(kde.P), kde.P.shape)[0]]\n", + " param_values[\"S_8\"].update(\n", + " {\n", + " \"MAP_2D\": s8_kde_map,\n", + " }\n", + " )\n", + " param_values[\"OMEGA_M\"].update(\n", + " {\n", + " \"MAP_2D\": om_kde_map,\n", + " }\n", + " )\n", + "\n", + " return param_values\n", + "\n", + "\n", + "def concatenate_param_stats(name, param_values, verbose=False):\n", + " output = [name]\n", + " for key in param_values.keys():\n", + " param_stat = param_values[key]\n", + " if verbose:\n", + " print(\n", + " f\"{name} - {key}: {param_stat['mean']:.4f} +{param_stat['1sigma_plus']:.4f}/-{param_stat['1sigma_minus']:.4f} (1σ), +{param_stat['2sigma_plus']:.4f}/-{param_stat['2sigma_minus']:.4f} (2σ)\"\n", + " )\n", + "\n", + " param_list = [\n", + " param_stat[\"mean\"],\n", + " param_stat[\"1sigma_minus\"],\n", + " param_stat[\"1sigma_plus\"],\n", + " param_stat[\"2sigma_minus\"],\n", + " param_stat[\"2sigma_plus\"],\n", + " param_stat[\"MAP\"],\n", + " ]\n", + "\n", + " if key == \"S_8\":\n", + " param_list.append(param_stat[\"MAP_2D\"])\n", + "\n", + " if key == \"OMEGA_M\":\n", + " param_list.append(param_stat[\"MAP_2D\"])\n", + "\n", + " output += param_list\n", + "\n", + " return output\n", + "\n", + "\n", + "def merge_param_stats(params_configuration, params_harmonic):\n", + " merged_params = {}\n", + " for key in params_configuration.keys():\n", + " if key in params_harmonic:\n", + " merged_params[key] = {\n", + " \"configuration\": params_configuration[key],\n", + " \"harmonic\": params_harmonic[key],\n", + " }\n", + " return merged_params\n", + "\n", + "\n", + "def concatenate_merge_params(name, merged_params, verbose=False):\n", + " output = [name]\n", + " for key in merged_params.keys():\n", + " param_config = merged_params[key][\"configuration\"]\n", + " param_harm = merged_params[key][\"harmonic\"]\n", + "\n", + " if verbose:\n", + " print(\n", + " f\"{name} - {key} (Configuration): {param_config['mean']:.4f} +{param_config['1sigma_plus']:.4f}/-{param_config['1sigma_minus']:.4f} (1σ), +{param_config['2sigma_plus']:.4f}/-{param_config['2sigma_minus']:.4f} (2σ)\"\n", + " )\n", + " print(\n", + " f\"{name} - {key} (Harmonic): {param_harm['mean']:.4f} +{param_harm['1sigma_plus']:.4f}/-{param_harm['1sigma_minus']:.4f} (1σ), +{param_harm['2sigma_plus']:.4f}/-{param_harm['2sigma_minus']:.4f} (2σ)\"\n", + " )\n", + "\n", + " param_list = [\n", + " param_config[\"mean\"],\n", + " param_config[\"1sigma_minus\"],\n", + " param_config[\"1sigma_plus\"],\n", + " param_config[\"2sigma_minus\"],\n", + " param_config[\"2sigma_plus\"],\n", + " param_config[\"MAP\"],\n", + " param_harm[\"mean\"],\n", + " param_harm[\"1sigma_minus\"],\n", + " param_harm[\"1sigma_plus\"],\n", + " param_harm[\"2sigma_minus\"],\n", + " param_harm[\"2sigma_plus\"],\n", + " param_harm[\"MAP\"],\n", + " ]\n", + "\n", + " output += param_list\n", + "\n", + " return output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": { + "lines_to_next_cell": 0 + }, + "outputs": [], + "source": [ + "chain_harmonic = []\n", + "chain_config = []\n", + "\n", + "for i, root in enumerate(tqdm(roots)):\n", + " if os.path.isfile(f\"{root_dir}/{root}/{root}/getdist_{root}.txt\"):\n", + " # Load samples and write paramnames for harmonic space\n", + " load_samples_and_write_paramames(root_dir, root, chain_type=\"harmonic\")\n", + " write_samples_getdist_format(root_dir, root, chain_type=\"harmonic\")\n", + " chain_harm = g.samples_for_root(\n", + " root_dir + f\"/{root}/{root}/getdist_{root}_cell\",\n", + " cache=False,\n", + " settings={\n", + " \"ignore_rows\": 0.0,\n", + " \"smooth_scale_2D\": 0.5,\n", + " \"smooth_scale_1D\": 0.5,\n", + " },\n", + " )\n", + " chain_harmonic.append(chain_harm)\n", + "\n", + " # Load samples and write paramnames for harmonic space\n", + " load_samples_and_write_paramames(root_dir, root, chain_type=\"configuration\")\n", + " write_samples_getdist_format(root_dir, root, chain_type=\"configuration\")\n", + " chain_conf = g.samples_for_root(\n", + " root_dir + f\"/{root}/{root}/getdist_{root}\",\n", + " cache=False,\n", + " settings={\n", + " \"ignore_rows\": 0.0,\n", + " \"smooth_scale_2D\": 0.5,\n", + " \"smooth_scale_1D\": 0.5,\n", + " },\n", + " )\n", + " chain_config.append(chain_conf)\n", + "# # %%\n", + "param_names = [\"S_8\", \"OMEGA_M\", \"SIGMA_8\", \"a\"]\n", + "\n", + "output_mocks_harm = np.array(\n", + " [\n", + " \"Name\",\n", + " \"S8_mean\",\n", + " \"S8_1sigma_minus\",\n", + " \"S8_1sigma_plus\",\n", + " \"S8_2sigma_minus\",\n", + " \"S8_2sigma_plus\",\n", + " \"S8_MAP\",\n", + " \"S8_MAP_2D\",\n", + " \"OMEGA_M_mean\",\n", + " \"OMEGA_M_1sigma_minus\",\n", + " \"OMEGA_M_1sigma_plus\",\n", + " \"OMEGA_M_2sigma_minus\",\n", + " \"OMEGA_M_2sigma_plus\",\n", + " \"OMEGA_M_MAP\",\n", + " \"OMEGA_M_MAP_2D\",\n", + " \"SIGMA_8_mean\",\n", + " \"SIGMA_8_1sigma_minus\",\n", + " \"SIGMA_8_1sigma_plus\",\n", + " \"SIGMA_8_2sigma_minus\",\n", + " \"SIGMA_8_2sigma_plus\",\n", + " \"SIGMA_8_MAP\",\n", + " \"a_mean\",\n", + " \"a_1sigma_minus\",\n", + " \"a_1sigma_plus\",\n", + " \"a_2sigma_minus\",\n", + " \"a_2sigma_plus\",\n", + " \"a_MAP\",\n", + " ]\n", + ")\n", + "\n", + "output_mocks_config = np.array(\n", + " [\n", + " \"Name\",\n", + " \"S8_mean\",\n", + " \"S8_1sigma_minus\",\n", + " \"S8_1sigma_plus\",\n", + " \"S8_2sigma_minus\",\n", + " \"S8_2sigma_plus\",\n", + " \"S8_MAP\",\n", + " \"S8_MAP_2D\",\n", + " \"OMEGA_M_mean\",\n", + " \"OMEGA_M_1sigma_minus\",\n", + " \"OMEGA_M_1sigma_plus\",\n", + " \"OMEGA_M_2sigma_minus\",\n", + " \"OMEGA_M_2sigma_plus\",\n", + " \"OMEGA_M_MAP\",\n", + " \"OMEGA_M_MAP_2D\",\n", + " \"SIGMA_8_mean\",\n", + " \"SIGMA_8_1sigma_minus\",\n", + " \"SIGMA_8_1sigma_plus\",\n", + " \"SIGMA_8_2sigma_minus\",\n", + " \"SIGMA_8_2sigma_plus\",\n", + " \"SIGMA_8_MAP\",\n", + " \"a_mean\",\n", + " \"a_1sigma_minus\",\n", + " \"a_1sigma_plus\",\n", + " \"a_2sigma_minus\",\n", + " \"a_2sigma_plus\",\n", + " \"a_MAP\",\n", + " ]\n", + ")\n", + "\n", + "for i, root in enumerate(tqdm(roots[:-1])):\n", + " param_values_harm = extract_param_chain(chain_harmonic[i], param_names)\n", + "\n", + " param_harm = concatenate_param_stats(root, param_values_harm, verbose=False)\n", + "\n", + " output_mocks_harm = np.vstack((output_mocks_harm, param_harm))\n", + "\n", + " param_values_config = extract_param_chain(chain_config[i], param_names)\n", + "\n", + " param_config = concatenate_param_stats(root, param_values_config, verbose=False)\n", + "\n", + " output_mocks_config = np.vstack((output_mocks_config, param_config))\n", + "\n", + "np.savetxt(\n", + " f\"summary_parameter_constraints_harmonic_space_{chain_version}.txt\",\n", + " output_mocks_harm,\n", + " fmt=\"%s\",\n", + " delimiter=\";\",\n", + ")\n", + "np.savetxt(\n", + " f\"summary_parameter_constraints_configuration_space_{chain_version}.txt\",\n", + " output_mocks_config,\n", + " fmt=\"%s\",\n", + " delimiter=\";\",\n", + ")\n", + "print(\n", + " f\"Saved summary of parameter constraints for harmonic space in summary_parameter_constraints_harmonic_space_{chain_version}.txt\"\n", + ")\n", + "print(\n", + " f\"Saved summary of parameter constraints for configuration space in summary_parameter_constraints_configuration_space_{chain_version}.txt\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": { + "lines_to_next_cell": 0 + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "output_df_harm = pd.read_csv(\n", + " f\"summary_parameter_constraints_harmonic_space_{chain_version}.txt\",\n", + " delimiter=\";\",\n", + " skiprows=1,\n", + " names=output_mocks_harm[0],\n", + ")\n", + "\n", + "output_df_config = pd.read_csv(\n", + " f\"summary_parameter_constraints_configuration_space_{chain_version}.txt\",\n", + " delimiter=\";\",\n", + " skiprows=1,\n", + " names=output_mocks_config[0],\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Define the true value of the parameters\n", + "from astropy.cosmology import Planck18 as planck\n", + "\n", + "Omega_m_fid = planck.Om0\n", + "sigma_8_fid = 0.8102\n", + "s8_fid = sigma_8_fid * (Omega_m_fid / 0.3) ** 0.5\n", + "h = planck.h\n", + "Omega_b_fig = planck.Ob0\n", + "n_s_fid = 0.9665\n", + "print(\n", + " f\"Fiducial values: Omega_m = {Omega_m_fid}, sigma_8 = {sigma_8_fid}, S_8 = {s8_fid}\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "sns.histplot(\n", + " output_df_harm[\"S8_mean\"] - output_df_config[\"S8_mean\"],\n", + " kde=True,\n", + " bins=30,\n", + " label=\"Mean\",\n", + ")\n", + "# sns.histplot(\n", + "# output_df_harm[\"S8_MAP\"]-output_df_config[\"S8_MAP\"],\n", + "# kde=True,\n", + "# bins=20,\n", + "# label=\"MAP\",\n", + "# )\n", + "sns.histplot(\n", + " output_df_harm[\"S8_MAP_2D\"] - output_df_config[\"S8_MAP_2D\"],\n", + " kde=True,\n", + " bins=30,\n", + " label=\"2D Mode\",\n", + " alpha=0.5,\n", + ")\n", + "plt.axvline(0, color=\"black\", linestyle=\"--\")\n", + "plt.legend(fontsize=12)\n", + "\n", + "plt.xlabel(r\"$\\Delta S_8$\")\n", + "plt.savefig(\n", + " \"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/notebooks/Plots/S8_comparison_harmonic_vs_configuration.pdf\",\n", + " bbox_inches=\"tight\",\n", + ")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "output_df_config[\"S8_MAP_2D\"].shape\n", + "output_df_harm[\"S8_MAP_2D\"].shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "# Create JointGrid\n", + "g = sns.JointGrid(\n", + " x=output_df_config[\"OMEGA_M_MAP_2D\"],\n", + " y=output_df_config[\"S8_MAP_2D\"],\n", + " height=7,\n", + " ratio=5,\n", + " space=0,\n", + ")\n", + "\n", + "# Main 2D histogram\n", + "sns.histplot(\n", + " x=output_df_config[\"OMEGA_M_MAP_2D\"],\n", + " y=output_df_config[\"S8_MAP_2D\"],\n", + " bins=25,\n", + " cmap=\"Greens\",\n", + " cbar=False,\n", + " ax=g.ax_joint,\n", + ")\n", + "\n", + "# Marginal histograms\n", + "sns.histplot(\n", + " x=output_df_config[\"OMEGA_M_MAP_2D\"], bins=25, color=\"#2ca25f\", ax=g.ax_marg_x\n", + ")\n", + "sns.histplot(y=output_df_config[\"S8_MAP_2D\"], bins=25, color=\"#2ca25f\", ax=g.ax_marg_y)\n", + "\n", + "# Add dashed reference lines\n", + "g.ax_joint.axvline(Omega_m_fid, color=\"k\", linestyle=\"--\")\n", + "g.ax_joint.axhline(s8_fid, color=\"k\", linestyle=\"--\")\n", + "\n", + "# Labels\n", + "g.set_axis_labels(\n", + " r\"$\\Omega_m$ estimated from mocks (Configuration space)\",\n", + " r\"$S_8$ estimated from mocks (Configuration space)\",\n", + ")\n", + "\n", + "# Optional styling tweaks\n", + "g.ax_joint.tick_params(labelsize=12)\n", + "plt.savefig(\n", + " \"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/notebooks/Plots/S8_vs_OmegaM_configuration_space_mocks.pdf\",\n", + " bbox_inches=\"tight\",\n", + ")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "-all", + "main_language": "python", + "notebook_metadata_filter": "-all" + }, + "kernelspec": { + "display_name": "my_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/masking.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/masking.ipynb new file mode 100644 index 00000000..b0dd4bbc --- /dev/null +++ b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/masking.ipynb @@ -0,0 +1,132 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Covmat mask analysis\n", + "\n", + "This notebook creates the plots to look at the ratio of the covaraiance matrices when applying the mask or not" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import healpy as hp\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import seaborn as sns\n", + "\n", + "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", + "\n", + "plt.rcParams[\"axes.labelsize\"] = 18\n", + "plt.rcParams[\"xtick.labelsize\"] = 18\n", + "plt.rcParams[\"ytick.labelsize\"] = 18\n", + "\n", + "plt.rcParams[\"text.usetex\"] = True\n", + "sns.set_palette(\"husl\")\n", + "\n", + "cat_dir = \"/n17data/UNIONS/WL/v1.4.x/\"\n", + "catalog_ver = \"v1.4.6.3\"\n", + "blind = \"B\"\n", + "\n", + "nside = 8192\n", + "npix = hp.nside2npix(nside)\n", + "\n", + "data_dir = \"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/data/\"\n", + "curr_dir = os.getcwd()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# PLOT 2D MAP OF COVMAT masked vs unmasked RATIOS\n", + "nbins = 20\n", + "ndata = nbins * 2\n", + "full_ratio = np.zeros((ndata, ndata))\n", + "\n", + "cov = np.loadtxt(data_dir + f\"/covs/cov_SP_{catalog_ver}_{blind}.txt\")\n", + "cov_masked = np.loadtxt(data_dir + f\"/covs/cov_masked_SP_{catalog_ver}_{blind}.txt\")\n", + "\n", + "for i in range(ndata):\n", + " for j in range(ndata):\n", + " full_ratio[i][j] = cov_masked[i][j] / cov[i][j]\n", + "\n", + "fig = plt.figure()\n", + "ax = fig.add_subplot(1, 1, 1)\n", + "extent = (0, ndata, ndata, 0)\n", + "\n", + "vmin, vmax = np.percentile(full_ratio, [1, 99])\n", + "\n", + "im3 = ax.imshow(full_ratio, cmap=\"RdBu_r\", vmin=vmin, vmax=vmax, extent=extent)\n", + "\n", + "cbar = fig.colorbar(im3, ax=ax, fraction=0.046, pad=0.04)\n", + "\n", + "ax.text(int(ndata / 4), ndata + 5, r\"$\\xi_+$\", fontsize=15)\n", + "ax.text(3 * int(ndata / 4), ndata + 5, r\"$\\xi_-$\", fontsize=15)\n", + "ax.text(-8, int(ndata / 4), r\"$\\xi_+$\", fontsize=15, rotation=90)\n", + "ax.text(-8, 3 * int(ndata / 4), r\"$\\xi_-$\", fontsize=15, rotation=90)\n", + "ax.set_xticks([0, 10, 20, 30, 40])\n", + "ax.set_yticks([0, 10, 20, 30, 40])\n", + "ax.set_yticklabels([\"1'\", \"125'\", \"250'\", \"125'\", \"250'\"])\n", + "ax.set_xticklabels([\"1'\", \"125'\", \"250'\", \"125'\", \"250'\"])\n", + "plt.axvline(x=int(ndata / 2), color=\"white\", linewidth=1.0)\n", + "plt.axhline(y=int(ndata / 2), color=\"white\", linewidth=1.0)\n", + "\n", + "plt.savefig(\n", + " f\"{curr_dir}/../Plots/covmat_masked_unmasked_ratio_{catalog_ver}_{blind}.pdf\",\n", + " bbox_inches=\"tight\",\n", + ")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "theta = np.linspace(1, 250, 20)\n", + "plt.axhline(y=1, color=\"k\", ls=\"--\")\n", + "plt.plot(theta, np.diag(cov_masked)[:20] / np.diag(cov)[:20], label=r\"$\\xi_+$\")\n", + "plt.plot(theta, np.diag(cov_masked)[20:] / np.diag(cov)[20:], label=r\"$\\xi_-$\")\n", + "\n", + "plt.xlabel(r\"$\\theta$ (arcmin)\")\n", + "plt.ylabel(\"Cov masked / Cov unmasked\")\n", + "plt.legend(fontsize=20)\n", + "plt.savefig(\n", + " f\"{curr_dir}/../Plots/covmat_masked_unmasked_ratio_diag.pdf\", bbox_inches=\"tight\"\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "my_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/nonlin_k_analysis.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/nonlin_k_analysis.ipynb new file mode 100644 index 00000000..4ab9c3c5 --- /dev/null +++ b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/nonlin_k_analysis.ipynb @@ -0,0 +1,174 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Nonlinear $k$ contributions\n", + "\n", + "This notebook plots the 2D heatmap of ratio of scale contributions to the $\\xi_\\pm$ 2PCF given angular scale $\\theta$ and wavenumber $k$." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import matplotlib.pylab as plt\n", + "import numpy as np\n", + "import seaborn as sns\n", + "\n", + "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", + "\n", + "plt.rcParams[\"text.usetex\"] = True\n", + "\n", + "plt.rcParams.update(\n", + " {\n", + " \"font.size\": 20,\n", + " \"axes.titlesize\": 21,\n", + " \"axes.labelsize\": 20,\n", + " \"xtick.labelsize\": 20,\n", + " \"ytick.labelsize\": 20,\n", + " \"legend.fontsize\": 20,\n", + " \"figure.titlesize\": 21,\n", + " }\n", + ")\n", + "sns.set_palette(\"husl\")\n", + "\n", + "blind = \"B\"\n", + "ver = \"v1.4.6.3\"\n", + "\n", + "%matplotlib inline\n", + "\n", + "data_dir = \"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/data/\"\n", + "curr_dir = os.getcwd()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plotting from script" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Read the 2D array from the text file\n", + "\n", + "file_headers = [\"xip_%s_%s\" % (ver, blind), \"xim_%s_%s\" % (ver, blind)]\n", + "\n", + "for f in file_headers:\n", + " xis = np.loadtxt(data_dir + f\"theta_k_{f}.txt\")\n", + " xis_reshaped = xis.reshape(-1, 201)\n", + " sorted_xis = xis_reshaped[np.argsort(xis_reshaped[:, 0])]\n", + "\n", + " np.savetxt(data_dir + f\"theta_k_{f}_sorted.txt\", sorted_xis)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig, axs = plt.subplots(2, 1, figsize=(8, 10))\n", + "\n", + "# --- k grid ---\n", + "h = 0.6766\n", + "k_plot = np.logspace(-4, 2, 200)\n", + "\n", + "file_header = \"%s_%s\" % (ver, blind)\n", + "\n", + "xi_thetas = np.loadtxt(data_dir + f\"theta_k_xip_{file_header}_sorted.txt\")\n", + "thetas = xi_thetas[:, 0]\n", + "xis = xi_thetas[:, 1:]\n", + "\n", + "# normalise\n", + "xi_plot = xis / np.max(xis, axis=1, keepdims=True)\n", + "\n", + "T, K = np.meshgrid(thetas, k_plot)\n", + "\n", + "axs[0].contour(T, K, xi_plot.T, levels=[0.9], colors=\"red\", linewidths=1.7)\n", + "pcm = axs[0].pcolormesh(T, K, xi_plot.T, shading=\"auto\", cmap=\"viridis\")\n", + "pcm.set_rasterized(True)\n", + "\n", + "axs[0].axvline(5, color=\"k\", ls=\"dashed\", lw=1.2)\n", + "axs[0].axvline(12, color=\"white\", ls=\"dashed\", lw=1.6)\n", + "axs[0].axhline(1, color=\"k\", ls=\"dashed\", lw=1.2) # converted to h/Mpc space if needed\n", + "axs[0].axhline(0.425, color=\"white\", ls=\"dashed\", lw=1.6)\n", + "\n", + "axs[0].set_yscale(\"log\")\n", + "axs[0].set_xlabel(r\"$\\theta\\ \\mathrm{(arcmin)}$\")\n", + "axs[0].set_ylabel(r\"$k\\ (h$ Mpc$^{-1})$\")\n", + "\n", + "axs[0].set_title(r\"$\\xi_+$\")\n", + "\n", + "xi_thetas = np.loadtxt(data_dir + f\"theta_k_xim_{file_header}_sorted.txt\")\n", + "thetas = xi_thetas[:, 0]\n", + "xis = xi_thetas[:, 1:]\n", + "\n", + "xi_plot = xis / np.max(xis, axis=1, keepdims=True)\n", + "\n", + "T, K = np.meshgrid(thetas, k_plot)\n", + "\n", + "axs[1].contour(T, K, xi_plot.T, levels=[0.9], colors=\"red\", linewidths=1.7)\n", + "pcm = axs[1].pcolormesh(T, K, xi_plot.T, shading=\"nearest\", cmap=\"viridis\")\n", + "pcm.set_rasterized(True)\n", + "\n", + "axs[1].axvline(12, color=\"white\", ls=\"dashed\", lw=1.6)\n", + "axs[1].axhline(2.85, color=\"white\", ls=\"dashed\", lw=1.6)\n", + "\n", + "\n", + "axs[1].set_yscale(\"log\")\n", + "axs[1].set_xlabel(r\"$\\theta\\ \\mathrm{(arcmin)}$\")\n", + "axs[1].set_ylabel(r\"$k\\ (h$ Mpc$^{-1})$\")\n", + "axs[1].set_title(r\"$\\xi_-$\")\n", + "\n", + "\n", + "fig.tight_layout()\n", + "\n", + "cbar_ax = fig.add_axes([0.99, 0.15, 0.02, 0.7])\n", + "cbar = fig.colorbar(pcm, cax=cbar_ax)\n", + "\n", + "fig.savefig(\n", + " curr_dir + f\"/../Plots/theta_k_xip_xim_{ver}_{blind}.pdf\", bbox_inches=\"tight\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "my_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_unblinding/unblinding_party_plots.py b/cosmo_inference/notebooks/2D_cosmic_shear_unblinding/unblinding_party_plots.py new file mode 100644 index 00000000..13af7ed2 --- /dev/null +++ b/cosmo_inference/notebooks/2D_cosmic_shear_unblinding/unblinding_party_plots.py @@ -0,0 +1,894 @@ +# %% +import os +import sys +import warnings + +# Append any useful folder in the path +sys.path.append("/home/guerrini/sp_validation/cosmo_inference/scripts/") +sys.path.append( + "/home/guerrini/sp_validation/cosmo_inference/notebooks/2D_cosmic_shear_unblinding/" +) + +import matplotlib.pyplot as plt +import matplotlib.scale as mscale +import numpy as np +import seaborn as sns +from astropy.io import fits +from getdist import plots +from IPython.display import Markdown, display + +from sp_validation.rho_tau import SquareRootScale + +mscale.register_scale(SquareRootScale) + +import IPython + +ipython = IPython.get_ipython() + +if ipython is not None: + ipython.run_line_magic("load_ext", "autoreload") + ipython.run_line_magic("autoreload", "2") + +if ipython is not None: + ipython.run_line_magic("matplotlib", "inline") + +import chain_postprocessing as cp +import utils + +plt.style.use( + "/home/guerrini/sp_validation/papers/harmonic/matplotlib_config/paper.mplstyle" +) + +plt.rcParams["text.usetex"] = True + +sns.set_palette("husl") + +g = plots.get_subplot_plotter(width_inch=30) +g.settings.axes_fontsize = 30 +g.settings.axes_labelsize = 30 +g.settings.alpha_filled_add = 0.7 +g.settings.legend_fontsize = 40 + +# Directory where the chains are located +root_dir = "/n09data/guerrini/output_chains" + +# THE BLIND TO USE FOR THE PLOTS +blind = "B" # Options are "A", "B", or "C" +catalog_version = "SP_v1.4.6.3" +fiducial_root_cell = f"SP_v1.4.6.3_leak_corr_{blind}" +label_fiducial_cell = r"UNIONS $C_{\ell}$" +fiducial_root_xi_data = f"SP_v1.4.6.3_leak_corr_{blind}_masked" +fiducial_root_xi_chains = f"SP_v1.4.6.3_{blind}_fiducial_config" +label_fiducial_xi = r"UNIONS $\xi_{\pm}$" + +# Path to the ini files used +path_ini_files = "/home/guerrini/sp_validation/cosmo_inference/cosmosis_config" +path_datavectors = "/home/guerrini/sp_validation/cosmo_inference/data/" +path_output_chains = "/n09data/guerrini/output_chains/" + +# %% +# 0. Do a funny print with emojis for the unblinding +display(Markdown("## 🎉 Let the Unblinding Party begin 🎉")) +# %% +# 1. Plot the datavectors without best-fit +display(Markdown("### 1.a. Plot the datavectors without best-fit")) + +# Plot Cells EE +data = fits.open( + os.path.join( + path_datavectors, f"{fiducial_root_cell}/cosmosis_{fiducial_root_cell}.fits" + ) +) +cell_ee = data["CELL_EE"].data +cov_mat = data["COVMAT"].data + +# Plot hyperparameter +loc_legend = "lower center" +bbox_to_anchor = (0.685, 0.70) + +fig, ax = plt.subplots(1, 1, figsize=(8, 5)) + +ell, cell = cell_ee["ANG"], cell_ee["VALUE"] +ax.errorbar( + ell, + ell * cell, + yerr=ell * np.sqrt(np.diag(cov_mat)), + fmt="o", + label=r"UNIONS $C_{\ell}$ data", + color="black", + capsize=2, +) + +# Plot the scale cuts for different k_max +ax.axvline(x=1800, color="black", linestyle="--", alpha=0.5) +ax.axvline(x=2048, color="black", linestyle="--", alpha=1.0) +ax.axvline(x=500, color="black", linestyle="--", alpha=0.3) + +ymin = ax.get_ylim()[0] +ymax = ax.get_ylim()[1] +# Shadowing cut scaled +ax.fill_betweenx( + y=[ymin, ymax], + x1=0, + x2=300, + color="gray", + alpha=0.2, + label=r"$B$-mode informed scale cut", +) +ax.fill_betweenx(y=[ymin, ymax], x1=1600, x2=2048, color="gray", alpha=0.2) + +ax.set_ylim(ymin, ymax) + +# Add labels directly under the tick +ax.text( + 1740, + 0.90, + r"$k_\mathrm{max} = 3 h$ Mpc$^{-1}$", + transform=ax.get_xaxis_transform(), + ha="center", + va="top", + fontsize=10, + rotation=90, +) + +ax.text( + 1978, + 0.90, + r"$k_\mathrm{max} = 5 h$ Mpc$^{-1}$", + transform=ax.get_xaxis_transform(), + ha="center", + va="top", + fontsize=10, + rotation=90, +) + +ax.text( + 470, + 0.90, + r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", + transform=ax.get_xaxis_transform(), + ha="center", + va="top", + fontsize=10, + rotation=90, +) + +ell, cell = cell_ee["ANG"], cell_ee["VALUE"] +ax.set_ylabel(r"$\ell C_\ell$", fontsize=16) +ax.set_xlabel(r"$\ell$", fontsize=16) +ax.set_xlim(ell.min() - 10, ell.max() + 100) +ax.set_xscale("squareroot") +ax.set_xticks(np.array([100, 400, 900, 1600])) +ax.minorticks_on() +ax.tick_params(axis="x", which="minor", length=2, width=0.8) +minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] +ax.xaxis.set_ticks(minor_ticks, minor=True) +ax.tick_params(axis="both", which="major", labelsize=14) +ax.tick_params(axis="both", which="minor", labelsize=10) +ax.yaxis.get_offset_text().set_fontsize(14) + +plt.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor, fontsize=12) + +plt.show() + +# Plots xi_+ and xi_- + +# TODO: add the plot for xi_+ and xi_- +display(Markdown(r"### 1.b. Plot the datavectors without best-fit ($\xi_\pm$)")) + +# Plot xi_pm's +data = fits.open( + os.path.join( + path_datavectors, + f"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}/cosmosis_{fiducial_root_xi_data}.fits", + ) +) +xi_p_data = data["XI_PLUS"].data +xi_m_data = data["XI_MINUS"].data +cov_mat = data["COVMAT"].data + +# Plot hyperparameter +loc_legend = "lower center" +bbox_to_anchor_xip = (0.685, 0.03) +bbox_to_anchor_xim = (0.3, 0.65) + +fig, [ax, ax2] = plt.subplots(2, 1, figsize=(8, 9)) + +theta, xi_p = xi_p_data["ANG"], xi_p_data["VALUE"] +ax.errorbar( + theta, + theta * xi_p, + yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])), + fmt="o", + label=r"UNIONS $\xi_+$ data", + color="black", + capsize=2, +) + +# Plot the scale cuts for different k_max +ax.axvline(x=3.2, color="black", linestyle="--", alpha=0.3) + +ymin = ax.get_ylim()[0] +ymax = ax.get_ylim()[1] +# Shadowing cut scaled +ax.fill_betweenx( + y=[ymin, ymax], + x1=0, + x2=12, + color="gray", + alpha=0.2, + label=r"$B$-mode informed scale cut", +) +ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color="gray", alpha=0.2) + +ax.set_ylim(ymin, ymax) + +# Add labels directly under the tick +ax.text( + 3, + 1e-4, + r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", + # transform=ax.get_xaxis_transform(), + ha="center", + va="top", + fontsize=10, + rotation=90, +) + +ax.set_ylabel(r"$\theta \xi_+$", fontsize=16) +# ax.set_xlabel('$\theta$', fontsize=16) +# ax.set_xlim([theta.min()-0.1, theta.max()+20]) +ax.set_xscale("log") +ax.set_xticks(np.array([1, 10, 100])) +ax.tick_params(axis="x", which="minor", length=2, width=0.8) +ax.tick_params(axis="both", which="major", labelsize=14) +ax.tick_params(axis="both", which="minor", labelsize=10) +ax.yaxis.get_offset_text().set_fontsize(14) +ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) +ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=12) + +theta, xi_m = xi_p_data["ANG"], xi_m_data["VALUE"] +ax2.errorbar( + theta, + theta * xi_m, + yerr=theta + * np.sqrt( + np.diag(cov_mat[len(theta) : 2 * len(theta), len(theta) : 2 * len(theta)]) + ), + fmt="o", + label=r"UNIONS $\xi_-$ data", + color="black", + capsize=2, +) + +# Plot the scale cuts for different k_max +ax2.axvline(x=24, color="black", linestyle="--", alpha=0.3) + +ymin = ax2.get_ylim()[0] +ymax = ax2.get_ylim()[1] +# Shadowing cut scaled +ax2.fill_betweenx( + y=[ymin, ymax], + x1=0, + x2=12, + color="gray", + alpha=0.2, + label=r"$B$-mode informed scale cut", +) +ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color="gray", alpha=0.2) + +ax2.set_ylim(ymin, ymax) + +# Add labels directly under the tick +ax2.text( + 22.3, + 9e-5, + r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", + # transform=ax.get_xaxis_transform(), + ha="center", + va="top", + fontsize=10, + rotation=90, +) + +ax2.set_ylabel(r"$\theta÷ \xi_-$", fontsize=16) +ax2.set_xlabel("$\theta$", fontsize=16) +ax2.set_xlim([theta.min() - 0.1, theta.max() + 20]) +ax2.set_xscale("log") +ax2.set_xticks(np.array([1, 10, 100])) +ax2.tick_params(axis="x", which="minor", length=2, width=0.8) +ax2.tick_params(axis="both", which="major", labelsize=14) +ax2.tick_params(axis="both", which="minor", labelsize=10) +ax2.yaxis.get_offset_text().set_fontsize(14) +ax2.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) +ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=12) + +plt.show() + +# %% +# 2. Plot the best-fit datavectors +display(Markdown("### 2. Plot the best-fit datavectors")) + +# Perform the computation for the fiducial of Cell +path_samples_fiducial_cell = os.path.join( + path_output_chains, + fiducial_root_cell, + fiducial_root_cell, + f"samples_{fiducial_root_cell}_cell.txt", +) +path_gd_fiducial_cell = os.path.join( + path_output_chains, + fiducial_root_cell, + fiducial_root_cell, + f"getdist_{fiducial_root_cell}_cell", +) +cp.load_samples_and_write_paramnames( + path_samples_fiducial_cell, path_gd_fiducial_cell + ".paramnames" +) +cp.write_samples_getdist_format( + path_samples_fiducial_cell, path_gd_fiducial_cell + ".txt", chain_type="polychord" +) + +chain_fiducial_cell = cp.load_chain(path_gd_fiducial_cell, smoothing_scale=0.3) + +best_fit_params_fiducial_cell = cp.extract_best_fit_params(chain_fiducial_cell) + +cp.compute_best_fit( + path_ini_files, + best_fit_params_fiducial_cell, + fiducial_root_cell, + is_harmonic=True, + blind=blind, +) + +# Perform the computation for the fiducial of xi +path_samples_fiducial_xi = os.path.join( + path_output_chains, + fiducial_root_xi_chains, + f"samples_{fiducial_root_xi_chains}.txt", +) +path_gd_fiducial_xi = os.path.join( + path_output_chains, fiducial_root_xi_chains, f"getdist_{fiducial_root_xi_chains}" +) +cp.load_samples_and_write_paramnames( + path_samples_fiducial_xi, path_gd_fiducial_xi + ".paramnames" +) +cp.write_samples_getdist_format( + path_samples_fiducial_xi, path_gd_fiducial_xi + ".txt", chain_type="polychord" +) + +chain_fiducial_xi = cp.load_chain(path_gd_fiducial_xi, smoothing_scale=0.3) + +best_fit_params_fiducial_xi = cp.extract_best_fit_params(chain_fiducial_xi) + +ini_file_root = os.path.join( + path_ini_files, + f"config_space_v1.4.6.3_fiducial/pipeline/blind_{blind}/fiducial.ini", +) +cp.compute_best_fit( + path_ini_files, + best_fit_params_fiducial_xi, + fiducial_root_xi_chains, + is_harmonic=False, + blind=blind, + ini_file_root=ini_file_root, +) + +# %% +# Make the plot for the best-fit datavector for Cell EE +root_to_plot = [fiducial_root_cell, fiducial_root_xi_chains] + +labels = [r"UNIONS $C_\ell$", r"UNIONS $\xi_\pm(\vartheta)$"] + +line_args = [ + {"color": "royalblue", "linestyle": "-"}, + {"color": "orange", "linestyle": "-"}, +] + +properties = {} + +properties = utils.update_properties_w_roots( + properties, fiducial_root_cell, path_ini_files, with_configuration=False +) +properties = utils.update_properties_w_roots( + properties, + fiducial_root_xi_chains, + path_ini_files, + with_configuration=True, + path_to_this_ini=ini_file_root, +) + +utils.plot_best_fit( + fiducial_root_cell, + root_to_plot, + path_output_chains, + line_args, + savefile=None, + labels=labels, + loc_legend=loc_legend, + bbox_to_anchor=bbox_to_anchor, + properties=properties, +) + +# TODO: add the plot for xi +# %% +# Plot best-fit xi_+ and xi_- (also from C_ell's) + +path_best_fit_xi_theta = os.path.join( + path_output_chains, fiducial_root_xi_chains, "best_fit/shear_xi_plus/theta.txt" +) +theta_rad = np.loadtxt(path_best_fit_xi_theta) + +cp.compute_best_fit_xi_from_cell( + path_output_chains, fiducial_root_cell, best_fit_params_fiducial_cell, theta_rad +) + + +xi_data_path = os.path.join( + path_datavectors, + f"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}/cosmosis_{fiducial_root_xi_data}.fits", +) +utils.plot_best_fit_config( + xi_data_path, + root_to_plot, + path_output_chains, + line_args, + savefile=None, + labels=labels, + loc_legend=loc_legend, + bbox_to_anchor_xip=bbox_to_anchor_xip, + bbox_to_anchor_xim=bbox_to_anchor_xim, + properties=properties, +) + +# %% +# 3. Do a whisker plot with external experiments and our constraints +display(Markdown("### 🥸 Time to look at the whisker plot 🥸")) +colour_blind = {"A": "royalblue", "B": "crimson", "C": "forestgreen"} + +roots = [ + f"SP_v1.4.6.3_leak_corr_{blind}", + f"SP_v1.4.6.3_{blind}_fiducial_config", + "Planck18", + "DES_Y3", + "DES_Y3_cell", + "KiDS-1000", + "KiDS-1000_cosebis", + "KiDS-1000_bp", + "DES+KiDS", + "HSC_Y3", + "HSC_Y3_cell", +] + +legend_labels = [ + r"UNIONS $C_\ell$, unblind", + r"UNIONS $\xi_\pm(\vartheta)$, unblind", + r"\textit{Planck} 2018", + r"DES Y3 $\xi_\pm(\vartheta)$", + r"DES Y3 $C_\ell$", + r"KiDS-1000 $\xi_\pm(\vartheta)$", + r"KiDS-1000 $E_n$", + r"KiDS-1000 $C_E$", + r"DES Y3 + KiDS-1000 combined", + r"HSC Y3 $\xi_\pm(\vartheta)$", + r"HSC Y3 $C_\ell$", +] + +colours = [ + colour_blind[blind], + colour_blind[blind], + "violet", + "black", + "black", + "black", + "black", + "black", + "black", + "black", + "black", +] + +categories = [ + "harmonic", + "configuration", + "external", + "external", + "external", + "external", + "external_compute_sample", + "external_compute_sample", + "external", + "external", + "external_compute_sample", +] + +for bl in ["A", "B", "C"]: + if bl != blind: + roots.append(f"SP_v1.4.6.3_leak_corr_{bl}") + roots.append(f"SP_v1.4.6.3_{bl}_fiducial_config") + legend_labels.append(rf"UNIONS $C_\ell$, Blind {bl}") + legend_labels.append(rf"UNIONS $\xi_\pm(\vartheta)$, Blind {bl}") + colours.append(colour_blind[bl]) + colours.append(colour_blind[bl]) + categories.append("harmonic") + categories.append("configuration") + +# Loop on all versions to load the chain +chains = [] +for i, root in enumerate(roots): + category = categories[i] + if category != "external": + if category == "configuration": + path_samples = os.path.join( + path_output_chains, f"{root}/samples_{root}.txt" + ) + path_getdist = os.path.join(path_output_chains, f"{root}/getdist_{root}") + elif category == "harmonic": + path_samples = os.path.join( + path_output_chains, f"{root}/{root}/samples_{root}_cell.txt" + ) + path_getdist = os.path.join( + path_output_chains, f"{root}/{root}/getdist_{root}" + ) + elif category == "external_compute_sample": + path_samples = os.path.join( + path_output_chains, f"ext_data/{root}/samples_{root}.txt" + ) + path_getdist = os.path.join( + path_output_chains, f"ext_data/{root}/getdist_{root}" + ) + else: + raise ValueError(f"The category, {category}, of {root} is not correct") + + cp.load_samples_and_write_paramnames(path_samples, path_getdist + ".paramnames") + cp.write_samples_getdist_format(path_samples, path_getdist + ".txt") + chains.append(cp.load_chain(path_getdist, smoothing_scale=0.5)) + else: + path_getdist = os.path.join( + path_output_chains, f"ext_data/{root}/getdist_{root}" + ) + chains.append(cp.load_chain(path_getdist)) + +# Give labels for the chains +name_list = [ + "OMEGA_M", + "ombh2", + "h0", + "n_s", + "SIGMA_8", + "S_8", + "s_8_input", + "logt_agn", + "a", + "m1", + "bias_1", +] +label_list = [ + r"\Omega_{\rm m}", + r"\omega_b h^2", + r"h_0", + r"n_s", + r"\sigma_8", + r"S_8", + r"S_8", + r"\log T_{\rm AGN}", + r"A_{\rm IA}", + r"m_1", + r"\Delta z_1", +] + +for i, chain in enumerate(chains): + print(legend_labels[i]) + param_names = chain.getParamNames() + for name, label in zip(name_list, label_list): + try: + param_names.parWithName(name).label = label + except Exception: + warnings.warn(f"Parameter {name} not found in chain {roots[i]}.") + +# Account for the missing parameter conventions +# OMEGA_M not in DES_Y3_cell +idx = roots.index("DES_Y3_cell") +cp.adjust_paramname_chain(chains[idx], "omega_m", "OMEGA_M", r"\Omega_{\rm m}") +cp.derive_parameter_S8(chains[idx]) + +# OMEGA_M not in KiDS-1000 +idx = roots.index("KiDS-1000") +cp.adjust_paramname_chain(chains[idx], "omega_m", "OMEGA_M", r"\Omega_{\rm m}") + +# OMEGA_M not in DES+KiDS +idx = roots.index("DES+KiDS") +cp.adjust_paramname_chain(chains[idx], "omega_m", "OMEGA_M", r"\Omega_{\rm m}") + +# OMEGA_M not in HSC_Y3_cell +idx = roots.index("HSC_Y3_cell") +cp.adjust_paramname_chain(chains[idx], "omega_m", "OMEGA_M", r"\Omega_{\rm m}") + +# Build an array containing the parameter values +param_values = np.array( + [ + "# Expt", + "Colour", + "S8_Mean", + "S8_low", + "S8_high", + "sigma_8_Mean", + "sigma_8_low", + "sigma_8_high", + "Omega_m_Mean", + "Omega_m_low", + "Omega_m_high", + ] +) +escaped = np.char.replace(legend_labels, "\\", "\\\\") +for i, chain in enumerate(chains): + print(chain.root) + margestats = chain.getMargeStats() + likestats = chain.getLikeStats() + + s8_stats = margestats.parWithName("S_8") + sigma8_stats = margestats.parWithName("SIGMA_8") + omegam_stats = margestats.parWithName("OMEGA_M") + + param_values = np.vstack( + ( + param_values, + [ + escaped[i], + colours[i], + s8_stats.mean, + s8_stats.mean - s8_stats.limits[0].lower, + s8_stats.limits[0].upper - s8_stats.mean, + sigma8_stats.mean, + sigma8_stats.mean - sigma8_stats.limits[0].lower, + sigma8_stats.limits[0].upper - sigma8_stats.mean, + omegam_stats.mean, + omegam_stats.mean - omegam_stats.limits[0].lower, + omegam_stats.limits[0].upper - omegam_stats.mean, + ], + ) + ) +print(param_values) +np.savetxt( + "./param_values.txt", param_values, fmt=["%s" for i in range(11)], delimiter=";" +) + +# Reload the table +# Load the value of the parameters +cosmo = np.loadtxt( + "./param_values.txt", + dtype={ + "names": ( + "Expt", + "colour", + "s8_mean", + "s8_low", + "s8_high", + "sigma8_mean", + "sigma8_low", + "sigma8_high", + "omegam_mean", + "omegam_low", + "omegam_high", + ), + "formats": ( + "U250", + "U20", + "U20", + "U20", + "U20", + "U20", + "U20", + "U20", + "U20", + "U20", + "U20", + ), + }, + skiprows=1, + delimiter=";", +) +expt = np.char.replace(cosmo["Expt"], "\\\\", "\\") +colours = cosmo["colour"] +s8_mean = cosmo["s8_mean"].astype(np.float64) +s8_low = cosmo["s8_low"].astype(np.float64) +s8_high = cosmo["s8_high"].astype(np.float64) +sigma8_mean = cosmo["sigma8_mean"].astype(np.float64) +sigma8_low = cosmo["sigma8_low"].astype(np.float64) +sigma8_high = cosmo["sigma8_high"].astype(np.float64) +omegam_mean = cosmo["omegam_mean"].astype(np.float64) +omegam_low = cosmo["omegam_low"].astype(np.float64) +omegam_high = cosmo["omegam_high"].astype(np.float64) + +# %% +# Perform the plot +from matplotlib.gridspec import GridSpec + +fig = plt.figure(figsize=(10, 6)) +gs = GridSpec(1, 3, width_ratios=[1, 0.5, 0.5]) +ax1 = fig.add_subplot(gs[0]) +ax2 = fig.add_subplot(gs[1], sharey=ax1) +ax3 = fig.add_subplot(gs[2], sharey=ax1) + +axs = [ax1, ax2, ax3] + +params = [ + (s8_mean, s8_low, s8_high, r"$S_8$"), + (sigma8_mean, sigma8_low, sigma8_high, r"$\sigma_8$"), + (omegam_mean, omegam_low, omegam_high, r"$\Omega_{\rm m}$"), +] +reference = r"UNIONS $C_\ell$, unblind" +separation_after = [ + r"UNIONS $\xi_\pm(\vartheta)$, unblind", + r"HSC Y3 $C_\ell$", +] +list_section_index = [r"(ii)", r"(iii)", r"(iv)", r"(v)", r"(vi)", r"(vii)"] + +preliminary_watermark = False +blind_axes = False +row_spacing = 0.1 + +index_ref = np.where(expt == reference)[0][0] + +y = np.arange(len(expt)) +for ax, param in zip(axs, params): + means, lows, highs, label = param + for i, mean, low, high, color in zip(y, means, lows, highs, colours): + ax.errorbar( + mean, + 0.05 + i * row_spacing, + xerr=np.array([low, high])[:, None], + fmt="o", + color=color, + ecolor=color, + elinewidth=2, + capsize=3, + ) + ax.set_xlabel(label, fontsize=14) + + ax.grid(False) + ax.tick_params(axis="y", left=False, labelleft=False) + if label == r"$S_8$": + ax.axvspan( + s8_mean[index_ref] - s8_low[index_ref], + s8_mean[index_ref] + s8_high[index_ref], + color=colours[index_ref], + alpha=0.2, + ) + ax.set_xlim(0.25, 1.1) + if blind_axes: + ref_tick = np.mean(s8_mean[:4]) + ax.set_xticks([ref_tick + i * 0.1 for i in range(-5, 5)], labels=[]) + elif label == r"$\sigma_8$": + ax.axvspan( + sigma8_mean[index_ref] - sigma8_low[index_ref], + sigma8_mean[index_ref] + sigma8_high[index_ref], + color=colours[index_ref], + alpha=0.2, + ) + ax.set_xlim(0.5, 1.2) + if blind_axes: + ref_tick = np.mean(sigma8_mean[:4]) + ax.set_xticks([ref_tick + i * 0.2 for i in range(-2, 2)], labels=[]) + elif label == r"$\Omega_{\rm m}$": + ax.axvspan( + omegam_mean[index_ref] - omegam_low[index_ref], + omegam_mean[index_ref] + omegam_high[index_ref], + color=colours[index_ref], + alpha=0.2, + ) + ax.set_xlim(0.1, 0.5) + if blind_axes: + ref_tick = np.mean(omegam_mean[:4]) + ax.set_xticks([ref_tick + i * 0.1 for i in range(-2, 3)], labels=[]) + + +axs[0].set_yticks(0.05 + y * row_spacing) +axs[0].set_yticklabels([]) +for label, color in zip(expt, colours): + axs[0].text( + 0.26, + 0.05 + row_spacing * np.where(expt == label)[0][0], + label, + fontsize=12, + ha="left", + va="center", + color=color, + ) + if label != reference: + index = np.where(expt == label)[0][0] + s8_tension = cp.get_sigma_tension( + s8_mean[index], + s8_low[index], + s8_high[index], + s8_mean[index_ref], + s8_low[index_ref], + s8_high[index_ref], + ) + sign_str = "+" if s8_tension > 0 else "-" + axs[0].text( + 1.095, + 0.05 + row_spacing * index, + rf"${sign_str}{np.abs(s8_tension):.2f}" + r"\, \sigma$", + fontsize=10, + ha="right", + va="center", + color=color, + ) +# Add separation lines +for i, sep in enumerate(separation_after): + index_sep = np.where(expt == sep)[0][0] + for ax in axs: + ax.axhline( + row_spacing * (index_sep + 1), + color="black", + linestyle="dotted", + linewidth=1, + ) + axs[0].text( + 0.25, + 0.05 + row_spacing * (index_sep + 1), + list_section_index[i], + fontsize=14, + fontweight="bold", + va="center", + ha="right", + ) + + +# --- Add section labels (i), (ii)) --- +axs[0].text(0.25, 0.05, r"(i)", fontsize=14, fontweight="bold", va="center", ha="right") + +if preliminary_watermark: + plt.figtext( + 0.5, + 0.5, + "PRELIMINARY", + fontsize=50, + color="gray", + ha="center", + va="center", + alpha=0.3, + rotation=330, + ) + +plt.gca().invert_yaxis() + +plt.tight_layout() + +plt.show() + +# %% +# 4. Make a contour plots +display(Markdown(r"### Here comes $S_8$ and $\Omega_m$")) +colours = ["royalblue", "orange", "violet"] + +filled = [True, True, False, False, False, False, False, False, False, False, False] + +line_args = [dict(color=col, ls="solid") for col in colours] + +g = plots.get_single_plotter(width_inch=30) +g.settings.axes_fontsize = 60 +g.settings.axes_labelsize = 60 +g.settings.alpha_filled_add = 0.7 +g.settings.legend_fontsize = 45 +g.settings.figure_legend_ncol = 3 +g.settings.legend_frame = False + +g.plot_2d( + chains[:-4], + "OMEGA_M", + "S_8", + filled=filled, + line_args=line_args, + contour_colors=colours, +) + +g.add_legend( + legend_labels[:-4], + legend_loc="upper center", + bbox_to_anchor=(0.5, 1.20), # moves legend above the axes +) + +plt.show() +# %% diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_unblinding/utils.py b/cosmo_inference/notebooks/2D_cosmic_shear_unblinding/utils.py new file mode 100644 index 00000000..8c2fb9e5 --- /dev/null +++ b/cosmo_inference/notebooks/2D_cosmic_shear_unblinding/utils.py @@ -0,0 +1,442 @@ +""" +Useful scripts to perform the plots for the unblinding party. +""" + +import configparser +import os +import sys + +# Append any useful folder in the path +sys.path.append("/home/guerrini/sp_validation/cosmo_inference/scripts/") + +import matplotlib.pyplot as plt +import matplotlib.scale as mscale +import numpy as np +from astropy.io import fits + +from sp_validation.rho_tau import SquareRootScale + +mscale.register_scale(SquareRootScale) + + +def read_config(path_ini_files, root, thisfile=None): + config = configparser.ConfigParser() + config.optionxform = str + if thisfile is not None: + read_path = thisfile + else: + read_path = os.path.join(path_ini_files, f"{root}.ini") + config.read(read_path) + return config + + +def update_properties_w_roots( + properties, root, path_ini_files, path_to_this_ini=None, with_configuration=False +): + config = read_config(path_ini_files, root, thisfile=path_to_this_ini) + + try: + lower_bound_cell_ee, upper_bound_cell_ee = map( + float, config["2pt_like"]["angle_range_CELL_EE_1_1"].split() + ) + properties[root].update( + { + "lower_bound_cell_ee": lower_bound_cell_ee, + "upper_bound_cell_ee": upper_bound_cell_ee, + } + ) + except KeyError: + properties[root] = {"lower_bound_cell_ee": 0.0, "upper_bound_cell_ee": 2048} + + if with_configuration: + # Also save the scale cuts in theta for xi + add_xi_sys = config["2pt_like"]["add_xi_sys"] + add_xi_sys = add_xi_sys == "T" + lower_bound_xi_plus, upper_bound_xi_plus = map( + float, config["2pt_like"]["angle_range_XI_PLUS_1_1"].split() + ) + lower_bound_xi_minus, upper_bound_xi_minus = map( + float, config["2pt_like"]["angle_range_XI_MINUS_1_1"].split() + ) + + properties[root].update( + { + "add_xi_sys": add_xi_sys, + "lower_bound_xi_plus": lower_bound_xi_plus, + "upper_bound_xi_plus": upper_bound_xi_plus, + "lower_bound_xi_minus": lower_bound_xi_minus, + "upper_bound_xi_minus": upper_bound_xi_minus, + } + ) + return properties + + +def plot_best_fit( + data_points, + root_to_plot, + output_folder, + line_args, + savefile, + ell_min=10.0, + ell_max=2048.0, + multiply_ell=True, + loc_legend="best", + bbox_to_anchor=None, + label_data="Fiducial data", + labels=None, + properties=None, + paths_to_bestfit=None, +): + data = fits.open( + f"/home/guerrini/sp_validation/cosmo_inference/data/{data_points}/cosmosis_{data_points}.fits" + ) + cell_ee = data["CELL_EE"].data + cov_mat = data["COVMAT"].data + + if labels is None: + labels = root_to_plot + + fig, ax = plt.subplots(1, 1, figsize=(8, 5)) + + ell, cell = cell_ee["ANG"], cell_ee["VALUE"] + ax.errorbar( + ell, + ell * cell, + yerr=ell * np.sqrt(np.diag(cov_mat)), + fmt="o", + label=label_data, + color="black", + capsize=2, + ) + + for idx, (label, root) in enumerate(zip(labels, root_to_plot)): + # Read the results + if paths_to_bestfit is None: + ell = np.loadtxt( + output_folder + + "{}/best_fit/shear_cl/ell.txt".format( + root, + ) + ) + shear_cl = np.loadtxt( + output_folder + + "{}/best_fit/shear_cl/bin_1_1.txt".format( + root, + ) + ) + else: + ell = np.loadtxt(paths_to_bestfit[idx] + "best_fit/shear_cl/ell.txt") + shear_cl = np.loadtxt( + paths_to_bestfit[idx] + "best_fit/shear_cl/bin_1_1.txt" + ) + + mask = (ell > ell_min) & (ell < ell_max) + + ax.plot( + ell[mask], + ell[mask] * shear_cl[mask] if multiply_ell else shear_cl[mask], + label=label, + **line_args[idx], + ) + + # Plot the scale cuts for different k_max + ax.axvline(x=1800, color="black", linestyle="--", alpha=0.5) + ax.axvline(x=2048, color="black", linestyle="--", alpha=1.0) + ax.axvline(x=500, color="black", linestyle="--", alpha=0.3) + + ymin = ax.get_ylim()[0] + ymax = ax.get_ylim()[1] + # Shadowing cut scaled + ax.fill_betweenx( + y=[ymin, ymax], + x1=0, + x2=300, + color="gray", + alpha=0.2, + label=r"$B$-mode informed scale cut", + ) + ax.fill_betweenx(y=[ymin, ymax], x1=1600, x2=2048, color="gray", alpha=0.2) + + ax.set_ylim(ymin, ymax) + + # Add labels directly under the tick + ax.text( + 1740, + 0.90, + r"$k_\mathrm{max} = 3 h$ Mpc$^{-1}$", + transform=ax.get_xaxis_transform(), + ha="center", + va="top", + fontsize=14, + rotation=90, + ) + + ax.text( + 1978, + 0.90, + r"$k_\mathrm{max} = 5 h$ Mpc$^{-1}$", + transform=ax.get_xaxis_transform(), + ha="center", + va="top", + fontsize=14, + rotation=90, + ) + + ax.text( + 470, + 0.90, + r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", + transform=ax.get_xaxis_transform(), + ha="center", + va="top", + fontsize=14, + rotation=90, + ) + + ell, cell = cell_ee["ANG"], cell_ee["VALUE"] + ax.set_ylabel(r"$\ell C_\ell \times 10^{-7}$", fontsize=20) + ax.set_xlabel(r"Multipole $\ell$", fontsize=20) + ax.set_xlim(ell.min() - 10, ell.max() + 100) + ax.set_xscale("squareroot") + ax.set_xticks(np.array([100, 400, 900, 1600])) + ax.minorticks_on() + ax.tick_params(axis="x", which="minor", length=2, width=0.8) + minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] + ax.xaxis.set_ticks(minor_ticks, minor=True) + ax.tick_params(axis="both", which="major", labelsize=14) + ax.tick_params(axis="both", which="minor", labelsize=10) + ax.yaxis.get_offset_text().set_visible(False) + + plt.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor, fontsize=11) + + if savefile is not None: + plt.savefig(savefile, bbox_inches="tight") + + plt.show() + + +def plot_best_fit_config( + data, + root_to_plot, + output_folder, + line_args, + savefile, + theta_min=1.0, + theta_max=250.0, + multiply_theta=True, + loc_legend="best", + bbox_to_anchor_xip=None, + bbox_to_anchor_xim=None, + label_data="Fiducial data", + labels=None, + properties=None, + paths_to_bestfit=None, +): + + data = fits.open(data) + + xi_p_data = data["XI_PLUS"].data + xi_m_data = data["XI_MINUS"].data + cov_mat = data["COVMAT"].data + + # Plot hyperparameter + loc_legend = "lower center" + + fig, [ax, ax2] = plt.subplots(2, 1, figsize=(8, 9)) + + theta, xi_p, xi_m = xi_p_data["ANG"], xi_p_data["VALUE"], xi_m_data["VALUE"] + ax.errorbar( + theta, + theta * xi_p, + yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])), + fmt="o", + label=r"UNIONS $\xi_+$ data", + color="black", + capsize=2, + ) + ax2.errorbar( + theta, + theta * xi_m, + yerr=theta + * np.sqrt( + np.diag(cov_mat[len(theta) : 2 * len(theta), len(theta) : 2 * len(theta)]) + ), + fmt="o", + label=r"UNIONS $\xi_-$ data", + color="black", + capsize=2, + ) + + for idx, (label, root) in enumerate(zip(labels, root_to_plot)): + # Read the results + if paths_to_bestfit is None: + theta = ( + ( + np.loadtxt( + output_folder + + "{}/best_fit/shear_xi_plus/theta.txt".format(root) + ) + ) + * 180 + / np.pi + * 60 + ) + xi_plus = np.loadtxt( + output_folder + "{}/best_fit/shear_xi_plus/bin_1_1.txt".format(root) + ) + xi_minus = np.loadtxt( + output_folder + "{}/best_fit/shear_xi_minus/bin_1_1.txt".format(root) + ) + if r"$C_\ell$" not in label: + xi_sys_plus = np.loadtxt( + output_folder + "{}/best_fit/xi_sys/shear_xi_plus.txt".format(root) + ) + xi_sys_minus = np.loadtxt( + output_folder + "{}/best_fit/xi_sys/shear_xi_minus.txt".format(root) + ) + theta_xi_sys = ( + np.loadtxt( + output_folder + "{}/best_fit/xi_sys/theta.txt".format(root) + ) + * 180 + / np.pi + * 60 + ) + xi_plus += np.interp(theta, theta_xi_sys, xi_sys_plus) + xi_minus += np.interp(theta, theta_xi_sys, xi_sys_minus) + else: + theta = ( + (np.loadtxt(paths_to_bestfit[idx] + "best_fit/shear_xi_plus/theta.txt")) + * 180 + / np.pi + * 60 + ) + xi_plus = np.loadtxt( + paths_to_bestfit[idx] + "best_fit/shear_xi_plus/bin_1_1.txt" + ) + xi_minus = np.loadtxt( + paths_to_bestfit[idx] + "best_fit/shear_xi_minus/bin_1_1.txt" + ) + if r"$C_\ell$" not in label: + xi_sys_plus = np.loadtxt( + output_folder + "{}/best_fit/xi_sys/shear_xi_plus.txt".format(root) + ) + xi_sys_minus = np.loadtxt( + output_folder + "{}/best_fit/xi_sys/shear_xi_minus.txt".format(root) + ) + theta_xi_sys = ( + np.loadtxt( + output_folder + "{}/best_fit/xi_sys/theta.txt".format(root) + ) + * 180 + / np.pi + * 60 + ) + xi_plus += np.interp(theta, theta_xi_sys, xi_sys_plus) + xi_minus += np.interp(theta, theta_xi_sys, xi_sys_minus) + + mask = (theta > theta_min) & (theta < theta_max) + theta = theta[mask] + ax.plot( + theta, + theta * xi_plus[mask] if multiply_theta else xi_plus[mask], + label=label, + **line_args[idx], + ) + ax2.plot( + theta, + theta * xi_minus[mask] if multiply_theta else xi_minus[mask], + label=label, + **line_args[idx], + ) + + # XI PLUS PLOT SETTINGS + + # Plot the scale cuts for different k_max + ax.axvline(x=3.2, color="black", linestyle="--", alpha=0.7) + + ymin = ax.get_ylim()[0] + ymax = ax.get_ylim()[1] + # Shadowing cut scaled + ax.fill_betweenx( + y=[ymin, ymax], + x1=0, + x2=12, + color="gray", + alpha=0.2, + label=r"$B$-mode informed scale cut", + ) + ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color="gray", alpha=0.2) + + ax.set_ylim(ymin, ymax) + + # Add labels directly under the tick + ax.text( + 2.9, + 1.23e-4, + r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", + ha="center", + va="top", + fontsize=14, + rotation=90, + ) + + # ax.set_ylabel('$\theta \xi_+$', fontsize=16) + # ax.set_xlabel('$\theta$', fontsize=16) + ax.set_xlim([theta.min() - 0.1, theta.max() + 20]) + ax.set_xscale("log") + ax.set_xticks(np.array([1, 10, 100])) + ax.tick_params(axis="x", which="minor", length=2, width=0.8) + ax.tick_params(axis="both", which="major", labelsize=14) + ax.tick_params(axis="both", which="minor", labelsize=10) + ax.yaxis.get_offset_text().set_fontsize(14) + ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) + ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=12) + + # XI_MINUS PLOT SETTINGS + + # Plot the scale cuts for different k_max + ax2.axvline(x=24, color="black", linestyle="--", alpha=0.7) + + ymin = ax2.get_ylim()[0] + ymax = ax2.get_ylim()[1] + # Shadowing cut scaled + ax2.fill_betweenx( + y=[ymin, ymax], + x1=0, + x2=12, + color="gray", + alpha=0.2, + label=r"$B$-mode informed scale cut", + ) + ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color="gray", alpha=0.2) + + ax2.set_ylim(ymin, ymax) + + # Add labels directly under the tick + ax2.text( + 21.8, + 1.15e-4, + r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", + ha="center", + va="top", + fontsize=14, + rotation=90, + ) + + ax2.set_ylabel(r"$\theta \xi_-$", fontsize=16) + ax2.set_xlabel(r"$\theta$", fontsize=16) + ax2.set_xlim([theta.min() - 0.1, theta.max() + 20]) + ax2.set_xscale("log") + ax2.set_xticks(np.array([1, 10, 100])) + ax2.tick_params(axis="x", which="minor", length=2, width=0.8) + ax2.tick_params(axis="both", which="major", labelsize=14) + ax2.tick_params(axis="both", which="minor", labelsize=10) + ax2.yaxis.get_offset_text().set_fontsize(14) + ax2.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) + ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=12) + + if savefile is not None: + plt.savefig(savefile, bbox_inches="tight") + + plt.show() diff --git a/cosmo_inference/notebooks/cfis_analysis.ipynb b/cosmo_inference/notebooks/cfis_analysis.ipynb new file mode 100644 index 00000000..ee93f4ec --- /dev/null +++ b/cosmo_inference/notebooks/cfis_analysis.ipynb @@ -0,0 +1,1065 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Analysis of a CFIS shear catalogue\n", + "First steps. Analysing both ShapePipe and Lensfit catalogues, for all blinds A,B and C" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import ipywidgets as widgets\n", + "import matplotlib.pylab as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import pyccl as ccl\n", + "import treecorr\n", + "from astropy.io import fits\n", + "from ipywidgets import interact\n", + "\n", + "%matplotlib inline\n", + "plt.rcParams.update({\"font.size\": 20, \"figure.figsize\": [12, 10]})\n", + "plt.rc(\"mathtext\", fontset=\"stix\")\n", + "plt.rc(\"font\", family=\"sans-serif\")\n", + "\n", + "# SPECIFY DIRECTORIES AND CATALOGUE PATHS\n", + "work_dir = \"/home/mkilbing/astro/data/UNIONS/v1.x/ShapePipe\"\n", + "\n", + "cat_dict = {\n", + " 1: {\n", + " \"dir\": work_dir + \"/Lensfit/lensfit_goldshape_2022v1.fits\",\n", + " \"label\": \"LF_full\",\n", + " \"e1_bias\": 0,\n", + " \"e2_bias\": 0,\n", + " \"ls\": \"solid\",\n", + " \"colour\": \"g\",\n", + " },\n", + " 2: {\n", + " \"dir\": work_dir + \"/ShapePipe/unions_shapepipe_2022_v1.0.fits\",\n", + " \"label\": \"SP_full\",\n", + " \"e1_bias\": 0,\n", + " \"e2_bias\": 0,\n", + " \"ls\": \"solid\",\n", + " \"colour\": \"b\",\n", + " },\n", + " 3: {\n", + " \"dir\": work_dir + \"/matched_LF_SP/masked_matched_lensfit_goldshape_2022v1.fits\",\n", + " \"label\": \"LF_matched_SP\",\n", + " \"e1_bias\": 3.939e-4,\n", + " \"e2_bias\": 6.482e-5,\n", + " \"ls\": \"dotted\",\n", + " \"colour\": \"g\",\n", + " },\n", + " 4: {\n", + " \"dir\": work_dir\n", + " + \"/matched_LF_SP/masked_matched_unions_shapepipe_extended_2022_v1.0.fits\",\n", + " \"label\": \"SP_matched_LF\",\n", + " \"e1_bias\": -5.6726e-5,\n", + " \"e2_bias\": 8.218e-4,\n", + " \"ls\": \"dotted\",\n", + " \"colour\": \"b\",\n", + " },\n", + " 5: {\n", + " \"dir\": work_dir + \"/matched_LF_SP/matched_footprint_shapepipe.fits\",\n", + " \"label\": \"SP Match LF Footprint\",\n", + " \"e1_bias\": 0,\n", + " \"e2_bias\": 0,\n", + " \"ls\": \"dashed\",\n", + " \"colour\": \"b\",\n", + " },\n", + " 6: {\n", + " \"dir\": work_dir + \"/cfis-shapepipe.parquet\",\n", + " \"label\": \"SP Match MegaPipe\",\n", + " \"e1_bias\": 0,\n", + " \"e2_bias\": 0,\n", + " \"ls\": \"dashdot\",\n", + " \"colour\": \"b\",\n", + " },\n", + " 7: {\n", + " \"dir\": work_dir + \"/ShapePipe/shapepipe_1500_goldshape_v1.fits\",\n", + " \"label\": \"SP_1500\",\n", + " \"e1_bias\": 7.156105098141909e-06,\n", + " \"e2_bias\": -6.00816359759969e-06,\n", + " \"ls\": \"dotted\",\n", + " \"colour\": \"b\",\n", + " },\n", + " 8: {\n", + " \"dir\": work_dir + \"/ShapePipe/unions_shapepipe_2022_v1.0.4.fits\",\n", + " \"label\": \"SP_cut_Fabian\",\n", + " \"e1_bias\": 0.0,\n", + " \"e2_bias\": 0.0,\n", + " \"ls\": \"dashdot\",\n", + " \"colour\": \"pink\",\n", + " },\n", + " 9: {\n", + " \"dir\": work_dir + \"/ShapePipe/unions_shapepipe_psf_2022_v1.0.2.fits\",\n", + " \"label\": \"SP_PSF\",\n", + " \"e1_bias\": 0.0,\n", + " \"e2_bias\": 0.0,\n", + " \"ls\": \"dashdot\",\n", + " \"colour\": \"b\",\n", + " },\n", + " 10: {\n", + " \"dir\": work_dir + \"/unions_shapepipe_2022_v1.3.fits\",\n", + " \"label\": \"SP_v1.3\",\n", + " \"e1_bias\": 0.0,\n", + " \"e2_bias\": 0.0,\n", + " \"ls\": \"dashdot\",\n", + " \"colour\": \"r\",\n", + " },\n", + " 11: {\n", + " \"dir\": work_dir + \"/ShapePipe/unions_shapepipe_star_2022_v1.3.fits\",\n", + " \"label\": \"SP_v1.3\",\n", + " \"e1_bias\": 0.0,\n", + " \"e2_bias\": 0.0,\n", + " \"ls\": \"dashdot\",\n", + " \"colour\": \"b\",\n", + " },\n", + " 12: {\n", + " \"dir\": work_dir + \"/unions_shapepipe_2024_v1.4.1.fits\",\n", + " \"label\": \"SP_v1.4.1\",\n", + " \"e1_bias\": 0.0,\n", + " \"e2_bias\": 0.0,\n", + " \"ls\": \"dashdot\",\n", + " \"colour\": \"b\",\n", + " },\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "# CATALOGUE OPTIONS:\n", + "# 1: LensFit Full\n", + "# 2: ShapePipe Full\n", + "# 3: LF Match SP\n", + "# 4: SP Match LF\n", + "# 5: SP Matched in LF footprint\n", + "# 6: SP Match MegaPipe\n", + "# 7: SP 1500deg2 (Axel's)\n", + "# 8: SP cut on large gals\n", + "# 12: SP psfex v1.4.1\n", + "\n", + "cat_options = [10, 12]\n", + "\n", + "dfs = []\n", + "\n", + "for cat_option in cat_options:\n", + " if cat_option == 6:\n", + " df = pd.read_parquet(cat_dict[cat_option][\"dir\"], engine=\"pyarrow\")\n", + " df = df.replace([np.inf, -np.inf], np.nan).dropna(axis=0)\n", + " else:\n", + " with fits.open(cat_dict[cat_option][\"dir\"]) as data:\n", + " df = pd.DataFrame(data[1].data)\n", + " if cat_option == 7:\n", + " df = df.rename(columns={\"g1\": \"e1\", \"g2\": \"e2\"})\n", + " if cat_option == 8 or cat_option == 10:\n", + " df = df.rename(columns={\"RA\": \"ra\", \"Dec\": \"dec\"})\n", + " if cat_option == 12:\n", + " df = df.rename(\n", + " columns={\"RA\": \"ra\", \"Dec\": \"dec\", \"e1\": \"e1_prev\", \"e2\": \"e2_prev\"}\n", + " )\n", + " df = df.rename(columns={\"e1_noleakage\": \"e1\", \"e2_noleakage\": \"e2\"})\n", + " dfs.append(df)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Catalogue Analysis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "for cat in cat_options:\n", + " plt.plot(df[\"ra\"], df[\"dec\"], \".\", label=cat_dict[cat][\"label\"])\n", + "plt.xlabel(\"RA [deg]\")\n", + "plt.ylabel(\"DEC [deg]\")\n", + "plt.legend(loc=\"upper right\")\n", + "# plt.savefig('plots/3500deg^2_plot.pdf',dpi=100)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# Ellipticity histograms\n", + "plt.rcParams.update({\"font.size\": 20, \"figure.figsize\": [22, 7]})\n", + "\n", + "fig, axs = plt.subplots(1, 2)\n", + "nbins = 200\n", + "\n", + "for idx, cat in enumerate(cat_options):\n", + " (n, bins, _) = axs[0].hist(\n", + " dfs[idx][\"e1\"],\n", + " bins=nbins,\n", + " density=True,\n", + " histtype=\"step\",\n", + " weights=dfs[idx][\"w\"],\n", + " label=\"e1 %s\" % cat_dict[cat][\"label\"],\n", + " )\n", + "axs[0].set_xlabel(r\"$e_1$\")\n", + "axs[0].legend()\n", + "axs[0].set_xlim([-1.5, 1.5])\n", + "\n", + "# axs[0].set_ylim([0,2e4])\n", + "\n", + "for idx, cat in enumerate(cat_options):\n", + " (n, bins, _) = axs[1].hist(\n", + " dfs[idx][\"e2\"],\n", + " bins=nbins,\n", + " density=True,\n", + " histtype=\"step\",\n", + " weights=dfs[idx][\"w\"],\n", + " label=\"e2 {}\".format(cat_dict[cat][\"label\"]),\n", + " )\n", + " print(\n", + " \"e1 sigma {}: {}\".format(\n", + " cat_dict[cat][\"label\"], np.std(dfs[idx][\"e1_noleakage\"])\n", + " )\n", + " )\n", + " print(\n", + " \"e2 sigma {}: {}\".format(\n", + " cat_dict[cat][\"label\"], np.std(dfs[idx][\"e2_noleakage\"])\n", + " )\n", + " )\n", + " print(\n", + " \"e1 bias {}: {}\".format(\n", + " cat_dict[cat][\"label\"],\n", + " np.average(\n", + " np.array(dfs[idx][\"e1_noleakage\"]), weights=np.array(dfs[idx][\"w\"])\n", + " ),\n", + " )\n", + " )\n", + " print(\n", + " \"e2 bias {}: {}\".format(\n", + " cat_dict[cat][\"label\"],\n", + " np.average(\n", + " np.array(dfs[idx][\"e2_noleakage\"]), weights=np.array(dfs[idx][\"w\"])\n", + " ),\n", + " )\n", + " )\n", + "axs[1].set_xlabel(r\"$e_2$\")\n", + "axs[1].legend()\n", + "axs[1].set_xlim([-1.5, 1.5])\n", + "# axs[1].set_ylim([0,2e4])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# Mag histograms\n", + "\n", + "plt.rcParams.update({\"font.size\": 20, \"figure.figsize\": [15, 10]})\n", + "\n", + "for idx, cat in enumerate(cat_options):\n", + " (n, bins, _) = plt.hist(\n", + " dfs[idx][\"mag\"],\n", + " bins=200,\n", + " density=False,\n", + " histtype=\"step\",\n", + " weights=dfs[idx][\"w\"],\n", + " label=\"Mag %s\" % cat_dict[cat][\"label\"],\n", + " )\n", + "\n", + "plt.xlim([19, 26])\n", + "plt.xlabel(\"Mag\")\n", + "plt.legend(loc=\"upper left\")" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": { + "tags": [] + }, + "source": [ + "## Plot n(z)'s from file\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "# nz_lf = fits.open(work_dir + '/nz/blind_nz_cfis_lensfit_goldshape_2022v1.fits')[1].data\n", + "nz = fits.open(work_dir + \"/nz/blind_nz_cfis_shapepipe_2022v1.fits\")[1].data\n", + "\n", + "# nz_lf_matched = fits.open(work_dir + '/nz/nz_masked_matched_lensfit_goldshape_2022v1.fits')[1].data\n", + "# nz_sp_matched = fits.open(work_dir + '/nz/nz_masked_matched_unions_shapepipe_extended_2022_v1.0.fits')[1].data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# FULL CATALOGUE NZ'S\n", + "from matplotlib.ticker import StrMethodFormatter\n", + "\n", + "blinds = [\"A\", \"B\", \"C\"]\n", + "\n", + "# for blind in blinds:\n", + "# z1 = nz_lf['Z_%s' %blind]\n", + "\n", + "# (n,bins,_)= plt.hist(z1, bins=200, range=(0,5.0), density=True, histtype='step', weights=None,label='LensFit Blind %s' %blind)\n", + "# # n_lf.append(list(n))\n", + "# # bins_lf.append(list(bins))\n", + "\n", + "# plt.xlabel('Redshifts')\n", + "# plt.ylabel('n(z)')\n", + "# print(\"zmin = \",min(z1))\n", + "# print(\"zmax = \",max(z1))\n", + "# plt.legend(fontsize=20)\n", + "# # plt.savefig('plots/Lensfit_nz_all_blinds.pdf' )\n", + "# plt.show()\n", + "#####################################################################################################\n", + "for blind in blinds:\n", + " z = nz[\"Z_%s\" % blind]\n", + " bins = np.linspace(0, 5, 100)\n", + "\n", + " y, edges = np.histogram(z, bins, density=True, weights=nz[\"som_w\"])\n", + " centers = 0.5 * (edges[1:] + edges[:-1])\n", + " plt.plot(centers, y, \"-o\", markersize=4, label=\"Blind %s\" % blind, alpha=0.7)\n", + "\n", + " # (n,bins,_)= plt.hist(z2, bins=50, range=(0,5.0), density=True, histtype='step',weights=nz['som_w'],label='Blind %s' %blind,alpha=0.5)\n", + " # n_sp.append(list(n))\n", + " # bins_sp.append(list(bins))\n", + "\n", + " plt.xlabel(r\"$z$\")\n", + " plt.ylabel(r\"$n(z)$\")\n", + " plt.ylim([0, 1.7])\n", + " plt.xlim([0, 5])\n", + " plt.grid(True)\n", + " plt.gca().xaxis.set_major_formatter(StrMethodFormatter(\"{x:,.1f}\"))\n", + " # print(\"zmin = \",min(z))\n", + " # print(\"zmax = \",max(z))\n", + " plt.legend(fontsize=20)\n", + " plt.savefig(\"../plots/unions_nz.pdf\", bbox_inches=\"tight\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": { + "tags": [] + }, + "source": [ + "## Compute shear-shear correlation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "# Create TreeCorr catalogue\n", + "treecorr.set_omp_threads(8)\n", + "\n", + "sep_units = \"arcmin\"\n", + "theta_min = 1\n", + "theta_max = 200\n", + "\n", + "TreeCorrConfig = {\n", + " \"ra_units\": \"degrees\",\n", + " \"dec_units\": \"degrees\",\n", + " \"max_sep\": str(theta_max),\n", + " \"min_sep\": str(theta_min),\n", + " \"sep_units\": sep_units,\n", + " \"nbins\": 20,\n", + " \"var_method\": \"jackknife\",\n", + "}\n", + "\n", + "cat_ggs = []\n", + "for idx, cat in enumerate(cat_options):\n", + " cat_gal = treecorr.Catalog(\n", + " ra=dfs[idx][\"ra\"],\n", + " dec=dfs[idx][\"dec\"],\n", + " g1=dfs[idx][\"e1\"] - cat_dict[cat][\"e1_bias\"],\n", + " g2=dfs[idx][\"e2\"] - cat_dict[cat][\"e2_bias\"],\n", + " w=dfs[idx][\"w\"],\n", + " ra_units=\"degrees\",\n", + " dec_units=\"degrees\",\n", + " npatch=50,\n", + " )\n", + " gg = treecorr.GGCorrelation(TreeCorrConfig)\n", + " gg.process(cat_gal)\n", + " cat_ggs.append(gg)\n", + " print(\"done for cat %s\" % cat)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "plt.rcParams.update({\"font.size\": 25, \"figure.figsize\": [10, 7]})\n", + "\n", + "ax1 = plt.subplot(111)\n", + "for idx, cat in enumerate(cat_options):\n", + " ax1.plot(\n", + " cat_ggs[idx].meanr,\n", + " cat_ggs[idx].npairs,\n", + " label=r\"$n_{pairs}$ %s\" % (cat_dict[cat][\"label\"]),\n", + " ls=cat_dict[cat][\"ls\"],\n", + " color=cat_dict[cat][\"colour\"],\n", + " )\n", + "ax1.set_xlabel(rf\"$\\theta$ [{sep_units}]\")\n", + "ax1.set_ylabel(r\"$n_{pairs}$\")\n", + "plt.show()\n", + "\n", + "ax2 = plt.subplot(111)\n", + "for idx, cat in enumerate(cat_options):\n", + " ax2.errorbar(\n", + " cat_ggs[idx].meanr,\n", + " cat_ggs[idx].xip,\n", + " yerr=np.sqrt(cat_ggs[idx].varxip),\n", + " label=r\"$\\xi_+$ %s\" % (cat_dict[cat][\"label\"]),\n", + " ls=cat_dict[cat][\"ls\"],\n", + " color=cat_dict[cat][\"colour\"],\n", + " )\n", + " ax2.axvspan(0, 10, color=\"gray\", alpha=0.3)\n", + " # ax2.axvspan(100,200,color='gray', alpha=0.3)\n", + "\n", + "ax2.text(\n", + " 0.85,\n", + " 0.88,\n", + " \"1,1\",\n", + " transform=ax2.transAxes,\n", + " bbox=dict(facecolor=\"white\", edgecolor=\"black\", boxstyle=\"round\", pad=0.5),\n", + ")\n", + "ax2.set_xscale(\"log\")\n", + "ax2.set_yscale(\"log\")\n", + "ax2.set_xlabel(rf\"$\\theta$ [{sep_units}]\")\n", + "ax2.set_xlim([0, 200])\n", + "_ = ax2.set_ylabel(r\"$\\xi_+(\\theta)$\")\n", + "ax2.legend(loc=\"lower left\")\n", + "# plt.savefig('../plots/xi_plus_%s.pdf' %cat_dict[cat]['label'],bbox_inches='tight')\n", + "plt.show()\n", + "\n", + "ax3 = plt.subplot(111)\n", + "for idx, cat in enumerate(cat_options):\n", + " ax3.errorbar(\n", + " cat_ggs[idx].meanr,\n", + " cat_ggs[idx].xim,\n", + " yerr=np.sqrt(cat_ggs[idx].varxim),\n", + " label=r\"$\\xi_-$ %s\" % (cat_dict[cat][\"label\"]),\n", + " ls=\"dotted\",\n", + " color=cat_dict[cat][\"colour\"],\n", + " )\n", + " ax3.axvspan(0, 20, color=\"gray\", alpha=0.3)\n", + " # ax3.axvspan(100,200,color='gray', alpha=0.3)\n", + "\n", + "ax3.text(\n", + " 0.85,\n", + " 0.88,\n", + " \"1,1\",\n", + " transform=ax3.transAxes,\n", + " bbox=dict(facecolor=\"white\", edgecolor=\"black\", boxstyle=\"round\", pad=0.5),\n", + ")\n", + "ax3.set_xscale(\"log\")\n", + "ax3.set_yscale(\"log\")\n", + "ax3.set_xlabel(rf\"$\\theta$ [{sep_units}]\")\n", + "ax3.set_xlim([0, 200])\n", + "ax3.legend(loc=\"lower left\")\n", + "_ = ax3.set_ylabel(r\"$\\xi_-(\\theta)$\")\n", + "# plt.savefig('../plots/xi_minus_%s.pdf' %cat_dict[cat]['label'],bbox_inches='tight')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "13", + "metadata": { + "tags": [] + }, + "source": [ + "## Comparison with theory PyCCL" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "nz = np.loadtxt(\n", + " \"/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/v1.0/nz/dndz_SP_v1.0_A.txt\",\n", + " usecols=1,\n", + ")\n", + "bins = np.loadtxt(\n", + " \"/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/v1.0/nz/dndz_SP_v1.0_A.txt\",\n", + " usecols=0,\n", + ")\n", + "\n", + "\n", + "def theory_cls(Omega_c, Omega_b, h, n_s, sigma_8):\n", + " # Set cosmology\n", + " cosmo = ccl.Cosmology(Omega_c, Omega_b, h, n_s, sigma_8)\n", + "\n", + " ell = np.arange(2, 2000)\n", + " theta_deg = np.logspace(\n", + " np.log10(theta_min / 60), np.log10(theta_max / 60), num=20\n", + " ) # Theta is in degrees\n", + " # CALCULATION OF THEORY XI_PM\n", + " xi_plus_lf = []\n", + " xi_minus_lf = []\n", + "\n", + " for i in range(len(nz)):\n", + " bias_ia = 0 * np.ones_like(bins[i][:-1])\n", + " lens_ia = ccl.WeakLensingTracer(\n", + " cosmo,\n", + " dndz=(np.array(bins[i][:-1]), np.array(nz[i])),\n", + " ia_bias=(np.array(bins[i][:-1]), bias_ia),\n", + " )\n", + " cl = ccl.angular_cl(cosmo, lens_ia, lens_ia, ell)\n", + "\n", + " xi_plus_lf.append(\n", + " list(\n", + " ccl.correlation(cosmo, ell, cl, theta_deg, type=\"GG+\", method=\"FFTLog\")\n", + " )\n", + " )\n", + " xi_minus_lf.append(\n", + " list(\n", + " ccl.correlation(cosmo, ell, cl, theta_deg, type=\"GG-\", method=\"FFTLog\")\n", + " )\n", + " )\n", + "\n", + " style = [\":\", \"--\", \"-.\"]\n", + " plt.errorbar(\n", + " gg.meanr,\n", + " gg.xip,\n", + " yerr=np.sqrt(gg.varxip),\n", + " ls=\"\",\n", + " label=r\"$\\xi_+$ TreeCorr (LF)\",\n", + " capsize=5,\n", + " marker=\"o\",\n", + " color=\"b\",\n", + " )\n", + " plt.errorbar(\n", + " gg.meanr,\n", + " gg.xim,\n", + " yerr=np.sqrt(gg.varxim),\n", + " ls=\"\",\n", + " label=r\"$\\xi_-$ TreeCorr (LF)\",\n", + " capsize=5,\n", + " marker=\"o\",\n", + " color=\"g\",\n", + " )\n", + "\n", + " for i in range(len(blinds)):\n", + " plt.plot(\n", + " theta_deg * 60,\n", + " xi_plus_lf[i],\n", + " color=\"b\",\n", + " ls=style[i],\n", + " label=r\"$\\xi_+$ PyCCL (LF) blind %s\" % blinds[i],\n", + " )\n", + " plt.plot(\n", + " theta_deg * 60,\n", + " xi_minus_lf[i],\n", + " color=\"g\",\n", + " ls=style[i],\n", + " label=r\"$\\xi_-$ PyCCL (LF) blind %s\" % blinds[i],\n", + " )\n", + "\n", + " plt.xscale(\"log\")\n", + " # plt.yscale('log')\n", + " plt.legend(fontsize=20)\n", + " plt.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", + " plt.xlim([1, 200])\n", + " plt.ylim([0, 10e-5])\n", + " plt.ylabel(r\"$\\xi_\\pm(\\theta)$\")\n", + " plt.xlabel(r\"$\\theta$ [arcmin]\")\n", + " # plt.savefig('plots/pyccl_comparison_lensfit.pdf')\n", + "\n", + "\n", + "interact(\n", + " theory_cls,\n", + " Omega_c=widgets.FloatSlider(\n", + " value=0.26, min=0.01, max=0.5, step=0.01, description=r\"$\\Omega_c$\"\n", + " ),\n", + " Omega_b=widgets.FloatSlider(\n", + " value=0.04, min=0.001, max=0.07, step=0.001, description=r\"$\\Omega_b$\"\n", + " ),\n", + " h=widgets.FloatSlider(value=0.7, min=0.3, max=0.9, step=0.01, description=r\"$h$\"),\n", + " n_s=widgets.FloatSlider(\n", + " value=0.96, min=0.6, max=1.1, step=0.01, description=r\"$n_s$\"\n", + " ),\n", + " sigma_8=widgets.FloatSlider(\n", + " value=0.8, min=0.3, max=1.2, step=0.01, description=r\"$\\sigma_8$\"\n", + " ),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "15", + "metadata": { + "tags": [] + }, + "source": [ + "## Plot varxipm's\n", + "Error bars are computed by treecorr, either through the 'shot' or 'jackknife' method." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "for idx, cat in enumerate(cat_options):\n", + " blind = \"A\"\n", + " label = \"SP_v1.4\"\n", + "\n", + " cc = \"/n23data1/n06data/lgoh/scratch/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/data/{}/covs/cov_{}\".format(\n", + " label + \"_{}\".format(blind), label\n", + " )\n", + "\n", + " cc_var = np.diag(np.loadtxt(cc + \".txt\"))\n", + " cc_varxip = cc_var[:20]\n", + " cc_varxim = cc_var[20:]\n", + "\n", + " cc_var = np.diag(np.loadtxt(cc + \"_g.txt\"))\n", + " cc_varxip_g = cc_var[:20]\n", + " cc_varxim_g = cc_var[20:]\n", + "\n", + " plt.loglog(\n", + " cat_ggs[idx].meanr,\n", + " cat_ggs[idx].varxip,\n", + " \"-k\",\n", + " label=r\"$\\sigma(\\xi_+)$ TreeCorr jackknife %s\" % cat_dict[cat][\"label\"],\n", + " )\n", + " plt.loglog(\n", + " cat_ggs[idx].meanr,\n", + " cc_varxip,\n", + " ls=\"--\",\n", + " c=\"%s\" % cat_dict[cat][\"colour\"],\n", + " label=r\"$\\sigma(\\xi_+)$ CosmoCov %s\" % cat_dict[cat][\"label\"],\n", + " )\n", + " plt.loglog(\n", + " cat_ggs[idx].meanr,\n", + " cc_varxip_g,\n", + " ls=\":\",\n", + " c=\"%s\" % cat_dict[cat][\"colour\"],\n", + " label=r\"$\\sigma(\\xi_+)$ CosmoCov Gaussian %s\" % cat_dict[cat][\"label\"],\n", + " )\n", + " plt.grid()\n", + " plt.xlim([cat_ggs[idx].meanr[0], cat_ggs[idx].meanr[-1]])\n", + " plt.legend(fontsize=15)\n", + " plt.xlabel(rf\"$\\theta$ [{sep_units}]\")\n", + " plt.ylabel(r\"$\\sigma(\\xi_+)$\")\n", + " plt.show()\n", + " # plt.savefig()\n", + "\n", + " plt.loglog(\n", + " cat_ggs[idx].meanr,\n", + " cat_ggs[idx].varxim,\n", + " \"-k\",\n", + " label=r\"$\\sigma(\\xi_-)$ TreeCorr jackknife %s\" % cat_dict[cat][\"label\"],\n", + " )\n", + " plt.loglog(\n", + " cat_ggs[idx].meanr,\n", + " cc_varxim,\n", + " ls=\"--\",\n", + " c=\"%s\" % cat_dict[cat][\"colour\"],\n", + " label=r\"$\\sigma(\\xi_-)$ CosmoCov (SP) %s\" % cat_dict[cat][\"label\"],\n", + " )\n", + " plt.loglog(\n", + " cat_ggs[idx].meanr,\n", + " cc_varxim_g,\n", + " ls=\":\",\n", + " c=\"%s\" % cat_dict[cat][\"colour\"],\n", + " label=r\"$\\sigma(\\xi_-)$ CosmoCov (SP) Gaussian %s\" % cat_dict[cat][\"label\"],\n", + " )\n", + " plt.grid()\n", + " plt.xlim([cat_ggs[idx].meanr[0], cat_ggs[idx].meanr[-1]])\n", + " plt.legend(fontsize=15)\n", + " plt.xlabel(rf\"$\\theta$ [{sep_units}]\")\n", + " plt.ylabel(r\"$\\sigma(\\xi_-)$\")\n", + " plt.show()\n", + " # plt.savefig()" + ] + }, + { + "cell_type": "markdown", + "id": "17", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "## Run systematic tests" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### C_sys" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "# CALCULATE XI_SYS FOR SHAPEPIPE\n", + "\n", + "sep_units = \"arcmin\"\n", + "theta_min = 1\n", + "theta_max = 200\n", + "\n", + "TreeCorrConfig = {\n", + " \"ra_units\": \"degrees\",\n", + " \"dec_units\": \"degrees\",\n", + " \"max_sep\": str(theta_max),\n", + " \"min_sep\": str(theta_min),\n", + " \"sep_units\": sep_units,\n", + " \"nbins\": 20,\n", + " \"var_method\": \"jackknife\",\n", + "}\n", + "\n", + "with fits.open(cat_dict[11][\"dir\"]) as data:\n", + " df_psf = pd.DataFrame(data[1].data)\n", + "\n", + "cat_psf = treecorr.Catalog(\n", + " ra=df_psf[\"RA\"],\n", + " dec=df_psf[\"DEC\"],\n", + " g1=df_psf[\"HSM_G1_PSF\"],\n", + " g2=df_psf[\"HSM_G2_PSF\"],\n", + " ra_units=\"degrees\",\n", + " dec_units=\"degrees\",\n", + " npatch=50,\n", + ")\n", + "\n", + "gg_psf = treecorr.GGCorrelation(TreeCorrConfig)\n", + "gg_psf.process(cat_psf)\n", + "\n", + "ggs_psf_star = []\n", + "for idx, cat in enumerate(cat_options):\n", + " cat_gal = treecorr.Catalog(\n", + " ra=dfs[idx][\"ra\"],\n", + " dec=dfs[idx][\"dec\"],\n", + " g1=dfs[idx][\"e1\"] - cat_dict[cat][\"e1_bias\"],\n", + " g2=dfs[idx][\"e2\"] - cat_dict[cat][\"e2_bias\"],\n", + " w=dfs[idx][\"w\"],\n", + " ra_units=\"degrees\",\n", + " dec_units=\"degrees\",\n", + " npatch=50,\n", + " )\n", + " gg_psf_star = treecorr.GGCorrelation(TreeCorrConfig)\n", + " gg_psf_star.process(cat_gal, cat_psf)\n", + " ggs_psf_star.append(gg_psf_star)\n", + "\n", + " print(\"done for cat %s\" % cat)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "for idx, cat in enumerate(cat_options):\n", + " C_sys_xip = gg_psf.xip\n", + " C_sys_xim = gg_psf.xim\n", + "\n", + " # delta_C_sys_xip = C_sys_xip*np.sqrt((2*np.sqrt(ggs_psf_star[idx].varxip)/ggs_psf_star[idx].xip)**2+(np.sqrt(gg_psf.varxip)/gg_psf.xip)**2)\n", + " # delta_C_sys_xim = C_sys_xim*np.sqrt((2*np.sqrt(ggs_psf_star[idx].varxim)/ggs_psf_star[idx].xim)**2+(np.sqrt(gg_psf.varxim)/gg_psf.xim)**2)\n", + "\n", + " plt.errorbar(\n", + " ggs_psf_star[idx].meanr,\n", + " C_sys_xip,\n", + " yerr=0,\n", + " label=r\"$(\\xi^{sys}_+)$ Catalogue %s\" % cat_dict[cat][\"label\"],\n", + " ls=cat_dict[cat][\"ls\"],\n", + " color=cat_dict[cat][\"colour\"],\n", + " )\n", + " plt.legend()\n", + " plt.xlabel(r\"$\\theta[arcmin]$\")\n", + " plt.ylabel(r\"$\\xi^{sys}_\\pm$\")\n", + " # plt.ylim([-2e-8,2e-8])\n", + " plt.xscale(\"log\")\n", + " plt.ticklabel_format(style=\"sci\", axis=\"y\", scilimits=(0, 0))\n", + " plt.grid(True)\n", + "\n", + " # plt.errorbar(ggs_psf_star[idx].meanr, C_sys_xim, yerr=delta_C_sys_xim, label=r'$(\\xi^{sys}_-)$ Catalogue %s'%cat_dict[cat]['label'],color='g')\n", + " # plt.legend()\n", + " # plt.xlabel(r'$\\theta[arcmin]$')\n", + " # plt.ylabel(r'$\\xi^{sys}_\\pm$')\n", + " # plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))\n", + " # # plt.ylim([-2e-8,2e-8])\n", + " # plt.xscale('log')\n", + " # plt.grid(True)" + ] + }, + { + "cell_type": "markdown", + "id": "21", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### M_ap" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", + "metadata": {}, + "outputs": [], + "source": [ + "for idx, cat in enumerate(cat_options):\n", + " R = cat_ggs[idx].rnom\n", + "\n", + " (Map_lf, mapsq_im_lf, Mx_lf, mxsq_im_lf, varMapsq_lf) = cat_ggs[idx].calculateMapSq(\n", + " R=R, m2_uform=\"Schneider\"\n", + " )\n", + " (Map_sp, mapsq_im_sp, Mx_sp, mxsq_im_sp, varMapsq_sp) = cat_ggs[idx].calculateMapSq(\n", + " R=R, m2_uform=\"Schneider\"\n", + " )\n", + "\n", + " plt.errorbar(\n", + " R,\n", + " Map_lf,\n", + " yerr=np.sqrt(varMapsq_lf),\n", + " label=r\"$$ {}\".format(cat_dict[cat][\"label\"]),\n", + " ls=\":\",\n", + " color=\"b\",\n", + " )\n", + " plt.errorbar(\n", + " R,\n", + " Mx_lf,\n", + " yerr=np.sqrt(varMapsq_lf),\n", + " label=r\"$$ {}\".format(cat_dict[cat][\"label\"]),\n", + " ls=\":\",\n", + " color=\"r\",\n", + " )\n", + " plt.axhline(y=0, xmin=0, xmax=200, color=\"k\")\n", + " plt.xlabel(r\"$\\theta[arcmin]$\")\n", + " plt.ylabel(r\"$$\")\n", + " plt.xscale(\"log\")\n", + " plt.ylim([-2e-5, 1e-5])\n", + " # plt.xlim([1,200])\n", + " plt.grid(True)\n", + " plt.legend()\n", + "\n", + " plt.errorbar(\n", + " R,\n", + " Map_sp,\n", + " yerr=np.sqrt(varMapsq_sp),\n", + " label=r\"$$ {}\".format(cat_dict[cat][\"label\"]),\n", + " ls=\":\",\n", + " color=\"b\",\n", + " )\n", + " plt.errorbar(\n", + " R,\n", + " Mx_sp,\n", + " yerr=np.sqrt(varMapsq_sp),\n", + " label=r\"$$ {}\".format(cat_dict[cat][\"label\"]),\n", + " ls=\":\",\n", + " color=\"r\",\n", + " )\n", + " plt.axhline(y=0, xmin=0, xmax=200, color=\"k\")\n", + " plt.xscale(\"log\")\n", + " plt.xlabel(r\"$\\theta[arcmin]$\")\n", + " plt.ylabel(r\"$$\")\n", + " plt.ticklabel_format(style=\"sci\", axis=\"y\", scilimits=(0, 0))\n", + " plt.ylim([-2e-5, 1e-5])\n", + " plt.grid(True)\n", + " plt.legend()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "24", + "metadata": {}, + "source": [ + "## Plot Covariance Matrix" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25", + "metadata": {}, + "outputs": [], + "source": [ + "from numpy import linalg as LA\n", + "\n", + "%matplotlib inline\n", + "\n", + "\n", + "def get_cov(filename):\n", + "\n", + " data = np.loadtxt(filename)\n", + " ndata = int(np.max(data[:, 0])) + 1\n", + "\n", + " print(\"Dimension of cov: %dx%d\" % (ndata, ndata))\n", + "\n", + " # ndata_min = int(np.min(data[:,0]))\n", + " cov_g = np.zeros((ndata, ndata))\n", + " cov_ng = np.zeros((ndata, ndata))\n", + " for i in range(0, data.shape[0]):\n", + " cov_g[int(data[i, 0]), int(data[i, 1])] = data[i, 8]\n", + " cov_g[int(data[i, 1]), int(data[i, 0])] = data[i, 8]\n", + " cov_ng[int(data[i, 0]), int(data[i, 1])] = data[i, 9]\n", + " cov_ng[int(data[i, 1]), int(data[i, 0])] = data[i, 9]\n", + "\n", + " return cov_g, cov_ng, ndata\n", + "\n", + "\n", + "covfile = \"/feynman/work/dap/lcs/lg268561/UNIONS/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/data/SP_cut_Fabian/covs/out_cov_ssss_+-_cov_Ntheta20_Ntomo1_3\"\n", + "\n", + "c_g, c_ng, ndata = get_cov(covfile)\n", + "cov = c_ng + c_g\n", + "cov_g = c_g\n", + "\n", + "b = np.sort(LA.eigvals(cov))\n", + "print(\"min+max eigenvalues cov: %e, %e\" % (np.min(b), np.max(b)))\n", + "if np.min(b) <= 0.0:\n", + " print(\"non-positive eigenvalue encountered! Covariance Invalid!\")\n", + " exit()\n", + "\n", + "print(\"Covariance is postive definite!\")\n", + "\n", + "pp_var = []\n", + "for i in range(ndata):\n", + " pp_var.append(cov[i][i])\n", + "\n", + "\n", + "cmap = \"seismic\"\n", + "\n", + "pp_norm = np.zeros((ndata, ndata))\n", + "for i in range(ndata):\n", + " for j in range(ndata):\n", + " pp_norm[i][j] = cov[i][j] / np.sqrt(cov[i][i] * cov[j][j])\n", + "\n", + "print(\"Plotting correlation matrix ...\")\n", + "\n", + "# plot_path = covfile+'_plot.pdf'\n", + "\n", + "fig = plt.figure()\n", + "ax = fig.add_subplot(1, 1, 1)\n", + "ax.xaxis.tick_top()\n", + "ax.xaxis.set_ticks(np.arange(0, 41, 1))\n", + "ax.yaxis.set_ticks(np.arange(0, 41, 1))\n", + "\n", + "\n", + "plt.axvline(x=19.5, color=\"black\", linewidth=1.5)\n", + "plt.axhline(y=19.5, color=\"black\", linewidth=1.5)\n", + "\n", + "\n", + "im3 = ax.imshow(pp_norm, cmap=cmap, vmin=-1, vmax=1)\n", + "ax.get_xaxis().set_ticklabels([])\n", + "ax.get_yaxis().set_ticklabels([])\n", + "cbar = fig.colorbar(im3, orientation=\"vertical\", shrink=0.6, ticks=[-1, 0, 1])\n", + "cbar.ax.tick_params(labelsize=15)\n", + "cbar.ax.set_yticklabels([r\"$-1$\", r\"$0$\", r\"$1$\"])\n", + "\n", + "ax.text(8, -2, r\"$\\xi_+^{ij}(\\theta)$\", fontsize=22)\n", + "ax.text(30, -2, r\"$\\xi_-^{ij}(\\theta)$\", fontsize=22)\n", + "ax.text(-6, 10, r\"$\\xi_+^{ij}(\\theta)$\", fontsize=22)\n", + "ax.text(-6, 30, r\"$\\xi_-^{ij}(\\theta)$\", fontsize=22)\n", + "# ax.set_title('Blind A',fontsize=15)\n", + "\n", + "plt.savefig(\"../plots/unions_covmat.pdf\", bbox_inches=\"tight\")\n", + "\n", + "\n", + "plt.show()\n", + "# print(\"Plot saved as %s\"%(plot_path))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "my_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosmo_inference/notebooks/cfis_mcmc.ipynb b/cosmo_inference/notebooks/cfis_mcmc.ipynb new file mode 100644 index 00000000..124eaf99 --- /dev/null +++ b/cosmo_inference/notebooks/cfis_mcmc.ipynb @@ -0,0 +1,1546 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from getdist import plots\n", + "\n", + "# import uncertainties\n", + "\n", + "plt.rc(\"mathtext\", fontset=\"stix\")\n", + "plt.rc(\"font\", family=\"sans-serif\")\n", + "\n", + "g = plots.get_subplot_plotter(width_inch=30)\n", + "g.settings.axes_fontsize = 30\n", + "g.settings.axes_labelsize = 30\n", + "g.settings.alpha_filled_add = 0.7\n", + "g.settings.legend_fontsize = 40\n", + "\n", + "\n", + "# SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE\n", + "root_dir = \"/n09data/guerrini/output_chains/\"\n", + "\n", + "\"\"\" lower_bound = ['3.0', '3.0', '3.0', '3.0', '3.0', '10.0', '10.0']\n", + "upper_bound = ['200.0', '150.0', '100.0', '80.0', '60.0', '150.0', '60.0']\n", + "roots = [\n", + " f'SP_v1.4.5_leak_corr_sc_{lc}_{hc}_10.0_200.0' for lc, hc in zip(lower_bound, upper_bound)\n", + " ] \"\"\"\n", + "\n", + "roots = [\n", + " \"SP_v1.4.5_glass_mock_1\",\n", + " \"SP_v1.4.5_glass_mock_1_takahashi\",\n", + " \"SP_v1.4.5_glass_mock_1_HM_code\",\n", + "]\n", + "\n", + "roots = [\n", + " \"SP_v1.4.5_A\",\n", + " # \"SP_v1.4.5_A_no_IA\",\n", + " # \"SP_v1.4.5_A_no_dz\",\n", + " # \"SP_v1.4.5_A_no_m_bias\",\n", + " \"SP_v1.4.5_A_sc_3_150\",\n", + " \"SP_v1.4.5_A_sc_3_60\",\n", + " # \"SP_v1.4.5_A_sc_10_150\",\n", + " # \"SP_v1.4.5_A_sc_10_60\",\n", + " # \"SP_v1.4.5_A_sc_5_150\",\n", + " # \"SP_v1.4.5_A_sc_7_150\",\n", + " \"SP_v1.4.5_A_no_leakage\",\n", + " \"SP_v1.4.5_A_no_leakage_150\",\n", + " \"SP_v1.4.5_A_no_leakage_60\",\n", + "]\n", + "\n", + "\"\"\" roots = [\n", + " f\"SP_v1.4.5_glass_mock_{i}\" for i in range(1, 17)\n", + "] \"\"\"\n", + "\n", + "\"\"\" roots = [\n", + " \"SP_v1.4.5_glass_mock_A_IA_m5_5\",\n", + " \"SP_v1.4.5_glass_mock_A_IA_G_0.57_0.5\",\n", + "] \"\"\"\n", + "\n", + "\n", + "roots = [\n", + " f\"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_{int(i)}.0_80.0_10.0_80.0\"\n", + " for i in [3, 5, 7, 10, 11]\n", + "]\n", + "\n", + "roots = [\n", + " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0\",\n", + " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0_no_alpha_beta\",\n", + "]\n", + "\n", + "roots = [\n", + " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0\",\n", + " \"SP_v1.4.5_leak_corr_cell\",\n", + "]\n", + "\n", + "print(roots)" + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Retrieve the chains" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "# MAKE PARAMNAMES FILE\n", + "\n", + "for root in roots:\n", + " with open(root_dir + \"{}/samples_{}.txt\".format(\"/\" + root, root), \"r\") as file:\n", + " params = file.readline()[1:].split(\"\\t\")[:-4]\n", + " file.close()\n", + "\n", + " with open(\n", + " root_dir + \"{}/getdist_{}.paramnames\".format(\"/\" + root, root), \"w\"\n", + " ) as file:\n", + " for i in range(len(params)):\n", + " if len(params[i].split(\"--\")) > 1:\n", + " file.write(params[i].split(\"--\")[1] + \"\\n\")\n", + " else:\n", + " file.write(params[i].split(\"--\")[0] + \"\\n\")\n", + " file.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "# READ CHAIN\n", + "\n", + "chains = []\n", + "\n", + "for root in roots:\n", + " samples = np.loadtxt(root_dir + \"{}/samples_{}.txt\".format(root, root))\n", + " print(len(samples))\n", + " if \"nautilus\" in root:\n", + " samples = np.column_stack(\n", + " (np.exp(samples[:, -3]), samples[:, -1] - samples[:, -2], samples[:, 0:-3])\n", + " )\n", + " else:\n", + " samples = np.column_stack((samples[:, -1], samples[:, -3], samples[:, 0:-4]))\n", + " np.savetxt(root_dir + \"{}/getdist_{}.txt\".format(root, root), samples)\n", + "\n", + " chain = g.samples_for_root(\n", + " root_dir + \"{}/getdist_{}\".format(root, root),\n", + " cache=False,\n", + " settings={\"ignore_rows\": 0, \"smooth_scale_2D\": 0.5, \"smooth_scale_1D\": 0.5},\n", + " )\n", + "\n", + " chains.append(chain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "name_list = [\n", + " \"OMEGA_M\",\n", + " \"ombh2\",\n", + " \"h0\",\n", + " \"n_s\",\n", + " \"SIGMA_8\",\n", + " \"s_8_input\",\n", + " \"logt_agn\",\n", + " \"a\",\n", + " \"m1\",\n", + " \"bias_1\",\n", + "] # ,'alpha','beta']\n", + "label_list = [\n", + " r\"\\Omega_m\",\n", + " r\"\\omega_b h^2\",\n", + " \"h_0\",\n", + " \"n_s\",\n", + " r\"\\sigma_8\",\n", + " \"S_8\",\n", + " \"log T_{AGN}\",\n", + " \"A_{IA}\",\n", + " \"m_1\",\n", + " r\"\\Delta z_1\",\n", + "] # , '\\\\alpha_{PSF}', '\\\\beta_{PSF}']\n", + "\n", + "for chain in chains:\n", + " param_names = chain.getParamNames()\n", + " for name, label in zip(name_list, label_list):\n", + " param_names.parWithName(name).label = label" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "## Plot the chain" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "\"\"\" legend_labels = [\n", + " rf'$\\theta \\\\in$ [{lc}-{hc}]' for lc, hc in zip(lower_bound, upper_bound)\n", + "] \"\"\"\n", + "\n", + "legend_labels = [rf\"GLASS mock {i}\" for i in range(1, 17)]\n", + "\n", + "legend_labels = [\"GLASS mock 1\", \"GLASS mock 1 takahashi\", \"GLASS mock 1 HM code\"]\n", + "\n", + "legend_labels = [\n", + " \"SP_v1.4.5 blind A\",\n", + " # \"SP_v1.4.5 blind A no IA\",\n", + " # r\"SP_v1.4.5 blind A no $\\Delta z$\",\n", + " # r\"SP_v1.4.5 blind A no $m_1$\",\n", + " r\"SP_v1.4.5 blind A, $\\theta \\in [3-150]$\",\n", + " r\"SP_v1.4.5 blind A, $\\theta \\in [3-60]$\",\n", + " # r\"SP_v1.4.5 blind A, $\\theta \\in [10-150]$\",\n", + " # r\"SP_v1.4.5 blind A, $\\theta \\in [10-60]$\",\n", + " # r\"SP_v1.4.5 blind A, $\\theta \\in [5-150]$\",\n", + " # r\"SP_v1.4.5 blind A, $\\theta \\in [7-150]$\",\n", + " r\"SP_v1.4.5 blind A no leakage\",\n", + " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [3-150]$\",\n", + " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [3-60]$\",\n", + "]\n", + "\n", + "legend_labels = [\n", + " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [3-80]$\",\n", + " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [5-180]$\",\n", + " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [7-180]$\",\n", + " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [10-80]$\",\n", + " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [11-80]$\",\n", + "]\n", + "\n", + "legend_labels = [\n", + " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [10-80]$\",\n", + " r\"SP_v1.4.5 blind A no leakage, $C_\\ell$\",\n", + "]\n", + "\n", + "contour_colors = [\n", + " \"cornflowerblue\",\n", + " \"salmon\",\n", + " \"darkorange\",\n", + " \"forestgreen\",\n", + " \"turquoise\",\n", + " \"darkviolet\",\n", + " \"crimson\",\n", + " \"gold\",\n", + " \"lightcoral\",\n", + " \"mediumseagreen\",\n", + " \"lightsteelblue\",\n", + " \"black\",\n", + " \"silver\",\n", + " \"peru\",\n", + " \"maroon\",\n", + " \"olive\",\n", + "]\n", + "\n", + "\"\"\" legend_labels = [\n", + " r\"GLASS mock 3\",\n", + " r\"GLASS mock 3 no PSF\",\n", + " r\"GLASS mock 3 no PSF baryons\",\n", + "] \"\"\"\n", + "\n", + "marker = {\n", + " \"OMEGA_LAMBDA\": 0.7013160542257656,\n", + " \"ombh2\": 0.024499999999999997,\n", + " \"omch2\": 0.12249999999999998,\n", + " \"h0\": 0.70,\n", + " \"n_s\": 0.96,\n", + " \"SIGMA_8\": 0.793897,\n", + " \"s_8_input\": 0.79563645,\n", + " \"m1\": 0.0,\n", + " \"bias_1\": 0.0,\n", + " #'alpha': -0.0005,\n", + " #'beta': 0.0631,\n", + " \"a\": 0.0,\n", + "}\n", + "\n", + "marker = {\n", + " \"bias_1\": -0.045,\n", + " \"m1\": 0.0,\n", + " \"a\": 0.5,\n", + " #'alpha': 0.0169,\n", + " #'beta': 1.0789\n", + "}\n", + "g.triangle_plot(\n", + " chains,\n", + " [\n", + " \"OMEGA_M\",\n", + " \"ombh2\",\n", + " \"h0\",\n", + " \"n_s\",\n", + " \"SIGMA_8\",\n", + " \"s_8_input\",\n", + " \"logt_agn\",\n", + " \"a\",\n", + " \"m1\",\n", + " \"bias_1\",\n", + " \"alpha\",\n", + " \"beta\",\n", + " ],\n", + " legend_labels=legend_labels,\n", + " legend_loc=\"upper right\",\n", + " # param_limits={'bias_1':[-0.8,0.5]},\n", + " contour_colors=contour_colors,\n", + " line_args=[{\"color\": contour_colors[i], \"ls\": \"solid\"} for i in range(16)],\n", + " # title_limit=1,\n", + " filled=True,\n", + " markers=marker,\n", + ")\n", + "\n", + "g.export(\"contour_plot_unions_cell.png\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\" legend_labels = [\n", + " rf'$\\theta \\\\in$ [{lc}-{hc}]' for lc, hc in zip(lower_bound, upper_bound)\n", + "] \"\"\"\n", + "g.triangle_plot(\n", + " chains,\n", + " [\"OMEGA_M\", \"s_8_input\", \"SIGMA_8\", \"a\"],\n", + " legend_labels=legend_labels,\n", + " legend_loc=\"upper right\",\n", + " # param_limits={'bias_1':[-0.8,0.5]},\n", + " contour_colors=contour_colors,\n", + " line_args=[{\"color\": contour_colors[i], \"ls\": \"solid\"} for i in range(16)],\n", + " title_limit=1,\n", + " filled=True,\n", + " markers=marker,\n", + ")\n", + "\n", + "g.export(\"contour_plot_s8_unions_cell.png\")" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### Output bestfit and sigma values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "#########BESTFIT AND SIGMA VALS##########\n", + "params = [\n", + " \"OMEGA_M\",\n", + " \"omega_b\",\n", + " \"h0\",\n", + " \"n_s\",\n", + " \"a_s\",\n", + " \"SIGMA_8\",\n", + " \"S_8\",\n", + " \"logt_agn\",\n", + " \"a\",\n", + " \"m1\",\n", + " \"bias_1\",\n", + " \"alpha\",\n", + " \"beta\",\n", + " \"omch2\",\n", + " \"ombh2\",\n", + "]\n", + "latex_params = [\n", + " r\"$\\Omega_{\\rm m,0}$\",\n", + " r\"$\\Omega_{\\rm b,0}$\",\n", + " r\"$h$\",\n", + " r\"$n_{\\rm s}$\",\n", + " r\"$A_{\\rm s}$\",\n", + " r\"$\\sigma_8$\",\n", + " r\"$S_8$\",\n", + " r\"$\\log_{10}{T_{\\rm AGN}}$\",\n", + " r\"$\\mathcal{A}_rm IA}$\",\n", + " r\"$m_1$\",\n", + " r\"$\\Delta z$\",\n", + " r\"$\\alpha$\",\n", + " r\"$\\beta$\",\n", + " r\"$\\Omega_{\\rm c,0}$\",\n", + " r\"$\\Omega_{\\rm b,0}$\",\n", + "]\n", + "\n", + "for chain in chains:\n", + " margestats = chain.getMargeStats()\n", + " likestats = chain.getLikeStats()\n", + " p = chain.getParams()\n", + "\n", + " for no in range(len(latex_params)):\n", + " if hasattr(p, params[no]):\n", + " param_stats = margestats.parWithName(params[no])\n", + " a = np.array(\n", + " [\n", + " param_stats.mean,\n", + " param_stats.mean - param_stats.limits[0].lower,\n", + " param_stats.limits[0].upper - param_stats.mean,\n", + " ]\n", + " )\n", + " if \"%.2g\" % a[1] == \"%.2g\" % a[2]:\n", + " latex_params[no] += r\"&$%.3g\\pm%.2g$\" % (a[0], a[1])\n", + " else:\n", + " latex_params[no] += \"&$%.3g_{-%.2g}^{+%.2g}$\" % (a[0], a[1], a[2])\n", + " else:\n", + " latex_params[no] += \"&$-$\"\n", + "\n", + "\n", + "for param in latex_params:\n", + " param += r\"\\\\\"\n", + " print(param)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "chain = chains[0]\n", + "\n", + "margestats = chain.getMargeStats()\n", + "likestats = chain.getLikeStats()\n", + "p = chain.getParams()\n", + "\n", + "for no in range(len(latex_params)):\n", + " if hasattr(p, params[no]):\n", + " param_stats = margestats.parWithName(params[no])\n", + " a = np.array([param_stats.mean])\n", + " print(params[no], a[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "#########BESTFIT AND SIGMA VALS##########\n", + "params = [\n", + " \"OMEGA_M\",\n", + " \"omega_b\",\n", + " \"h0\",\n", + " \"n_s\",\n", + " \"a_s\",\n", + " \"SIGMA_8\",\n", + " \"S_8\",\n", + " \"logt_agn\",\n", + " \"a\",\n", + " \"m1\",\n", + " \"bias_1\",\n", + " \"alpha\",\n", + " \"beta\",\n", + " \"omch2\",\n", + " \"ombh2\",\n", + "]\n", + "latex_params = [\n", + " r\"$\\Omega_{\\rm m,0}$\",\n", + " r\"$\\Omega_{\\rm b,0}$\",\n", + " r\"$h$\",\n", + " r\"$n_{\\rm s}$\",\n", + " r\"$A_{\\rm s}$\",\n", + " r\"$\\sigma_8$\",\n", + " r\"$S_8$\",\n", + " r\"$\\log_{10}{T_{\\rm AGN}}$\",\n", + " r\"$\\mathcal{A}_rm IA}$\",\n", + " r\"$m_1$\",\n", + " r\"$\\Delta z$\",\n", + " r\"$\\alpha$\",\n", + " r\"$\\beta$\",\n", + " r\"$\\Omega_{\\rm c,0}$\",\n", + " r\"$\\Omega_{\\rm b,0}$\",\n", + "]\n", + "\n", + "values = {param: [] for param in params}\n", + "for i, chain in enumerate(chains):\n", + " margestats = chain.getMargeStats()\n", + " likestats = chain.getLikeStats()\n", + " p = chain.getParams()\n", + "\n", + " print(legend_labels[i])\n", + "\n", + " for param in params:\n", + " if hasattr(p, param):\n", + " param_stats = margestats.parWithName(param)\n", + " a = np.array(\n", + " [\n", + " param_stats.mean,\n", + " param_stats.mean - param_stats.limits[0].lower,\n", + " param_stats.limits[0].upper - param_stats.mean,\n", + " ]\n", + " )\n", + " print(f\"{param}: {a[0]:.3g}_-{a[1]:.2g}^+{a[1]:.2g}\")\n", + " values[param].append(a[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "bestfit_ix = np.argmax(chains[0].loglikes)\n", + "maxlike = chains[0].loglikes[bestfit_ix]\n", + "print(maxlike)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "chains[0].loglikes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "print(chains[0].likeStats)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(15, 5))\n", + "\n", + "plt.subplot(131)\n", + "\n", + "plt.hist(values[\"OMEGA_M\"], bins=10, color=\"cornflowerblue\", alpha=0.5)\n", + "plt.axvline(0.301316, color=\"black\", linestyle=\"--\", label=\"True value\")\n", + "plt.xlabel(r\"$\\Omega_{\\rm m,0}$\")\n", + "plt.ylabel(\"Counts\")\n", + "plt.legend()\n", + "\n", + "plt.subplot(132)\n", + "\n", + "plt.hist(values[\"SIGMA_8\"], bins=10, color=\"cornflowerblue\", alpha=0.5)\n", + "plt.axvline(0.793897, color=\"black\", linestyle=\"--\", label=\"True value\")\n", + "plt.xlabel(r\"$\\sigma_8$\")\n", + "plt.ylabel(\"Counts\")\n", + "plt.legend()\n", + "\n", + "plt.subplot(133)\n", + "\n", + "plt.hist(values[\"S_8\"], bins=10, color=\"cornflowerblue\", alpha=0.5)\n", + "plt.axvline(0.79563645, color=\"black\", linestyle=\"--\", label=\"True value\")\n", + "plt.xlabel(r\"$S_8$\")\n", + "plt.ylabel(\"Counts\")\n", + "plt.legend()\n", + "plt.tight_layout()\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "np.sum(np.abs(np.array(values[\"S_8\"]) - 0.79563645) < 0.03) / len(values[\"S_8\"])" + ] + }, + { + "cell_type": "markdown", + "id": "17", + "metadata": {}, + "source": [ + "## Looking at best fit" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "from astropy.io import fits\n", + "\n", + "version = \"SP_v1.4.5_glass_mock_1\"\n", + "\n", + "data = fits.open(\n", + " f\"/home/guerrini/sp_validation/cosmo_inference/data/{version}/cosmosis_{version}.fits\"\n", + ")\n", + "xi_plus = data[\"XI_PLUS\"].data\n", + "xi_minus = data[\"XI_MINUS\"].data\n", + "cov_mat = data[\"COVMAT\"].data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(15, 15))\n", + "\n", + "plt.subplot(211)\n", + "\n", + "plt.errorbar(\n", + " xi_plus[\"ANG\"],\n", + " xi_plus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + ")\n", + "\n", + "plt.ylabel(r\"$\\xi_{+}$\")\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.axvline(3.0, color=\"grey\", linestyle=\"--\", label=\"3 arcmin\")\n", + "plt.axvline(100.0, color=\"grey\", linestyle=\"--\", label=\"100 arcmin\")\n", + "plt.legend()\n", + "\n", + "plt.subplot(212)\n", + "\n", + "plt.errorbar(\n", + " xi_minus[\"ANG\"],\n", + " xi_minus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + ")\n", + "\n", + "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", + "plt.ylabel(r\"$\\xi_{-}$\")\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.axvline(10.0, color=\"grey\", linestyle=\"--\", label=\"10 arcmin\")\n", + "plt.axvline(200.0, color=\"grey\", linestyle=\"--\", label=\"200 arcmin\")\n", + "plt.legend()\n", + "\n", + "plt.savefig(\"xi_data.png\")\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "import pyccl as ccl\n", + "\n", + "# Get theory correlation function from CCL\n", + "# Define the cosmology\n", + "theta_arcmin = np.logspace(np.log10(0.1), np.log10(250), 1000)\n", + "h = 0.7\n", + "Oc = 0.25\n", + "Ob = 0.05\n", + "sigma8 = 0.793897\n", + "n_s = 0.96\n", + "cosmo = ccl.Cosmology(\n", + " h=h,\n", + " Omega_c=Oc,\n", + " Omega_b=Ob,\n", + " sigma8=sigma8,\n", + " n_s=n_s,\n", + " transfer_function=\"boltzmann_camb\",\n", + ")\n", + "\n", + "# Define the redshift distribution\n", + "z, dndz = np.loadtxt(\n", + " \"/home/guerrini/sp_validation/cosmo_inference/cosmocov_config/dndz_test.txt\",\n", + " unpack=True,\n", + ")\n", + "\n", + "tracer = ccl.WeakLensingTracer(cosmo, dndz=(z, dndz), ia_bias=None)\n", + "\n", + "# COmpute the angular power spectrum C_ell\n", + "ell = np.logspace(0, np.log10(10000), 2000)\n", + "cl_gg = ccl.angular_cl(cosmo, tracer, tracer, ell)\n", + "\n", + "# Compute the 2PCF\n", + "theta_deg = theta_arcmin / 60\n", + "# xi+ fit\n", + "xi_p_theta_true = ccl.correlation(\n", + " cosmo, ell=ell, C_ell=cl_gg, theta=theta_deg, type=\"GG+\"\n", + ")\n", + "# xi- fit\n", + "xi_m_theta_true = ccl.correlation(\n", + " cosmo, ell=ell, C_ell=cl_gg, theta=theta_deg, type=\"GG-\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "# Get theory correlation function from CCL\n", + "# Define the cosmology\n", + "theta_arcmin = np.logspace(np.log10(0.1), np.log10(250), 1000)\n", + "h = 0.6982064176424748\n", + "Oc = 0.21522128974860827 / h**2\n", + "print(\"Omega_c:\", Oc)\n", + "Ob = 0.024410205304489712 / h**2\n", + "print(\"Omega_b:\", Ob)\n", + "sigma8 = 0.5330533925822226\n", + "print(\"S8:\", sigma8 * np.sqrt((Oc + Ob) / 0.3))\n", + "n_s = 0.9867762122563981\n", + "a_ia = 0.0\n", + "cosmo = ccl.Cosmology(\n", + " h=h,\n", + " Omega_c=Oc,\n", + " Omega_b=Ob,\n", + " sigma8=sigma8,\n", + " n_s=n_s,\n", + " transfer_function=\"boltzmann_camb\",\n", + ")\n", + "\n", + "# Define the redshift distribution\n", + "z, dndz = np.loadtxt(\n", + " \"/home/guerrini/sp_validation/cosmo_inference/cosmocov_config/dndz_test.txt\",\n", + " unpack=True,\n", + ")\n", + "\n", + "tracer = ccl.WeakLensingTracer(\n", + " cosmo, dndz=(z, dndz), ia_bias=(z, np.ones_like(z) * a_ia)\n", + ")\n", + "\n", + "# COmpute the angular power spectrum C_ell\n", + "ell = np.logspace(0, np.log10(10000), 2000)\n", + "cl_gg = ccl.angular_cl(cosmo, tracer, tracer, ell)\n", + "\n", + "# Compute the 2PCF\n", + "theta_deg = theta_arcmin / 60\n", + "# xi+ fit\n", + "xi_p_theta_fit = ccl.correlation(\n", + " cosmo, ell=ell, C_ell=cl_gg, theta=theta_deg, type=\"GG+\"\n", + ")\n", + "# xi- fit\n", + "xi_m_theta_fit = ccl.correlation(\n", + " cosmo, ell=ell, C_ell=cl_gg, theta=theta_deg, type=\"GG-\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", + "metadata": {}, + "outputs": [], + "source": [ + "# Get theory correlation function from CCL\n", + "# Define the cosmology\n", + "theta_arcmin = np.logspace(np.log10(0.1), np.log10(250), 1000)\n", + "h = 0.6982064176424748\n", + "Oc = 0.21522128974860827 / h**2\n", + "print(\"Omega_c:\", Oc)\n", + "Ob = 0.024410205304489712 / h**2\n", + "print(\"Omega_b:\", Ob)\n", + "sigma8 = 0.5330533925822226\n", + "print(\"S8:\", sigma8 * np.sqrt((Oc + Ob) / 0.3))\n", + "n_s = 0.9867762122563981\n", + "a_ia = -0.4225120529551343\n", + "cosmo = ccl.Cosmology(\n", + " h=h,\n", + " Omega_c=Oc,\n", + " Omega_b=Ob,\n", + " sigma8=sigma8,\n", + " n_s=n_s,\n", + " transfer_function=\"boltzmann_camb\",\n", + ")\n", + "\n", + "# Define the redshift distribution\n", + "z, dndz = np.loadtxt(\n", + " \"/home/guerrini/sp_validation/cosmo_inference/cosmocov_config/dndz_test.txt\",\n", + " unpack=True,\n", + ")\n", + "\n", + "tracer = ccl.WeakLensingTracer(\n", + " cosmo, dndz=(z, dndz), ia_bias=(z, np.ones_like(z) * a_ia)\n", + ")\n", + "\n", + "# COmpute the angular power spectrum C_ell\n", + "ell = np.logspace(0, np.log10(10000), 2000)\n", + "cl_gg = ccl.angular_cl(cosmo, tracer, tracer, ell)\n", + "\n", + "# Compute the 2PCF\n", + "theta_deg = theta_arcmin / 60\n", + "# xi+ fit\n", + "xi_p_theta_fit_IA = ccl.correlation(\n", + " cosmo, ell=ell, C_ell=cl_gg, theta=theta_deg, type=\"GG+\"\n", + ")\n", + "# xi- fit\n", + "xi_m_theta_fit_IA = ccl.correlation(\n", + " cosmo, ell=ell, C_ell=cl_gg, theta=theta_deg, type=\"GG-\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(15, 15))\n", + "\n", + "plt.subplot(211)\n", + "\n", + "plt.errorbar(\n", + " xi_plus[\"ANG\"],\n", + " xi_plus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + ")\n", + "plt.plot(theta_arcmin, xi_p_theta_fit, label=\"SP_v1.4.5 fit\", color=\"red\")\n", + "plt.plot(theta_arcmin, xi_p_theta_true, label=\"SP_v1.4.5 true\", color=\"blue\")\n", + "plt.plot(theta_arcmin, xi_p_theta_fit_IA, label=\"SP_v1.4.5 fit IA\", color=\"green\")\n", + "\n", + "plt.ylabel(r\"$\\xi_{+}$\")\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.axvline(3.0, color=\"grey\", linestyle=\"--\", label=\"3 arcmin\")\n", + "plt.axvline(100.0, color=\"grey\", linestyle=\"--\", label=\"100 arcmin\")\n", + "plt.legend()\n", + "\n", + "plt.subplot(212)\n", + "\n", + "plt.errorbar(\n", + " xi_minus[\"ANG\"],\n", + " xi_minus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + ")\n", + "plt.plot(theta_arcmin, xi_m_theta_fit, label=\"SP_v1.4.5 fit\", color=\"red\")\n", + "plt.plot(theta_arcmin, xi_m_theta_true, label=\"SP_v1.4.5 true\", color=\"blue\")\n", + "plt.plot(theta_arcmin, xi_m_theta_fit_IA, label=\"SP_v1.4.5 fit IA\", color=\"green\")\n", + "\n", + "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", + "plt.ylabel(r\"$\\xi_{-}$\")\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.axvline(10.0, color=\"grey\", linestyle=\"--\", label=\"10 arcmin\")\n", + "plt.axvline(200.0, color=\"grey\", linestyle=\"--\", label=\"200 arcmin\")\n", + "plt.legend()\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], + "source": [ + "# Add best-fit model\n", + "root_dir = \"/n09data/guerrini/output_chains/output_result/glass_mock_1/\"\n", + "xi_plus_bf_no_psf = np.loadtxt(root_dir + \"best_fit/shear_xi_plus/bin_1_1.txt\")\n", + "xi_minus_bf_no_psf = np.loadtxt(root_dir + \"best_fit/shear_xi_minus/bin_1_1.txt\")\n", + "xi_sys_p = np.loadtxt(root_dir + \"best_fit/xi_sys/shear_xi_plus.txt\")\n", + "xi_sys_m = np.loadtxt(root_dir + \"best_fit/xi_sys/shear_xi_minus.txt\")\n", + "theta_xi_sys = np.loadtxt(root_dir + \"best_fit/xi_sys/theta.txt\")\n", + "theta_xi_sys = theta_xi_sys * 180 * 60 / np.pi\n", + "angle = np.loadtxt(root_dir + \"best_fit/shear_xi_plus/theta.txt\")\n", + "angle = angle * 180 * 60 / np.pi\n", + "\n", + "mask = (angle < 250) & (angle > 0.1)\n", + "\n", + "from scipy.interpolate import interp1d\n", + "\n", + "xi_sys_p_interp = interp1d(\n", + " theta_xi_sys, xi_sys_p, kind=\"linear\", fill_value=\"extrapolate\"\n", + ")\n", + "xi_sys_m_interp = interp1d(\n", + " theta_xi_sys, xi_sys_m, kind=\"linear\", fill_value=\"extrapolate\"\n", + ")\n", + "xi_plus_bf = xi_plus_bf_no_psf[mask] + xi_sys_p_interp(angle[mask])\n", + "xi_minus_bf = xi_minus_bf_no_psf[mask] + xi_sys_m_interp(angle[mask])\n", + "\n", + "xi_plus_bf_no_baryons = np.loadtxt(\n", + " root_dir + \"best_fit_no_feedback/shear_xi_plus/bin_1_1.txt\"\n", + ")\n", + "xi_minus_bf_no_baryons = np.loadtxt(\n", + " root_dir + \"best_fit_no_feedback/shear_xi_minus/bin_1_1.txt\"\n", + ")\n", + "\n", + "xi_plus_bf_no_IA = np.loadtxt(root_dir + \"best_fit_no_IA/shear_xi_plus/bin_1_1.txt\")\n", + "xi_minus_bf_no_IA = np.loadtxt(root_dir + \"best_fit_no_IA/shear_xi_minus/bin_1_1.txt\")\n", + "\n", + "xi_plus_truth_cosmosis = np.loadtxt(root_dir + \"/truth/shear_xi_plus/bin_1_1.txt\")\n", + "xi_minus_truth_cosmosis = np.loadtxt(root_dir + \"/truth/shear_xi_minus/bin_1_1.txt\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(15, 15))\n", + "\n", + "plt.subplot(211)\n", + "\n", + "plt.errorbar(\n", + " xi_plus[\"ANG\"],\n", + " xi_plus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + ")\n", + "plt.plot(theta_arcmin, xi_p_theta_true, label=\"SP_v1.4.5 true\", color=\"blue\")\n", + "plt.plot(angle[mask], xi_plus_bf, label=\"SP_v1.4.5 COSMOSIS\", color=\"orange\")\n", + "plt.plot(\n", + " angle[mask],\n", + " xi_plus_bf_no_psf[mask],\n", + " label=\"SP_v1.4.5 COSMOSIS no PSF\",\n", + " color=\"green\",\n", + ")\n", + "plt.plot(\n", + " angle[mask],\n", + " xi_plus_bf_no_baryons[mask],\n", + " label=\"SP_v1.4.5 COSMOSIS no baryons\",\n", + " color=\"red\",\n", + ")\n", + "plt.plot(\n", + " angle[mask],\n", + " xi_plus_bf_no_IA[mask],\n", + " label=\"SP_v1.4.5 COSMOSIS no IA\",\n", + " color=\"purple\",\n", + ")\n", + "plt.plot(\n", + " angle[mask],\n", + " xi_plus_truth_cosmosis[mask],\n", + " label=\"SP_v1.4.5 COSMOSIS truth\",\n", + " color=\"black\",\n", + " linestyle=\"--\",\n", + ")\n", + "\n", + "plt.ylabel(r\"$\\xi_{+}$\")\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.axvline(3.0, color=\"grey\", linestyle=\"--\", label=\"3 arcmin\")\n", + "plt.axvline(150.0, color=\"grey\", linestyle=\"--\", label=\"150 arcmin\")\n", + "plt.legend()\n", + "\n", + "plt.subplot(212)\n", + "\n", + "plt.errorbar(\n", + " xi_minus[\"ANG\"],\n", + " xi_minus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + " markersize=2,\n", + ")\n", + "plt.plot(theta_arcmin, xi_m_theta_true, label=\"SP_v1.4.5 true\", color=\"blue\")\n", + "plt.plot(angle[mask], xi_minus_bf, label=\"SP_v1.4.5 COSMOSIS\", color=\"orange\")\n", + "plt.plot(\n", + " angle[mask],\n", + " xi_minus_bf_no_psf[mask],\n", + " label=\"SP_v1.4.5 COSMOSIS no PSF\",\n", + " color=\"green\",\n", + ")\n", + "plt.plot(\n", + " angle[mask],\n", + " xi_minus_bf_no_baryons[mask],\n", + " label=\"SP_v1.4.5 COSMOSIS no baryons\",\n", + " color=\"red\",\n", + ")\n", + "plt.plot(\n", + " angle[mask],\n", + " xi_minus_bf_no_IA[mask],\n", + " label=\"SP_v1.4.5 COSMOSIS no IA\",\n", + " color=\"purple\",\n", + ")\n", + "plt.plot(\n", + " angle[mask],\n", + " xi_minus_truth_cosmosis[mask],\n", + " label=\"SP_v1.4.5 COSMOSIS truth\",\n", + " color=\"black\",\n", + " linestyle=\"--\",\n", + ")\n", + "\n", + "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", + "plt.ylabel(r\"$\\xi_{-}$\")\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.axvline(10.0, color=\"grey\", linestyle=\"--\", label=\"10 arcmin\")\n", + "plt.axvline(200.0, color=\"grey\", linestyle=\"--\", label=\"200 arcmin\")\n", + "plt.legend()\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26", + "metadata": {}, + "outputs": [], + "source": [ + "import treecorr\n", + "\n", + "theta_min = 0.1\n", + "theta_max = 250.0\n", + "nbins = 20\n", + "var_method = \"jackknife\"\n", + "\n", + "treecorr_config = {\n", + " \"ra_units\": \"degrees\",\n", + " \"dec_units\": \"degrees\",\n", + " \"min_sep\": theta_min,\n", + " \"max_sep\": theta_max,\n", + " \"sep_units\": \"arcmin\",\n", + " \"nbins\": nbins,\n", + " \"var_method\": var_method,\n", + "}\n", + "\n", + "gg = treecorr.GGCorrelation(treecorr_config)\n", + "\n", + "# Load the measurement\n", + "cat = fits.getdata(\n", + " \"/n09data/guerrini/glass_mock/results/unions_glass_sim_00001_4096.fits\"\n", + ")\n", + "\n", + "e1 = cat[\"e1\"]\n", + "e2 = cat[\"e2\"]\n", + "ra = cat[\"ra\"]\n", + "dec = cat[\"dec\"]\n", + "\n", + "# Create the catalog\n", + "cat = treecorr.Catalog(\n", + " ra=ra, dec=dec, ra_units=\"degrees\", dec_units=\"degrees\", g1=e1, g2=e2, npatch=200\n", + ")\n", + "\n", + "# Process the catalog\n", + "gg.process(cat)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [ + "cov = treecorr.estimate_multi_cov([gg], method=\"jackknife\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "28", + "metadata": {}, + "outputs": [], + "source": [ + "data_vector_sim = []\n", + "\n", + "for ver in [f\"SP_v1.4.5_glass_mock_{i}\" for i in range(1, 17)]:\n", + " data = fits.open(\n", + " f\"/home/guerrini/sp_validation/cosmo_inference/data/{ver}/cosmosis_{ver}.fits\"\n", + " )\n", + " xi_plus = data[\"XI_PLUS\"].data\n", + " xi_minus = data[\"XI_MINUS\"].data\n", + " data_vector_sim.append(np.concatenate((xi_plus[\"VALUE\"], xi_minus[\"VALUE\"])))\n", + "\n", + "data_vector_sim = np.array(data_vector_sim)\n", + "\n", + "data_vector_sim.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "cov_sim = np.cov(data_vector_sim.T)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30", + "metadata": {}, + "outputs": [], + "source": [ + "ver_sacha = \"SP_v1.4.5\"\n", + "\n", + "cov_th_sacha = np.loadtxt(\n", + " \"/home/guerrini/sp_validation/cosmo_inference/data/{}/covs/cov_{}.txt\".format(\n", + " ver_sacha, ver_sacha\n", + " )\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure()\n", + "\n", + "plt.plot(xi_plus[\"ANG\"], np.diag(cov)[:20])\n", + "plt.plot(xi_plus[\"ANG\"], np.diag(cov_sim)[:20], label=\"SP_v1.4.5 data\", color=\"red\")\n", + "plt.plot(\n", + " xi_plus[\"ANG\"], np.diag(cov_th_sacha)[:20], label=\"SP_v1.4.5 data\", color=\"green\"\n", + ")\n", + "plt.plot(xi_plus[\"ANG\"], np.diag(cov_mat)[:20], label=\"SP_v1.4.5 data\", color=\"black\")\n", + "\n", + "plt.ylabel(\"Diagonal of the covariance\")\n", + "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", + "\n", + "plt.yscale(\"log\")\n", + "plt.xscale(\"log\")\n", + "plt.savefig(\"check_cov.png\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "32", + "metadata": {}, + "outputs": [], + "source": [ + "gg.varxip" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33", + "metadata": {}, + "outputs": [], + "source": [ + "np.sqrt(gg.varxip)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34", + "metadata": {}, + "outputs": [], + "source": [ + "np.sqrt(np.diag(cov_mat)[0:20])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(15, 15))\n", + "\n", + "plt.subplot(211)\n", + "\n", + "plt.errorbar(\n", + " xi_plus[\"ANG\"],\n", + " xi_plus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + ")\n", + "plt.plot(angle[mask], xi_plus_bf[mask], label=\"Best-fit model\", color=\"red\")\n", + "\n", + "plt.ylabel(r\"$\\xi_{+}$\")\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.axvline(3.0, color=\"grey\", linestyle=\"--\", label=\"3 arcmin\")\n", + "plt.axvline(100.0, color=\"grey\", linestyle=\"--\", label=\"100 arcmin\")\n", + "plt.legend()\n", + "\n", + "plt.subplot(212)\n", + "\n", + "plt.errorbar(\n", + " xi_minus[\"ANG\"],\n", + " xi_minus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + ")\n", + "plt.plot(angle[mask], xi_minus_bf[mask], label=\"Best-fit model\", color=\"red\")\n", + "\n", + "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", + "plt.ylabel(r\"$\\xi_{-}$\")\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.axvline(10.0, color=\"grey\", linestyle=\"--\", label=\"10 arcmin\")\n", + "plt.axvline(200.0, color=\"grey\", linestyle=\"--\", label=\"200 arcmin\")\n", + "plt.legend()\n", + "\n", + "plt.savefig(\"xi_data_bf.png\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "36", + "metadata": {}, + "outputs": [], + "source": [ + "# Add PSF systematic\n", + "xi_sys_plus = np.loadtxt(root_dir + \"/xi_sys/shear_xi_plus.txt\")\n", + "xi_sys_minus = np.loadtxt(root_dir + \"/xi_sys/shear_xi_minus.txt\")\n", + "theta_sys = np.loadtxt(root_dir + \"/xi_sys/theta.txt\")\n", + "theta_sys = theta_sys * 180 * 60 / np.pi" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(15, 15))\n", + "\n", + "plt.subplot(211)\n", + "\n", + "plt.errorbar(\n", + " xi_plus[\"ANG\"],\n", + " xi_plus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + ")\n", + "plt.plot(angle[mask], xi_plus_bf[mask], label=\"Best-fit model wo SYS\", color=\"red\")\n", + "plt.plot(theta_sys, xi_sys_plus, label=r\"$\\xi_{\\rm sys}$\", color=\"green\")\n", + "\n", + "plt.ylabel(r\"$\\xi_{+}$\")\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.axvline(3.0, color=\"grey\", linestyle=\"--\", label=\"3 arcmin\")\n", + "plt.axvline(100.0, color=\"grey\", linestyle=\"--\", label=\"100 arcmin\")\n", + "plt.legend()\n", + "\n", + "plt.subplot(212)\n", + "\n", + "plt.errorbar(\n", + " xi_minus[\"ANG\"],\n", + " xi_minus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + ")\n", + "plt.plot(angle[mask], xi_minus_bf[mask], label=\"Best-fit model wo SYS\", color=\"red\")\n", + "plt.plot(theta_sys, xi_sys_minus, label=r\"$\\xi_{\\rm sys}$\", color=\"green\")\n", + "\n", + "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", + "plt.ylabel(r\"$\\xi_{-}$\")\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.axvline(10.0, color=\"grey\", linestyle=\"--\", label=\"10 arcmin\")\n", + "plt.axvline(200.0, color=\"grey\", linestyle=\"--\", label=\"200 arcmin\")\n", + "plt.legend()\n", + "\n", + "plt.savefig(\"xi_data_bf_sys.png\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38", + "metadata": {}, + "outputs": [], + "source": [ + "shear_cl = np.loadtxt(root_dir + \"/shear_cl/bin_1_1.txt\")\n", + "shear_cl_gg = np.loadtxt(root_dir + \"/shear_cl_gg/bin_1_1.txt\")\n", + "shear_cl_gi = np.loadtxt(root_dir + \"/shear_cl_gi/bin_1_1.txt\")\n", + "shear_cl_ii = np.loadtxt(root_dir + \"/shear_cl_ii/bin_1_1.txt\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "39", + "metadata": {}, + "outputs": [], + "source": [ + "A = 3.355083374185272\n", + "np.isclose(shear_cl, shear_cl_gg + 2 * shear_cl_gi + shear_cl_ii)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41", + "metadata": {}, + "outputs": [], + "source": [ + "# Add the lensing part without intrinsic alignment\n", + "root_dir = \"/n09data/guerrini/output_chains/test_pipeline/\"\n", + "xi_plus_wo_ia = np.loadtxt(root_dir + \"/shear_xi_plus_wo_IA/bin_1_1.txt\")\n", + "xi_minus_wo_ia = np.loadtxt(root_dir + \"/shear_xi_minus_wo_IA/bin_1_1.txt\")\n", + "angle = np.loadtxt(root_dir + \"/shear_xi_plus_wo_IA/theta.txt\")\n", + "angle = angle * 180 * 60 / np.pi\n", + "\n", + "mask = (angle < 250) & (angle > 0.1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "42", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(15, 15))\n", + "\n", + "plt.subplot(211)\n", + "\n", + "plt.errorbar(\n", + " xi_plus[\"ANG\"],\n", + " xi_plus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + ")\n", + "plt.plot(angle[mask], xi_plus_bf[mask], label=\"Best-fit model wo SYS\", color=\"red\")\n", + "plt.plot(theta_sys, xi_sys_plus, label=r\"$\\xi_{\\rm sys}$\", color=\"green\")\n", + "plt.plot(\n", + " angle[mask], xi_plus_wo_ia[mask], label=\"Best-fit model wo IA and SYS\", color=\"blue\"\n", + ")\n", + "\n", + "plt.ylabel(r\"$\\xi_{+}$\")\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.axvline(3.0, color=\"grey\", linestyle=\"--\", label=\"3 arcmin\")\n", + "plt.axvline(100.0, color=\"grey\", linestyle=\"--\", label=\"100 arcmin\")\n", + "plt.legend()\n", + "\n", + "plt.subplot(212)\n", + "\n", + "plt.errorbar(\n", + " xi_minus[\"ANG\"],\n", + " xi_minus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + ")\n", + "plt.plot(angle[mask], xi_minus_bf[mask], label=\"Best-fit model wo SYS\", color=\"red\")\n", + "plt.plot(theta_sys, xi_sys_minus, label=r\"$\\xi_{\\rm sys}$\", color=\"green\")\n", + "plt.plot(\n", + " angle[mask],\n", + " xi_minus_wo_ia[mask],\n", + " label=\"Best-fit model wo IA and SYS\",\n", + " color=\"blue\",\n", + ")\n", + "\n", + "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", + "plt.ylabel(r\"$\\xi_{-}$\")\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.axvline(10.0, color=\"grey\", linestyle=\"--\", label=\"10 arcmin\")\n", + "plt.axvline(200.0, color=\"grey\", linestyle=\"--\", label=\"200 arcmin\")\n", + "plt.legend()\n", + "\n", + "plt.savefig(\"xi_data_bf_sys_ia.png\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "43", + "metadata": {}, + "outputs": [], + "source": [ + "# Add the lensing part without intrinsic alignment\n", + "root_dir = \"/n09data/guerrini/output_chains/test_pipeline/\"\n", + "xi_plus_reas = np.loadtxt(root_dir + \"/shear_xi_plus/bin_1_1.txt\")\n", + "xi_minus_reas = np.loadtxt(root_dir + \"/shear_xi_minus/bin_1_1.txt\")\n", + "angle_reas = np.loadtxt(root_dir + \"/shear_xi_plus/theta.txt\")\n", + "angle_reas = angle_reas * 180 * 60 / np.pi\n", + "\n", + "mask = (angle < 250) & (angle > 0.1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(15, 15))\n", + "\n", + "plt.subplot(211)\n", + "\n", + "plt.errorbar(\n", + " xi_plus[\"ANG\"],\n", + " xi_plus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + ")\n", + "plt.plot(angle[mask], xi_plus_bf[mask], label=\"Best-fit model wo SYS\", color=\"red\")\n", + "plt.plot(theta_sys, xi_sys_plus, label=r\"$\\xi_{\\rm sys}$\", color=\"green\")\n", + "plt.plot(\n", + " angle[mask], xi_plus_wo_ia[mask], label=\"Best-fit model wo IA and SYS\", color=\"blue\"\n", + ")\n", + "plt.plot(\n", + " angle_reas[mask],\n", + " xi_plus_reas[mask],\n", + " label=\"Lower IA\",\n", + " color=\"orange\",\n", + " linestyle=\"--\",\n", + ")\n", + "\n", + "plt.ylabel(r\"$\\xi_{+}$\")\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.axvline(3.0, color=\"grey\", linestyle=\"--\", label=\"3 arcmin\")\n", + "plt.axvline(100.0, color=\"grey\", linestyle=\"--\", label=\"100 arcmin\")\n", + "plt.legend()\n", + "\n", + "plt.subplot(212)\n", + "\n", + "plt.errorbar(\n", + " xi_minus[\"ANG\"],\n", + " xi_minus[\"VALUE\"],\n", + " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", + " fmt=\"o\",\n", + " label=\"SP_v1.4.5 data\",\n", + " color=\"black\",\n", + ")\n", + "plt.plot(angle[mask], xi_minus_bf[mask], label=\"Best-fit model wo SYS\", color=\"red\")\n", + "plt.plot(theta_sys, xi_sys_minus, label=r\"$\\xi_{\\rm sys}$\", color=\"green\")\n", + "plt.plot(\n", + " angle[mask],\n", + " xi_minus_wo_ia[mask],\n", + " label=\"Best-fit model wo IA and SYS\",\n", + " color=\"blue\",\n", + ")\n", + "plt.plot(\n", + " angle_reas[mask],\n", + " xi_minus_reas[mask],\n", + " label=\"Lower IA\",\n", + " color=\"orange\",\n", + " linestyle=\"--\",\n", + ")\n", + "\n", + "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", + "plt.ylabel(r\"$\\xi_{-}$\")\n", + "plt.xscale(\"log\")\n", + "plt.yscale(\"log\")\n", + "plt.axvline(10.0, color=\"grey\", linestyle=\"--\", label=\"10 arcmin\")\n", + "plt.axvline(200.0, color=\"grey\", linestyle=\"--\", label=\"200 arcmin\")\n", + "plt.legend()\n", + "\n", + "plt.savefig(\"xi_data_bf_sys_ia_reas.png\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosmo_inference/notebooks/get_prior_psf_leakage.ipynb b/cosmo_inference/notebooks/get_prior_psf_leakage.ipynb new file mode 100644 index 00000000..1dae9a36 --- /dev/null +++ b/cosmo_inference/notebooks/get_prior_psf_leakage.ipynb @@ -0,0 +1,269 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "if not os.path.exists(\"./Plots\"):\n", + " os.makedirs(\"./Plots\")\n", + "\n", + "# Trick to plot with tex\n", + "os.environ[\"LD_LIBRARY_PATH\"] = \"\"\n", + "os.environ[\"CONDA_PREFIX\"] = \"/home/guerrini/.conda/envs/sp_validation_3.11\"\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import seaborn as sns\n", + "from astropy.io import fits\n", + "from getdist import MCSamples, plots\n", + "from shear_psf_leakage.rho_tau_stat import PSFErrorFit, RhoStat, TauStat\n", + "\n", + "# Use paper style and seaborn with husl palette\n", + "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", + "# Set default palette - will be updated per plot as needed\n", + "sns.set_palette(\"husl\")\n", + "%matplotlib inline\n", + "\n", + "g = plots.get_subplot_plotter(width_inch=30)\n", + "g.settings.axes_fontsize = 30\n", + "g.settings.axes_labelsize = 30\n", + "g.settings.alpha_filled_add = 0.7\n", + "g.settings.legend_fontsize = 25" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "data_path = \"/home/guerrini/sp_validation/cosmo_inference/data/\"\n", + "\n", + "path_cosmo_val = \"/home/guerrini/sp_validation/cosmo_val/output/\"\n", + "\n", + "roots_cosmo_val = [\"SP_v1.4.6\", \"SP_v1.4.6_leak_corr\"]\n", + "\n", + "roots = [\"SP_v1.4.6_no_leak_corr_A_masked\", \"SP_v1.4.6_leak_corr_A_masked\"]\n", + "\n", + "labels = [\"SP_v1.4.6_A\", \"SP_v1.4.6_A leakage corrected\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "data_vectors = []\n", + "\n", + "for root in roots:\n", + " data_vectors.append(fits.open(data_path + root + f\"/cosmosis_{root}.fits\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "def cov_to_corr(cov):\n", + " \"\"\"Convert a covariance matrix to a correlation matrix.\"\"\"\n", + " d = np.sqrt(np.diag(cov))\n", + " corr = cov / np.outer(d, d)\n", + " corr[cov == 0] = 0\n", + " return corr" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Print the covariance matrix for each root\n", + "for i, root in enumerate(roots):\n", + " print(f\"Covariance matrix for {labels[i]}:\")\n", + " cov = data_vectors[i][\"COVMAT\"].data\n", + "\n", + " n_bins = cov.shape[0] // 4\n", + "\n", + " fig, ax = plt.subplots(figsize=(10, 8))\n", + "\n", + " im = ax.imshow(cov_to_corr(cov), vmin=-1, vmax=1, cmap=\"seismic\")\n", + " ax.set_aspect(\"equal\")\n", + " ax.set_yticks(np.array([10, 30, 50, 70]))\n", + " ax.set_yticklabels(\n", + " [\n", + " r\"$\\xi_+(\\vartheta)$\",\n", + " r\"$\\xi_-(\\vartheta)$\",\n", + " r\"$\\tau_0(\\vartheta)$\",\n", + " r\"$\\tau_2(\\vartheta)$\",\n", + " ]\n", + " )\n", + " ax.set_xticks(np.array([10, 30, 50, 70]))\n", + " ax.set_xticklabels(\n", + " [\n", + " r\"$\\xi_+(\\vartheta)$\",\n", + " r\"$\\xi_-(\\vartheta)$\",\n", + " r\"$\\tau_0(\\vartheta)$\",\n", + " r\"$\\tau_2(\\vartheta)$\",\n", + " ],\n", + " rotation=45,\n", + " )\n", + " fig.colorbar(im, ax=ax)\n", + "\n", + " plt.savefig(f\"./Plots/cov_matrix_{root}.png\", bbox_inches=\"tight\", dpi=300)\n", + " plt.show()\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# Create dummy rho and tau stat handler.\n", + "\n", + "# Inference of the xi_sys parameters\n", + "sep_units = \"arcmin\"\n", + "coord_units = \"degrees\"\n", + "theta_min = 1.0\n", + "theta_max = 250\n", + "nbins = 20\n", + "\n", + "\n", + "TreeCorrConfig_xi = {\n", + " \"ra_units\": coord_units,\n", + " \"dec_units\": coord_units,\n", + " \"min_sep\": theta_min,\n", + " \"max_sep\": theta_max,\n", + " \"sep_units\": sep_units,\n", + " \"nbins\": nbins,\n", + " \"var_method\": \"jackknife\",\n", + "}\n", + "\n", + "rho_stats_handler = RhoStat(output=\".\", treecorr_config=TreeCorrConfig_xi, verbose=True)\n", + "\n", + "tau_stats_handler = TauStat(\n", + " catalogs=rho_stats_handler.catalogs,\n", + " output=\".\",\n", + " treecorr_config=TreeCorrConfig_xi,\n", + " verbose=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# Create a PSFErrorFit instance\n", + "psf_fitter = PSFErrorFit(\n", + " rho_stats_handler,\n", + " tau_stats_handler,\n", + " path_cosmo_val + \"rho_tau_stats/\",\n", + " use_eta=False,\n", + ")\n", + "\n", + "\n", + "def load_matrix_and_cut(root, root_cosmo_val, type=\"rho\"):\n", + " if type == \"rho\":\n", + " cov = np.load(f\"{root}/cov_rho_{root_cosmo_val}.npy\")\n", + " nbins = cov.shape[0] // 6\n", + " cov = cov[: nbins * 3, : nbins * 3]\n", + " np.save(f\"{root}/cov_rho_{root_cosmo_val}_cut.npy\", cov)\n", + " elif type == \"tau\":\n", + " cov = np.load(f\"{root}/cov_tau_{root_cosmo_val}_th.npy\")\n", + " nbins = cov.shape[0] // 3\n", + " cov = cov[: nbins * 2, : nbins * 2]\n", + " np.save(f\"{root}/cov_tau_{root_cosmo_val}_th_cut.npy\", cov)\n", + " else:\n", + " raise ValueError(\"type must be 'rho' or 'tau'\")\n", + "\n", + "\n", + "g = plots.get_subplot_plotter(width_inch=30)\n", + "\n", + "g.settings.axes_fontsize = 30\n", + "g.settings.axes_labelsize = 30\n", + "g.settings.alpha_filled_add = 0.7\n", + "g.settings.legend_fontsize = 40\n", + "\n", + "chains = []\n", + "\n", + "# Load rho-, tau-statistics, and cov_tau from the data_vector\n", + "for i, root_cosmo_val in enumerate(roots_cosmo_val):\n", + " print(\"Sampling PSF parameters for \", labels[i])\n", + " path_rho = f\"rho_stats_{root_cosmo_val}.fits\"\n", + " path_tau = f\"tau_stats_{root_cosmo_val}.fits\"\n", + " path_cov_rho = f\"cov_rho_{root_cosmo_val}.npy\"\n", + " path_cov_tau = f\"cov_tau_{root_cosmo_val}_th.npy\"\n", + "\n", + " load_matrix_and_cut(path_cosmo_val + \"/rho_tau_stats/\", root_cosmo_val, type=\"rho\")\n", + " load_matrix_and_cut(path_cosmo_val + \"/rho_tau_stats/\", root_cosmo_val, type=\"tau\")\n", + " path_cov_rho = f\"cov_rho_{root_cosmo_val}_cut.npy\"\n", + " path_cov_tau = f\"cov_tau_{root_cosmo_val}_th_cut.npy\"\n", + "\n", + " psf_fitter.load_rho_stat(path_rho)\n", + " psf_fitter.load_tau_stat(path_tau)\n", + " psf_fitter.load_covariance(path_cov_rho, cov_type=\"rho\")\n", + " psf_fitter.load_covariance(path_cov_tau, cov_type=\"tau\")\n", + " samples_lq, _, _ = psf_fitter.get_least_squares_params_samples(\n", + " npatch=None, apply_debias=False\n", + " )\n", + "\n", + " samples_gd = MCSamples(\n", + " samples=samples_lq, names=[r\"\\alpha\", r\"\\beta\"], labels=[r\"\\alpha\", r\"\\beta\"]\n", + " )\n", + "\n", + " chains.append(samples_gd)\n", + "\n", + "g.triangle_plot(chains, filled=True, legend_labels=labels, legend_loc=\"upper right\")\n", + "\n", + "plt.savefig(\"./Plots/psf_leakage_params.png\", bbox_inches=\"tight\", dpi=300)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sp_validation_3.11", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosmo_inference/pipeline.sh b/cosmo_inference/pipeline.sh new file mode 100755 index 00000000..f853d511 --- /dev/null +++ b/cosmo_inference/pipeline.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +# Transform long options to short ones +for arg in "$@"; do + shift + case "$arg" in + '--help') set -- "$@" '-h' ;; + '--pcf') set -- "$@" '-p' ;; + '--covmat') set -- "$@" '-c' ;; + '--inference') set -- "$@" '-i' ;; + '--mcmc_process') set -- "$@" '-m' ;; + *) set -- "$@" "$arg" ;; + esac +done + +# Parse short options +OPTIND=1 +while getopts "hpcim" opt +do + case "$opt" in + 'h') + echo "Please input a flag: --help, --pcf, --covmat, --inference or --mcmc_process "; + exit 0 + ;; + 'p') + echo "Running cosmo_val.py to calculate 2 point correlation functions"; + python cosmo_val/cosmo_val.py + ;; + 'c') + read -p 'COVARIANCE FILE: ' covmat_file; + read -p 'OUTPUT STUB (without extension): ' output_stub; + echo "Processing covariance matrix"; + python scripts/cosmocov_process.py $covmat_file $output_stub + ;; + 'i') + read -p 'XI ROOT: ' xi_root; + read -p 'TAU ROOT: ' tau_root; + read -p 'COSMOSIS ROOT: ' cosmosis_root; + read -p 'COSMO_VAL OUTPUT FOLDER: ' output_folder; + read -p 'NZ FILE:' nz_file; + read -p 'OUTPUT MCMC CHAIN FOLDER: ' data; + read -p 'USE PSEUDO_CELL? (y/n): ' pseudo_cell; + + if [ "${pseudo_cell}" == "y" ]; then + echo "Using pseudo cell" + + out_file="data/${root}/cosmosis_${root}_cell.fits" + + # Create the folder if it does not exist + if [ ! -d "data/$root" ]; then + mkdir -p "data/$root" + echo "Directory 'data/$root' created." + else + echo "Directory 'data/$root' already exists." + fi + + python scripts/cosmosis_fitting.py $root $output_folder $nz_file $pseudo_cell $out_file + else + + read -p 'USE RHO/TAU_STATS? (y/n): ' rhotau_stats; + echo $rhotau_stats + read -p 'COV_XI MAT TXT FILE:' covmat; + + out_file="data/${root}/cosmosis_${root}.fits"; + + # Create the folder if it does not exist + if [ ! -d "data/$root" ]; then + mkdir -p "data/$root" + echo "Directory 'data/$root' created." + else + echo "Directory 'data/$root' already exists." + fi + + #LG: add check if xi_plus/xi_minus fits file exists + python scripts/cosmosis_fitting.py $root $output_folder $nz_file $pseudo_cell $out_file $covmat $rhotau_stats; + + fi + + if [ "${pseudo_cell}" == "y" ]; then + output_ini_file="cosmosis_config/cosmosis_pipeline_${root}_cell.ini" + cp cosmosis_config/cosmosis_pipeline_A_ia_cell.ini $output_ini_file + else + output_ini_file="cosmosis_config/cosmosis_pipeline_${root}.ini" + if [ "${rhotau_stats}" == "y" ]; then + cp cosmosis_config/cosmosis_pipeline_A_psf.ini $output_ini_file; + else + cp cosmosis_config/cosmosis_pipeline_A_ia.ini $output_ini_file; + fi + fi + + sed -i "/^\[DEFAULT\]/a\SCRATCH = ${data}" $output_ini_file; + sed -i "/^\[DEFAULT\]/a\FITS_FILE = ${out_file}" $output_ini_file; + if [ "${pseudo_cell}" == "y" ]; then + sed -i "/^\[output\]/a\filename = %(SCRATCH)s/${root}_cell/samples_${root}_cell.txt" $output_ini_file; + sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_ia.ini" $output_ini_file; + sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors.ini" $output_ini_file; + sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" $output_ini_file; + sed -i "/^\[2pt_like]/a\data_sets=CELL_EE" $output_ini_file; + sed -i "/^\[polychord\]/a\polychord_outfile_root = ${root}_cell" $output_ini_file; + sed -i "/^\[test\]/a\save_dir = %(SCRATCH)s/best_fit/${root}_cell" $output_ini_file; + else + sed -i "/^\[output\]/a\filename = %(SCRATCH)s/${root}/samples_${root}.txt" $output_ini_file; + if [ "${rhotau_stats}" == "y" ]; then + sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_psf.ini" $output_ini_file; + sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors_psf.ini" $output_ini_file; + sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py" $output_ini_file; + sed -i "/^\[2pt_like]/a\data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS" $output_ini_file; + sed -i "/^\[2pt_like]/a\add_xi_sys=T" $output_ini_file; + else + sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_ia.ini" $output_ini_file; + sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors.ini" $output_ini_file; + sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" $output_ini_file; + sed -i "/^\[2pt_like]/a\data_sets=XI_PLUS XI_MINUS" $output_ini_file; + fi + sed -i "/^\[polychord\]/a\polychord_outfile_root = ${root}" $output_ini_file; + sed -i "/^\[test\]/a\save_dir = %(SCRATCH)s/best_fit/${root}" $output_ini_file; + fi + echo "Prepared CosmoSIS configuration file in $output_ini_file"; + echo "You can now run the inference with the command: cosmosis $output_ini_file" + ;; + 'm') + # LG: also convert this into a script to directly output contour plots + echo "Run the cosmo_inference/notebooks/MCMC.ipynb notebook to analyse your chains" + ;; + '?') + print_usage >&2; + exit 1 + ;; + esac +done +shift $(expr $OPTIND - 1) # remove options from positional parameters \ No newline at end of file diff --git a/cosmo_inference/scripts/2pt_like_xi_sys.py b/cosmo_inference/scripts/2pt_like_xi_sys.py new file mode 100644 index 00000000..93f11bd3 --- /dev/null +++ b/cosmo_inference/scripts/2pt_like_xi_sys.py @@ -0,0 +1,614 @@ +import gaussian_covariance +import numpy as np +import twopoint +from astropy.io import fits +from cosmosis.datablock import SectionOptions, names +from cosmosis.gaussian_likelihood import GaussianLikelihood +from scipy.interpolate import interp1d +from spec_tools import TheorySpectrum +from twopoint_cosmosis import theory_names, type_table + +default_array = np.repeat(-1.0, 99) + + +# To copy in cosmosis-standard-library/likelihood +def is_default(x): + return len(x) == len(default_array) and (x == default_array).all() + + +def convert_nz_steradian(n): + return n * (41253.0 * 60.0 * 60.0) / (4 * np.pi) + + +class TwoPointLikelihood(GaussianLikelihood): + # This is a sub-class of the class GaussianLikelihood + # which can be found in the file ${COSMOSIS_SRC_DIR}/cosmosis/gaussian_likelihood.py + # That super-class implements the generic behaviour that all Gaussian likelihoods + # follow - the basic form of the likelihoods, inverting covariance matrices, saving + # results, etc. This sub-clas does the parts that are specific to this 2-pt + # likelihood - loading data from a file, getting the specific theory prediction + # to which to compare it, etc. + like_name = "2pt" + + def __init__(self, options): + # We may decide to use an analytic gaussian covariance + # in that case we won't load the covmat. + self.gaussian_covariance = options.get_bool("gaussian_covariance", False) + if self.gaussian_covariance: + self.constant_covariance = False + + self.moped = options.get_string("moped", default="") + + super(TwoPointLikelihood, self).__init__(options) + + self.raw_data_x, self.raw_data_y = self.build_data() + + if self.moped: + print( + "Using compressed data from MOPED algorithm: {} data points".format( + len(self.moped_data) + ) + ) + if self.sellentin: + raise ValueError( + "Sellentin mode is incompatible with Moped mode in 2pt like" + ) + + def build_data(self): + filename = self.options.get_string("data_file") + + # Suffixes to added on to two point data from e.g. different experiments + suffix_string = self.options.get_string("suffixes", default="") + if suffix_string == "": + # If there are no suffixes provided, then we create a list of a single empty suffix + suffixes = [""] + else: + suffixes_temp = suffix_string.split() + suffixes = [] + for suffix in suffixes_temp: + if suffix.lower() == "none": + suffixes.append("") + else: + suffixes.append("_" + suffix) + self.suffixes = suffixes + + if self.gaussian_covariance: + covmat_name = None + area = self.options.get_double("survey_area") # in square degrees + self.sky_area = area * (np.pi * np.pi) / (180 * 180) + + def get_arr(x): + if self.options.has_value(x): + a = self.options[x] + if not isinstance(a, np.ndarray): + a = [a] + else: + a = default_array + return a + + self.number_density_shear_bin = get_arr("number_density_shear_bin") + self.number_density_lss_bin = get_arr("number_density_lss_bin") + self.sigma_e_bin = get_arr("sigma_e_bin") + + else: + covmat_name = self.options.get_string("covmat_name", "COVMAT") + + # This is the main work - read data in from the file + self.two_point_data = twopoint.TwoPointFile.from_fits(filename, covmat_name) + + # Potentially cut out lines. For some reason one version of + # this file used zeros to mark masked values. + if self.options.get_bool("cut_zeros", default=False): + print("Removing 2-point values with value=0.0") + self.two_point_data.mask_bad(0.0) + + if self.options.get_bool("cut_cross", default=False): + print("Removing 2-point values from cross-bins") + self.two_point_data.mask_cross() + + # All the names of two-points measurements that were found in the data + # file + all_names = [spectrum.name for spectrum in self.two_point_data.spectra] + + # We may not want to use all the likelihoods in the file. + # We can set an option to only use some of them + data_sets = self.options.get_string("data_sets", default="all") + if data_sets != "all": + data_sets = data_sets.split() + self.two_point_data.choose_data_sets(data_sets) + + # The ones we actually used. + self.used_names = [spectrum.name for spectrum in self.two_point_data.spectra] + + # Check for scale cuts. In general, this is a minimum and maximum angle for + # each spectrum, for each redshift bin combination. Which is clearly a massive pain... + # but what can you do? + + scale_cuts = {} + for name in self.used_names: + s = self.two_point_data.get_spectrum(name) + for b1, b2 in s.bin_pairs: + option_name = "angle_range_{}_{}_{}".format(name, b1, b2) + if self.options.has_value(option_name): + r = self.options.get_double_array_1d(option_name) + scale_cuts[(name, b1, b2)] = r + + # Now check for completely cut bins + # example: + # cut_wtheta = 1,2 1,3 2,3 + bin_cuts = [] + for name in self.used_names: + s = self.two_point_data.get_spectrum(name) + option_name = "cut_{}".format(name) + if self.options.has_value(option_name): + cuts = self.options[option_name].split() + cuts = [eval(cut) for cut in cuts] + for b1, b2 in cuts: + bin_cuts.append((name, b1, b2)) + + if scale_cuts or bin_cuts: + self.two_point_data.mask_scales(scale_cuts, bin_cuts) + else: + print("No scale cuts mentioned in ini file.") + + # Info on which likelihoods we do and do not use + print("Found these data sets in the file:") + total_data_points = 0 + final_names = [spectrum.name for spectrum in self.two_point_data.spectra] + for name in all_names: + if name in final_names: + data_points = len(self.two_point_data.get_spectrum(name)) + else: + data_points = 0 + if name in self.used_names: + print( + " - {} {} data points after cuts {}".format( + name, data_points, " [using in likelihood]" + ) + ) + total_data_points += data_points + else: + print( + " - {} {} data points after cuts {}".format( + name, data_points, " [not using in likelihood]" + ) + ) + print("Total data points used = {}".format(total_data_points)) + + # Convert all units to radians. The units in cosmosis are all + # in radians, so this is the easiest way to compare them. + for spectrum in self.two_point_data.spectra: + if spectrum.is_real_space(): + spectrum.convert_angular_units("rad") + # if self.options.get_bool("print physical scale",False): + # section,_,_=theory_names(spectrum) + # chi_peak = + # for ang in spectrum.angle: + + # build up the data vector from all the separate vectors. + # Just concatenation + data_vector = np.concatenate( + [spectrum.value for spectrum in self.two_point_data.spectra] + ) + + # Make sure + if len(data_vector) == 0: + raise ValueError( + "No data was chosen to be used from 2-point data file {0}. It was either not selectedin data_sets or cut out".format( + filename + ) + ) + + if self.moped: + data_file = fits.open(filename) + self.moped_data = data_file["MOPED-DATA-{}".format(self.moped)].data[ + "moped" + ] + self.moped_transform = data_file[ + "MOPED-TRANSFORM-{}".format(self.moped) + ].data + data_file.close() + + return None, self.moped_data + + # The x data is not especially useful here, so return None. + # We will access the self.two_point_data directly later to + # determine ell/theta values + return None, data_vector + + def build_covariance(self): + + C = np.array(self.two_point_data.covmat) + r = self.options.get_int("covariance_realizations", default=-1) + self.sellentin = self.options.get_bool("sellentin", default=False) + + if self.moped: + return np.identity(len(self.moped_data)) + + if self.sellentin: + if not self.constant_covariance: + print() + print("You asked for the Sellentin-Heavens correction to be applied") + print("But also asked for a non-constant (maybe Gaussian?) covariance") + print("matrix. I think that probably suggests you have made a mistake") + print("somewhere unless you have thought about this quite carefully.") + print() + if r < 0: + print() + print("ERROR: You asked for the Sellentin-Heavens corrections") + print( + "by setting sellentin=T, but you did not set covariance_realizations" + ) + print("If you want covariance_realizations=infinity you can use 0") + print( + "(unlikely, but it's also possible you were super-perverse and set it negative?)" + ) + print() + raise ValueError( + "Please set covariance_realizations for 2pt like. See message above." + ) + elif r == 0: + print() + print("NOTE: You asked for the Sellentin-Heavens corrections") + print("but set covariance_realizations=0. I am assuming you want") + print( + "the limit of an infinite number of realizations, so we will just go back" + ) + print("to the original Gaussian model") + print() + self.sellentin = False + else: + # use proper correction + self.covariance_realizations = r + print() + print( + "You set sellentin=T so I will apply the Sellentin-Heavens correction" + ) + print("for a covariance matrix estimated from Monte-Carlo simulations") + print("(you told us it was {} simulations in the ini file)".format(r)) + print( + "This analytic marginalization converts the Gaussian distribution" + ) + print("to a multivariate student's t distribution instead.") + print() + + elif r > 0: + # Just regular increase in covariance size, no Sellentin change. + p = C.shape[0] + # This x is the inverse of the alpha used in the old code + # because that applied to the weight matrix not the covariance + x = (r - 1.0) / (r - p - 2.0) + C = C * x + print() + print( + "You set covariance_realizations={} in the 2pt likelihood parameter file".format( + r + ) + ) + print( + "So I will apply the Anderson-Hartlap correction to the covariance matrix" + ) + print("The covariance matrix is nxn = {}x{}".format(p, p)) + print( + "So the correction scales the covariance matrix by (r - 1) / (r - n - 2) = {}".format( + x + ) + ) + print() + return C + + def extract_theory_points(self, block): + theory = [] + # We may want to save these splines for the covariance matrix later + self.theory_splines = {} + + # We have a collection of data vectors, one for each spectrum + # that we include. We concatenate them all into one long vector, + # so we do the same for our theory data so that they match + + # We will also save angles and bin indices for plotting convenience, + # although these are not actually used in the likelihood + angle = [] + bin1 = [] + bin2 = [] + + # Get appropriate suffixes + # If only a single suffix is provided, assume this applies to all data sets + if len(self.suffixes) == 1: + suffixes = np.tile(self.suffixes[0], len(self.two_point_data.spectra)) + elif len(self.suffixes) > 1 and len(self.suffixes) == len( + self.two_point_data.spectra + ): + suffixes = self.suffixes + else: + raise ValueError( + "The number of suffixes supplied does not match the number of two point spectra." + ) + + # Now we actually loop through our data sets + for ii, spectrum in enumerate(self.two_point_data.spectra): + theory_vector, angle_vector, bin1_vector, bin2_vector = ( + self.extract_spectrum_prediction(block, spectrum, suffixes[ii]) + ) + theory.append(theory_vector) + angle.append(angle_vector) + bin1.append(bin1_vector) + bin2.append(bin2_vector) + # dataset_name.append(np.repeat(spectrum.name, len(bin1_vector))) + + # We also collect the ell or theta values. + # The gaussian likelihood code itself is not expecting these, + # so we just save them here for convenience. + angle = np.concatenate(angle) + bin1 = np.concatenate(bin1) + bin2 = np.concatenate(bin2) + # dataset_name = np.concatenate(dataset_name) + block[names.data_vector, self.like_name + "_angle"] = angle + block[names.data_vector, self.like_name + "_bin1"] = bin1 + block[names.data_vector, self.like_name + "_bin2"] = bin2 + # block[names.data_vector, self.like_name+"_name"] = dataset_name + + # the thing it does want is the theory vector, for comparison with + # the data vector + theory = np.concatenate(theory) + + if self.moped: + return np.dot(self.moped_transform, theory) + + return theory + + def do_likelihood(self, block): + # Run the + super(TwoPointLikelihood, self).do_likelihood(block) + + if self.sellentin: + # The Sellentin-Heavens correction from arxiv 1511.05969 + # accounts for a finite number of Monte-Carlo realizations + # being used to estimate the covariance matrix. + + # Note that this invalidates the saved simulation used for + # the ABC sampler. I can't think of a better way of doing this + # than overwriting the whole things with NaNs - that will at + # least make clear there is a problem somewhere and not + # yield misleading results. + block[names.data_vector, self.like_name + "_simulation"] = ( + np.nan * block[names.data_vector, self.like_name + "_simulation"] + ) + + # It changes the Likelihood from Gaussian to a multivariate + # student's t distribution. Here we will have to do a little + # hack and overwrite the stuff that the original Gaussian + # method did above + N = self.covariance_realizations + chi2 = block[names.data_vector, self.like_name + "_CHI2"] + + # We might be using a cosmologically varying + # covariance matrix, though I'm not sure what that would mean. + # There is a warning about this above. + if self.constant_covariance: + log_det = 0.0 + else: + log_det = block[names.data_vector, self.like_name + "_LOG_DET"] + + like = -0.5 * log_det - 0.5 * N * np.log(1 + chi2 / (N - 1.0)) + + # overwrite the log-likelihood + block[names.likelihoods, self.like_name + "_LIKE"] = like + + # Should suffix be made into a keyword? + def extract_spectrum_prediction(self, block, spectrum, suffix): + + # We may need theory predictions for multiple different + # types of spectra: e.g. shear-shear, pos-pos, shear-pos. + # So first we find out from the spectrum where in the data + # block we expect to find these - mapping spectrum types + # to block names + section, x_name, y_name = theory_names(spectrum) + + # To handle multiple different data sets we allow a suffix + # to be applied to the section names, so that we can look up + # e.g. "shear_cl_des" instead of just "shear_cl". + section += suffix + + # Initialize TheorySpectrum class from block + bin_pairs = spectrum.get_bin_pairs() + theory_spec = TheorySpectrum.from_block(block, section, bin_pairs=bin_pairs) + + # If the theory spectrum has been bin-averaged then we expect the + # data to be so also. We check this by ensuring that angle_min is specified + # Based on this, we also generate the angle argument passed to the spectrum + # differently. The bin-averaged version expects a tuple angle_min and angle_max, + # whereas the interpolated version just wants a single angle. + if theory_spec.is_bin_averaged: + if spectrum.angle_min is None: + raise ValueError( + "Your theory pipeline produced angle-binnned values, but your data it not binned." + ) + angles = list(zip(spectrum.angle_min, spectrum.angle_max)) + else: + angles = spectrum.angle + + # We store the nominal mid-points for plotting later on, etc. + angle_mids = spectrum.angle + + # This is a bit of a hack, but later on if we are making a covariance + # we need all the splines, so pull them out here. + bin_splines = {} + + # We build up these vectors from all the data points. + # Only the theory vector is needed for the likelihood - the others + # are for convenience, debugging, etc. + theory_vector = [] + angle_vector = [] + bin1_vector = [] + bin2_vector = [] + + for b1, b2, angle, angle_mid in zip( + spectrum.bin1, spectrum.bin2, angles, angle_mids + ): + # The extra object will either be a spline (for interpolated spectra) + # or theta mid-point values (for bin-averaged ones, e.g. for plotting) + theory, extra = theory_spec.get_spectrum_value(b1, b2, angle) + + # We can only record the splines for non-bin-averaged values + if not theory_spec.is_bin_averaged: + bin_splines[y_name.format(b1, b2)] = extra + + # Build up the vector - we make this into an array later + theory_vector.append(theory) + angle_vector.append(angle_mid) + bin1_vector.append(b1) + bin2_vector.append(b2) + + self.theory_splines[section] = bin_splines + + # Return the whole collection as an array + theory_vector = np.array(theory_vector) + + # For convenience we also save the angle vector (ell or theta) + # and bin indices + angle_vector = np.array(angle_vector) + bin1_vector = np.array(bin1_vector, dtype=int) + bin2_vector = np.array(bin2_vector, dtype=int) + + return theory_vector, angle_vector, bin1_vector, bin2_vector + + def extract_covariance(self, block): + assert self.gaussian_covariance, ( + "Set constant_covariance=F but somehow not with Gaussian covariance. Internal error - please open an issue on the cosmosis site." + ) + + C = [] + # s and t index the spectra that we have. e.g. s or t=1 might be the full set of + # shear-shear measuremnts + for s, AB in enumerate(self.two_point_data.spectra[:]): + M = [] + for t, CD in enumerate(self.two_point_data.spectra[:]): + print( + "Looking at covariance between {} and {} (s={}, t={})".format( + AB.name, CD.name, s, t + ) + ) + # We only calculate the upper triangular. + # Get the lower triangular here. We have to + # transpose it compared to the upper one. + if s > t: + MI = C[t][s].T + else: + MI = gaussian_covariance.compute_gaussian_covariance( + self.sky_area, self._lookup_theory_cl, block, AB, CD + ) + M.append(MI) + C.append(M) + + # C is now a list of lists of 2D arrays. + # Now turn C into a big 2D array by stacking + # the arrays + C = np.vstack([np.hstack(CI) for CI in C]) + + return C + + def _lookup_theory_cl(self, block, A, B, i, j, ell): + """ + This is a helper function for the compute_gaussian_covariance code. + It looks up the theory value of C^{ij}_{AB}(ell) in the + """ + # We have already saved splines into the theory space earlier + # when constructing the theory vector. + # So now we just need to look those up again, using the same + # code we use in the twopoint library. + section, ell_name, value_name = type_table[A, B] + assert ell_name == "ell", ( + "Gaussian covariances are currently only written for C_ell, not other 2pt functions" + ) + d = self.theory_splines[section] + + # We save the splines with these names when we extract the theory vector + name_ij = value_name.format(i, j) + name_ji = value_name.format(j, i) + + # Hopefully we already have the theory spline extracted + if name_ij in d: + spline = d[name_ij] + # For symmetric spectra (not just auto-correlations, but any thing like C_EE or C_NN where + # we cross-correlate something with itself) we can use ji for ij as it is the same. This is + # not true for cross spectra + elif name_ji in d and (A == B): + spline = d[name_ji] + else: + # It's possible too that we need something for the covariance that we didn't need for the + # data vector - for example to got the covariance between C^EE and C^NN we need C^NE even + # if we don't have any actual measurements of NE. In that case we have to g + angle_theory = block[section, ell_name] + if block.has_value(section, name_ij): + theory = block[section, name_ij] + # The same symmetry argument as above applies + elif block.has_value(section, name_ji) and A == B: + theory = block[section, name_ji] + else: + raise ValueError( + "Could not find theory prediction {} in section {}".format( + value_name.format(i, j), section + ) + ) + + spline = interp1d(angle_theory, theory) + # Finally cache this so we don't have to do this again. + d[name_ij] = spline + + obs_cl = spline(ell) + + # For shear-shear the noise component is sigma^2 / number_density_bin + # and for position-position it is just 1/number_density_bin + if ( + (A == B) + and (A == twopoint.Types.galaxy_shear_emode_fourier.name) + and (i == j) + ): + if ( + i > len(self.number_density_shear_bin) + or i > len(self.sigma_e_bin) + or is_default(self.sigma_e_bin) + or is_default(self.number_density_shear_bin) + ): + raise ValueError("Not enough number density bins for shear specified") + noise = self.sigma_e_bin[i - 1] ** 2 / convert_nz_steradian( + self.number_density_shear_bin[i - 1] + ) + obs_cl += noise + if (A == B) and (A == twopoint.Types.galaxy_position_fourier.name) and (i == j): + if i > len(self.number_density_lss_bin) or is_default( + self.number_density_lss_bin + ): + raise ValueError("Not enough number density bins for lss specified") + noise = 1.0 / convert_nz_steradian(self.number_density_lss_bin[i - 1]) + obs_cl += noise + + return obs_cl + + def update_xi_w_sys(self, block): + self.data_y = self.raw_data_y + block["xi_sys", "xi_sys_vec"] + + @classmethod + def build_module(cls): + + def setup(options): + options = SectionOptions(options) + likelihoodCalculator = cls(options) + return likelihoodCalculator + + def execute(block, config): + likelihoodCalculator = config + likelihoodCalculator.update_xi_w_sys(block) + # print(likelihoodCalculator.data_y) + likelihoodCalculator.do_likelihood(block) + return 0 + + def cleanup(config): + likelihoodCalculator = config + likelihoodCalculator.cleanup() + + return setup, execute, cleanup + + +setup, execute, cleanup = TwoPointLikelihood.build_module() diff --git a/cosmo_inference/scripts/chain_postprocessing.py b/cosmo_inference/scripts/chain_postprocessing.py index d1462902..83cddf4b 100644 --- a/cosmo_inference/scripts/chain_postprocessing.py +++ b/cosmo_inference/scripts/chain_postprocessing.py @@ -7,7 +7,6 @@ import os import subprocess -import cs_util.cosmo as cs_cosmo import matplotlib.pyplot as plt import numpy as np from astropy.io import fits @@ -282,15 +281,16 @@ def compute_best_fit_xi_from_cell(output_folder, root, best_fit_params, theta_ra output_folder + "{}/best_fit/shear_cl/bin_1_1.txt".format(root) ) - cosmo = cs_cosmo.get_cosmo( - camb_params={ - "H0": best_fit_params["h0"], - "ombh2": best_fit_params["ombh2"], - "omch2": best_fit_params["omch2"], - "ns": best_fit_params["n_s"], - "sigma8": best_fit_params["SIGMA_8"], - }, - extra_params={ + import pyccl as ccl + + cosmo = ccl.Cosmology( + Omega_c=best_fit_params["omch2"] / (best_fit_params["h0"] / 100) ** 2, + Omega_b=best_fit_params["ombh2"] / (best_fit_params["h0"] / 100) ** 2, + h=best_fit_params["h0"] / 100, + n_s=best_fit_params["n_s"], + sigma8=best_fit_params["SIGMA_8"], + baryonic_effects=None, + extra_parameters={ "camb": { "halofit_version": "mead2020_feedback", "HMCode_logT_AGN": best_fit_params["logt_agn"], @@ -298,7 +298,9 @@ def compute_best_fit_xi_from_cell(output_folder, root, best_fit_params, theta_ra }, ) - xi_p, xi_m = cs_cosmo.c_ell_to_xi(cosmo, np.rad2deg(theta_rad) * 60, ell, shear_cl) + theta_deg = np.rad2deg(theta_rad) + xi_p = ccl.correlation(cosmo, ell=ell, C_ell=shear_cl, theta=theta_deg, type="GG+") + xi_m = ccl.correlation(cosmo, ell=ell, C_ell=shear_cl, theta=theta_deg, type="GG-") os.makedirs( output_folder + "{}/best_fit/shear_xi_minus".format(root), exist_ok=True diff --git a/cosmo_inference/scripts/cosmocov_process.py b/cosmo_inference/scripts/cosmocov_process.py new file mode 100644 index 00000000..c2c00996 --- /dev/null +++ b/cosmo_inference/scripts/cosmocov_process.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +import sys + +import matplotlib.pyplot as plt +import numpy as np + + +def get_cov(filename): + + data = np.loadtxt(filename) + ndata = int(np.max(data[:, 0])) + 1 + + print("Dimension of cov: %dx%d" % (ndata, ndata)) + + cov_g = np.zeros((ndata, ndata)) + cov_ng = np.zeros((ndata, ndata)) + for i in range(0, data.shape[0]): + cov_g[int(data[i, 0]), int(data[i, 1])] = data[i, 8] + cov_g[int(data[i, 1]), int(data[i, 0])] = data[i, 8] + cov_ng[int(data[i, 0]), int(data[i, 1])] = data[i, 9] + cov_ng[int(data[i, 1]), int(data[i, 0])] = data[i, 9] + + return cov_g, cov_ng, ndata + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: python cosmocov_process.py ") + sys.exit(1) + + covfile = sys.argv[1] + output_base = sys.argv[2] + + c_g, c_ng, ndata = get_cov(covfile) + + cov = c_ng + c_g + cov_g = c_g + + b = np.sort(np.linalg.eigvals(cov)) + print("min+max eigenvalues cov: %e, %e" % (np.min(b), np.max(b))) + if np.min(b) <= 0.0: + print("non-positive eigenvalue encountered! Covariance Invalid!") + exit() + + print("Covariance is positive definite!") + + np.savetxt(str(output_base) + ".txt", cov) + print("covmat saved as %s" % (str(output_base) + ".txt")) + + np.savetxt(str(output_base) + "_g.txt", cov_g) + print("Gaussian covmat saved as %s" % (str(output_base) + "_g.txt")) + + cmap = "seismic" + + pp_norm = np.zeros((ndata, ndata)) + for i in range(ndata): + for j in range(ndata): + pp_norm[i][j] = cov[i][j] / np.sqrt(cov[i][i] * cov[j][j]) + + print("Plotting correlation matrix ...") + + plot_path = str(output_base) + "_plot.pdf" + fig = plt.figure() + ax = fig.add_subplot(1, 1, 1) + extent = (0, ndata, ndata, 0) + im3 = ax.imshow(pp_norm, cmap=cmap, vmin=-1, vmax=1, extent=extent) + + plt.axvline(x=int(ndata / 2), color="black", linewidth=1.0) + plt.axhline(y=int(ndata / 2), color="black", linewidth=1.0) + + fig.colorbar(im3, orientation="vertical") + + ax.text(int(ndata / 4), ndata + 5, r"$\xi_+^{ij}(\theta)$", fontsize=12) + ax.text(3 * int(ndata / 4), ndata + 5, r"$\xi_-^{ij}(\theta)$", fontsize=12) + ax.text(-9, int(ndata / 4), r"$\xi_+^{ij}(\theta)$", fontsize=12) + ax.text(-9, 3 * int(ndata / 4), r"$\xi_-^{ij}(\theta)$", fontsize=12) + + plt.savefig(plot_path, dpi=2000) + plt.close() + print("Plot saved as %s" % (plot_path)) diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 7396ae35..6cfe8be8 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -3,8 +3,8 @@ """Prepare CosmoSIS inputs from UNIONS validation outputs. The script lives in ``cosmo_inference/scripts``. By default it reads templates -from ``cosmo_inference/cosmosis_config/templates`` and writes data products beneath -``cosmo_inference/data`` and ``cosmo_inference/cosmosis_config/output``. Override +from ``cosmo_inference/cosmosis_config`` and writes data products beneath +``cosmo_inference/data`` and ``cosmo_inference/cosmosis_config``. Override ``--template-dir`` or ``--output-root`` to use alternative locations. """ @@ -399,8 +399,8 @@ def _generate_ini_file( modifications.append((r"^\[output\]", output_section)) pipeline_section = ( - f"[pipeline]\nvalues = cosmosis_config/templates/{values_file}\npriors = " - f"cosmosis_config/templates/{priors_file}" + f"[pipeline]\nvalues = cosmosis_config/{values_file}\npriors = " + f"cosmosis_config/{priors_file}" ) modifications.append((r"^\[pipeline\]", pipeline_section)) @@ -617,7 +617,7 @@ def parse_args(): parser.add_argument( "--template-dir", type=str, - default=str(cosmo_inference_root / "cosmosis_config" / "templates"), + default=str(cosmo_inference_root / "cosmosis_config"), help=( "Directory containing CosmoSIS template INI files (defaults to the " "cosmosis_config folder next to this script)." @@ -636,7 +636,7 @@ def parse_args(): template_dir_path = Path(args.template_dir).expanduser().resolve() output_basename_path = Path(output_basename) data_dir_root = output_root_path / "data" / output_basename_path - config_dir_root = output_root_path / "cosmosis_config" / "output" + config_dir_root = output_root_path / "cosmosis_config" data_dir_root.mkdir(parents=True, exist_ok=True) config_dir_root.mkdir(parents=True, exist_ok=True) out_file_path = data_dir_root / f"cosmosis_{args.cosmosis_root}.fits" diff --git a/cosmo_inference/scripts/k_analysis.py b/cosmo_inference/scripts/k_analysis.py deleted file mode 100644 index d3939c10..00000000 --- a/cosmo_inference/scripts/k_analysis.py +++ /dev/null @@ -1,308 +0,0 @@ -import sys -from multiprocessing import Pool - -import astropy.constants as const -import astropy.units as u -import camb -import numpy as np -import scipy.integrate as integrate -from cs_util.cosmo import PLANCK18 -from scipy import interpolate -from scipy.special import j0, jn - -###################################################################################################### - -###################################################################################################### - - -def process_theta(theta, nz_file, output_root): - """Compute shear correlation functions for a single angular scale. - - For a given angular separation, this function computes the weak-lensing - correlation functions xi+ and xi- over a range of maximum wavenumbers - (kmax). The calculation includes nonlinear matter power spectra from - CAMB and optionally intrinsic-alignment contributions. Results are - appended to output text files. - - Parameters - ---------- - theta : float - Angular separation in arcminutes. - nz_file : str - Path to the source redshift distribution file. The file must contain - two columns giving redshift and n(z). - output_root : str - Prefix of the output files. Results are written to - ``{output_root}_xip.txt`` and ``{output_root}_xim.txt``. - - Returns - ------- - float - The input angular separation, returned for bookkeeping when running - in parallel. - """ - - def Hz(z): - """Return the Hubble expansion rate. - - Computes the Hubble parameter assuming a flat LCDM cosmology. - - Parameters - ---------- - z : float or ndarray - Redshift. - - Returns - ------- - float or ndarray - Hubble parameter in km s^-1 Mpc^-1. - """ - return H0 * np.sqrt(Omega_m * (1 + z) ** 3 + (1 - Omega_m)) - - def rz_interp(want_z): - """Create an interpolation between redshift and comoving distance. - - Computes the line-of-sight comoving distance by numerical integration - and returns an interpolation function in either direction. - - Parameters - ---------- - want_z : bool - If True, return an interpolator mapping comoving distance to - redshift. Otherwise return an interpolator mapping redshift to - comoving distance. - - Returns - ------- - scipy.interpolate.interp1d - Interpolation function relating redshift and comoving distance. - """ - - def hz_integrand(zz): - return c / Hz(zz) - - rz_ref = np.array([integrate.quad(hz_integrand, 0, z)[0] for z in zs]) - - if want_z == True: - return interpolate.interp1d( - rz_ref, zs, bounds_error=False, fill_value="extrapolate" - ) - else: - return interpolate.interp1d( - zs, rz_ref, bounds_error=False, fill_value="extrapolate" - ) - - def W_gg(z, rz): - """Compute the lensing efficiency kernel. - - Evaluates the lensing kernel for the supplied source redshift - distribution. - - Parameters - ---------- - z : float - Lens redshift. - rz : callable - Function returning comoving distance as a function of redshift. - - Returns - ------- - float - Weak-lensing efficiency kernel evaluated at z. - """ - z_integrate = np.linspace(z, zmax, n) - r_zmin = rz(z) - nz_int = som_nz_interp(z_integrate) * (1 - r_zmin / rz(z_integrate)) - prefactor = 3 * H0**2 * Omega_m * (1 + z) * r_zmin / (2 * c**2) - - return prefactor * integrate.simpson(nz_int, x=z_integrate) - - def C_ell(ell, kmax, want_IA): - """Compute the angular power spectrum. - - Calculates the Limber-approximated cosmic shear power spectrum, - optionally including intrinsic-alignment (GI and II) contributions. - - Parameters - ---------- - ell : float - Angular multipole. - kmax : float - Maximum wavenumber used to truncate the Limber integral. - want_IA : bool - If True, include intrinsic-alignment contributions. - - Returns - ------- - float - Total cosmic shear angular power spectrum at the specified multipole. - """ - z_min = rz_interp_wantz((ell + 0.5) / kmax) - z_valid = zs[zs >= z_min] - - if len(z_valid) == 0: - return 0.0 - - rzs = rz_interp_noz(z_valid) - W_ggs = W_gg_interp(z_valid) - Pks = pkz_nl_interp((z_valid, (ell + 0.5) / rzs)) - Hzs = Hz(z_valid) - - gg_integrand = c * W_ggs**2 * Pks / (Hzs * rzs**2) - C_ell_gg = integrate.simpson(gg_integrand, x=z_valid) - - if want_IA == True: - Dzs = pkz_lin_interp((z_valid, (ell + 0.5) / rzs)) / pkz_lin_interp( - (0, (ell + 0.5) / rzs) - ) - P_ia = -A_IA * c1 * Omega_m / Dzs - W_ias = Hzs * som_nz_interp(z_valid) / c - - gI_integrand = c * W_ggs * Pks * W_ias * P_ia / (Hzs * rzs**2) - II_integrand = c * Pks * W_ias**2 * P_ia**2 / (Hzs * rzs**2) - - C_ell_gI = integrate.simpson(gI_integrand, x=z_valid) - C_ell_II = integrate.simpson(II_integrand, x=z_valid) - - return C_ell_gg + C_ell_gI + C_ell_II - - return C_ell_gg - - def xi(theta_rad, kmax, want_IA): - """Compute the shear correlation functions. - - Evaluates the real-space shear correlation functions xi+ and xi- - by Hankel-transforming the convergence power spectrum. - - Parameters - ---------- - theta_rad : float - Angular separation in radians. - kmax : float - Maximum wavenumber used in the Limber integration. - want_IA : bool - If True, include intrinsic-alignment contributions. - - Returns - ------- - tuple of float - The pair (xi_plus, xi_minus). - """ - C_ell_vals = np.array([C_ell(ell, kmax, want_IA) for ell in ells]) - - xip_integrand = ells * C_ell_vals * j0(ells * theta_rad) - xim_integrand = ells * C_ell_vals * jn(4, ells * theta_rad) - - return integrate.simpson(xip_integrand, x=ells) / ( - 2 * np.pi - ), integrate.simpson(xim_integrand, x=ells) / (2 * np.pi) - - ########################################################################################### - - c = const.c.to("km/s") - H0 = PLANCK18["h"] * 100 - Omega_m = PLANCK18["Omega_m"] - - A_IA = 0.83 - c1 = 5e-14 * (u.Mpc**3.0) / u.solMass - - zmin = 1e-5 - zmax = 4 - n = 500 - zs = np.linspace(zmin, zmax, n) - ells = np.linspace(2, 1e5, int(1e5 - 1)) - - kmaxs = np.logspace(-4, 2, 200) - theta_rad = theta * (np.pi / (180 * 60)) - - ombh2 = PLANCK18["Omega_b"] * PLANCK18["h"] ** 2 - omch2 = (PLANCK18["Omega_m"] - PLANCK18["Omega_b"]) * PLANCK18["h"] ** 2 - pars = camb.set_params( - H0=H0, - ombh2=ombh2, - omch2=omch2, - mnu=PLANCK18["m_nu"], - As=PLANCK18["As"], - ns=PLANCK18["n_s"], - halofit_version="mead2020_feedback", - lmax=3000, - WantTransfer=True, - ) - - nz_z, som_nz = np.loadtxt(f"{nz_file}", unpack=True) - som_nz_interp = interpolate.interp1d( - nz_z, som_nz, bounds_error=False, fill_value=None - ) - - pars.set_matter_power(redshifts=np.linspace(zmin, zmax, 150), kmax=200) - results = camb.get_results(pars) - results.calc_power_spectra(pars) - k_nonlin, z_nonlin, pk_nonlin = results.get_nonlinear_matter_power_spectrum( - hubble_units=False, k_hunit=False - ) - - pkz_nl_interp = interpolate.RegularGridInterpolator( - (z_nonlin, k_nonlin), pk_nonlin, bounds_error=False, fill_value=None - ) - - k_lin, z_lin, pk_lin = results.get_linear_matter_power_spectrum( - hubble_units=False, k_hunit=False - ) - - pkz_lin_interp = interpolate.RegularGridInterpolator( - (z_lin, k_lin), pk_lin, bounds_error=False, fill_value=None - ) - - rz_interp_wantz = rz_interp(True) - rz_interp_noz = rz_interp(False) - W_gg_vals = np.array([W_gg(z, rz_interp_noz) for z in zs]) - W_gg_interp = interpolate.interp1d( - zs, W_gg_vals, bounds_error=False, fill_value="extrapolate" - ) - - ########################################################################################### - xis = np.array([xi(theta_rad, kmax, True) for kmax in kmaxs]) - xip = xis[:, 0] - xim = xis[:, 1] - - # Write results immediately to avoid thread conflicts - with open(f"{output_root}_xip.txt", "a") as f: - new_arr = np.concatenate(([theta], xip)) - np.savetxt(f, new_arr, fmt="%.8e") - - with open(f"{output_root}_xim.txt", "a") as f: - new_arr = np.concatenate(([theta], xim)) - np.savetxt(f, new_arr, fmt="%.8e") - - return theta - - -########################################################################################### - -if __name__ == "__main__": - """Run the shear-correlation calculation in parallel. - - The script expects three command-line arguments: - - 1. Block index specifying which subset of angular scales to process. - 2. Path to the source redshift distribution file. - 3. Output file prefix. - - The 50 angular scales between 1 and 20 arcmin are divided into - blocks of 10 values. Each block is processed in parallel using - multiprocessing, with one worker per angular scale. Each worker - computes xi+ and xi- over the predefined range of kmax values and - appends the results to the output files. - """ - i = int(sys.argv[1]) - nz_file = sys.argv[2] - output_root = sys.argv[3] - - thetas = np.linspace(1, 20, 50) - theta_block = thetas[i * 10 : (i + 1) * 10] - - # Run in parallel to speed up calculations for multiple angular scales - with Pool(processes=10) as pool: - pool.starmap( - process_theta, [(theta, nz_file, output_root) for theta in theta_block] - ) diff --git a/cosmo_inference/scripts/masking.py b/cosmo_inference/scripts/masking.py new file mode 100644 index 00000000..ba2f3c4c --- /dev/null +++ b/cosmo_inference/scripts/masking.py @@ -0,0 +1,319 @@ +import argparse +import os +from multiprocessing import Pool, cpu_count +from pathlib import Path + +import h5py +import healpy as hp +import numpy as np +import yaml + +# ------------------------- +# Spatially-structured cuts: these define the survey footprint. +# All other cuts (FLAGS, mag, SNR, shape measurement, PSF ellipticity, +# relative size) are per-galaxy quality cuts that should NOT affect +# the footprint definition. +SPATIAL_CUTS = { + "overlap", + "IMAFLAGS_ISO", + "N_EPOCH", + "4_Stars", + "8_Manual", + "64_r", + "1024_Maximask", + "npoint3", + "1_Faint_star_halos", + "2_Bright_star_halos", +} + +# ------------------------- +# Masking logic + + +def apply_condition(array, kind, value): + """ + Apply a logical condition to a NumPy array and return a boolean mask, based + on the "kind" key in the mask config YAML file. + """ + if kind == "equal": + return array == value + elif kind == "not_equal": + return array != value + elif kind == "greater_equal": + return array >= value + elif kind == "greater": + return array > value + elif kind == "less_equal": + return array <= value + elif kind == "less": + return array < value + elif kind == "range": + return (array >= value[0]) & (array <= value[1]) + else: + raise ValueError(f"Unknown kind: {kind}") + + +def apply_masks(data, data_ext, mask_config, footprint_only=False): + """ + Construct a boolean mask selecting galaxies that satisfy all + masking criteria defined in the YAML configuration file. + + Parameters + ---------- + data : numpy.ndarray or structured array + Slice of the HDF5 "data" group containing per-object + measurements (e.g. FLAGS, mag, NGMIX quantities). + + data_ext : numpy.ndarray or structured array + Slice of the HDF5 "data_ext" group containing external or + post-processing flags (e.g. star masks, footprint flags). + + mask_config : dict + Dictionary parsed from the YAML mask configuration file. + Expected structure: + - mask_config["dat"] : list of cuts applied to `data` + - mask_config["dat_ext"] : list of cuts applied to `data_ext` + - mask_config["metacal"] : derived-quantity parameters + (e.g. relative size limits) + + footprint_only : bool, optional + If True, only apply spatially-structured cuts (those in + SPATIAL_CUTS). Skips per-galaxy quality cuts (FLAGS, mag, + SNR, shape measurement, PSF ellipticity, relative size). + Used to define a consistent footprint from the comprehensive + catalog. Default is False. + + Returns + ------- + numpy.ndarray (bool) + Boolean array of length equal to the input data slice. + True indicates the object passes all cuts (kept), + False indicates the object is masked (removed). + """ + + # Initialize mask + mask = np.ones(len(data), dtype=bool) + + # --- dat group --- + for cut in mask_config.get("dat", []): + col = cut["col_name"] + if footprint_only and col not in SPATIAL_CUTS: + continue + kind = cut["kind"] + value = cut["value"] + + mask &= apply_condition(data[col], kind, value) + + # --- dat_ext group --- + for cut in mask_config.get("dat_ext", []): + col = cut["col_name"] + if footprint_only and col not in SPATIAL_CUTS: + continue + kind = cut["kind"] + value = cut["value"] + + mask &= apply_condition(data_ext[col], kind, value) + + # --- metacal relative size (skip for footprint-only) --- + if not footprint_only: + rel_size = np.divide( + data["NGMIX_T_NOSHEAR"], + data["NGMIX_T_PSF_RECONV_NOSHEAR"], + out=np.zeros_like(data["NGMIX_T_NOSHEAR"]), + where=(data["NGMIX_T_PSF_RECONV_NOSHEAR"] > 0), + ) + + rel_min = mask_config["metacal"]["gal_rel_size_min"] + rel_max = mask_config["metacal"]["gal_rel_size_max"] + + mask &= (rel_size >= rel_min) & (rel_size <= rel_max) + + return mask + + +# ------------------------- +# Process one chunk +def process_chunk(args): + """ + Process a chunk of the HDF5 catalogue and return the unique + HEALPix pixels containing unmasked galaxies,to be executed in + parallel. It reads a slice of the catalogue, applies + the defined masking criteria, converts the sky positions + (RA, Dec) of retained galaxies into HEALPix pixel indices, + and returns the unique pixel indices for that chunk. + + Parameters + ---------- + args : tuple + Tuple containing: + - start : int + Starting row index of the chunk (inclusive). + - stop : int + Ending row index of the chunk (exclusive). + - filename : str + Path to the input HDF5 catalogue. + - nside : int + HEALPix NSIDE parameter defining map resolution. + - mask_config : dict + Parsed YAML mask configuration. + + Returns + ------- + numpy.ndarray + Array of unique HEALPix pixel indices (int) corresponding + to sky locations of galaxies that pass all mask cuts in + this chunk. + """ + + start, stop, filename, nside, mask_config, footprint_only = args + with h5py.File(filename, "r") as f: + data = f["data"][start:stop] + data_ext = f["data_ext"][start:stop] + + mask = apply_masks(data, data_ext, mask_config, footprint_only=footprint_only) + + ra = data["RA"][mask] + dec = data["Dec"][mask] + + theta = np.radians(90.0 - dec) # colatitude + phi = np.radians(ra) # longitude + + pix = hp.ang2pix(nside, theta, phi) + + return np.unique(pix) + + +# ------------------------- +# Build mask map in parallel +def build_mask_map_hdf5( + filename, mask_config, nside, chunk_size=1_000_000, footprint_only=False +): + """ + Build a binary HEALPix mask map from an HDF5 galaxy catalogue. + + The catalogue is processed in chunks to limit memory usage. + + Parameters + ---------- + filename : str + Path to the input HDF5 catalogue containing "data" and + "data_ext" groups + mask_config : dict + Dictionary parsed from the YAML mask configuration file + nside : int + HEALPix NSIDE parameter defining the resolution of the + output map. + chunk_size : int, optional + Number of catalogue rows to process per chunk. + Default is 1,000,000. + footprint_only : bool, optional + If True, only apply spatially-structured cuts. + + Returns + ------- + numpy.ndarray + One-dimensional HEALPix map (dtype uint8) of length + hp.nside2npix(nside), where: + - 1 indicates at least one unmasked galaxy falls + in that pixel, + - 0 indicates no retained galaxies. + """ + with h5py.File(filename, "r") as f: + nrows = f["data"].shape[0] + + chunks = [ + (i, min(i + chunk_size, nrows), filename, nside, mask_config, footprint_only) + for i in range(0, nrows, chunk_size) + ] + + mask_map = np.zeros(hp.nside2npix(nside), dtype=np.uint8) + + with Pool(cpu_count()) as pool: + for pix_indices in pool.imap_unordered(process_chunk, chunks): + mask_map[pix_indices] = 1 + + return mask_map + + +############################################################################################################ +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Build HEALPix mask from HDF5 catalog") + parser.add_argument("nside", type=int, help="HEALPix NSIDE parameter") + parser.add_argument("--config", required=True, help="Path to mask config YAML") + parser.add_argument( + "--output-prefix", + required=True, + help="Output file prefix (e.g. 'footprint' or 'footprint_starhalo')", + ) + parser.add_argument( + "--footprint-only", + action="store_true", + help="Only apply spatially-structured cuts (for footprint definition)", + ) + parser.add_argument( + "--output-dir", + default=None, + help="Output directory (default: data/mask/ relative to script)", + ) + args = parser.parse_args() + + nside = args.nside + curr_dir = Path(os.path.dirname(os.path.abspath(__file__))) + + if args.output_dir: + out_dir = Path(args.output_dir) + else: + out_dir = curr_dir.parent / "data" / "mask" + out_dir.mkdir(parents=True, exist_ok=True) + + with open(args.config, "r") as f: + mask_config = yaml.safe_load(f) + + filename = f"/n17data/UNIONS/WL/v1.4.x/v1.4.5/{mask_config['params']['input_path']}" + prefix = args.output_prefix + + if args.footprint_only: + print(f"Footprint-only mode: applying only spatial cuts {SPATIAL_CUTS}") + + # Build mask map from comprehensive catalogue + mask_map = build_mask_map_hdf5( + filename, + mask_config, + nside, + chunk_size=500_000, + footprint_only=args.footprint_only, + ) + + # Get survey area after masking + npix = hp.nside2npix(nside) + pix_area_sr = 4 * np.pi / npix + pix_area_deg2 = (180 / np.pi) ** 2 * pix_area_sr + n_obs = mask_map.sum() + f_sky_obs = n_obs / npix + area_obs_deg2 = n_obs * pix_area_deg2 + print(f"Kept area = {area_obs_deg2:.2f} deg^2\n") + + # Compute Cls of the mask map + cl_mask = hp.anafast(mask_map, lmax=3 * nside - 1) + ells = np.arange(len(cl_mask)) + + # Save mask map and Cls + map_path = out_dir / f"mask_map_{prefix}_nside_{nside}.fits" + cls_path = out_dir / f"mask_cls_{prefix}_nside_{nside}.npz" + hp.write_map(map_path, mask_map, overwrite=True) + np.savez(cls_path, ells=ells, cl_mask=cl_mask) + + print(f"Mask map saved to {map_path}") + print(f"Mask Cls saved to {cls_path}\n") + + # Compute normalising factor for the mask Cls + integral_w = np.sum((2 * ells + 1) / (4 * np.pi) * cl_mask) / (np.pi / 180) ** 2 + norm_factor = area_obs_deg2 / integral_w + norm_cls = cl_mask * norm_factor + + # Save normalised Cls to text file + norm_path = out_dir / f"mask_cls_{prefix}_nside_{nside}_norm.txt" + idx = np.arange(len(cl_mask)) + data_to_save = np.column_stack((idx, norm_cls)) + np.savetxt(norm_path, data_to_save, fmt=["%d", "%.10e"]) + print(f"Normalised mask Cls saved to {norm_path}") diff --git a/cosmo_inference/scripts/matching.py b/cosmo_inference/scripts/matching.py new file mode 100644 index 00000000..a465e449 --- /dev/null +++ b/cosmo_inference/scripts/matching.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Mar 1 17:37:27 2023 +@author: fh272693 +""" + +import astropy.units as u +from astropy.coordinates import SkyCoord, match_coordinates_sky +from astropy.io import fits + +Cat1 = fits.open( + "/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/unions_shapepipe_2022_v1.0.fits" +) +Cat2 = fits.open( + "/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/lensfit_goldshape_2022v1.fits" +) + +coord_units = u.degree +Cat1_coord = SkyCoord( + ra=Cat1[1].data["ra"] * coord_units, dec=Cat1[1].data["dec"] * coord_units +) +Cat2_coord = SkyCoord( + ra=Cat2[1].data["ra"] * coord_units, dec=Cat2[1].data["dec"] * coord_units +) +idx, d2d, d3d = match_coordinates_sky(Cat1_coord, Cat2_coord) +max_sep = 1.0 * u.arcsec +sep_constraint = d2d < max_sep + +# Important here is that the first catalogue of match_coordinates_sky has +# indices [sep_constraint] and the second[idx[sep_constraint]] +Cat1_matches = Cat1[1].data[sep_constraint] + +# np.save('/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/shapepipe_unmatches_ra.npy',Cat1[1].data['ra'][sep_constraint]) +# np.save('/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/shapepipe_unmatches_dec.npy',Cat1[1].data['dec'][sep_constraint]) +# np.save('/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/shapepipe_unmatches_e1.npy',Cat1[1].data['e1'][sep_constraint]) +# np.save('/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/shapepipe_unmatches_e2.npy',Cat1[1].data['e2'][sep_constraint]) +# np.save('/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/shapepipe_unmatches_w.npy',Cat1[1].data['w'][sep_constraint]) + +print("there are ", len(Cat1_matches), " matching galaxies in catalogue", Cat1) +# print('there are ',len(Cat2_matches),' matching galaxies in catalogue', Cat2) diff --git a/cosmo_inference/scripts/nz_writeout.py b/cosmo_inference/scripts/nz_writeout.py new file mode 100644 index 00000000..81994335 --- /dev/null +++ b/cosmo_inference/scripts/nz_writeout.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[ ]: + +import sys + +import matplotlib.pylab as plt +import numpy as np +from astropy.io import fits + +nz_hdu = sys.argv[1] +root = sys.argv[2] +blind = sys.argv[3] + +hdu = fits.open(nz_hdu) +z = hdu[1].data["Z_%s" % blind] + +zmax = 5.0 + +(n, bins, _) = plt.hist(z, bins=200, range=(0, zmax), density=True, weights=None) + +print("zmin = ", min(z)) +print("zmax = ", max(z)) + +np.savetxt("data/" + root + "/nz_" + root + ".txt", np.column_stack((bins[:-1], n))) diff --git a/cosmo_inference/scripts/slurm.sh b/cosmo_inference/scripts/slurm.sh new file mode 100644 index 00000000..793aa7cf --- /dev/null +++ b/cosmo_inference/scripts/slurm.sh @@ -0,0 +1,22 @@ +#!/bin/bash +#SBATCH --job-name=unions_V1.4 +#SBATCH --mail-user=lgoh@roe.ac.uk +#SBATCH --mail-type=END,FAIL +#SBATCH --partition=compl +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=48 +#SBATCH --time=4-00:00:00 +#SBATCH --output=/n23data1/n06data/lgoh/scratch/CFIS-UNIONS/chains/SP_v1.4_A/inference_A.log + +module load gcc +module load intelpython/3-2024.1.0 +module load openmpi +source cosmosis-configure +source activate my_env + +cosmosis --mpi /n23data1/n06data/lgoh/scratch/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_1.ini + +# +# Return exit code +exit 0 \ No newline at end of file diff --git a/cosmo_inference/scripts/treecorr_calc.py b/cosmo_inference/scripts/treecorr_calc.py new file mode 100644 index 00000000..38eb855e --- /dev/null +++ b/cosmo_inference/scripts/treecorr_calc.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# coding: utf-8 + + +import os +import sys + +import numpy as np +import treecorr +from astropy.io import fits + +script_dir = os.path.dirname(os.path.abspath(sys.argv[0])) + +cat_name = sys.argv[1] +root = sys.argv[2] + +hdu = fits.open(cat_name) +data = hdu[1].data + +# Create TreeCorr catalogue +n_thread = 8 +treecorr.set_omp_threads(n_thread) + +sep_units = "arcmin" +nbins = 20 + +TreeCorrConfig = { + "ra_units": "degrees", + "dec_units": "degrees", + "max_sep": "200", + "min_sep": "1", + "sep_units": sep_units, + "nbins": nbins, + "var_method": "jackknife", +} + + +cat_gal = treecorr.Catalog( + ra=data["RA"], + dec=data["Dec"], + g1=data["e1_noleakage"], # for v1.4.1 + g2=data["e2_noleakage"], # for v1.4.1 + w=data["w"], + ra_units="degrees", + dec_units="degrees", + npatch=50, +) + +gg = treecorr.GGCorrelation(TreeCorrConfig) + +print("Running TreeCorr...") +gg.process(cat_gal) + + +lst = np.arange(1, nbins + 1) + +# create fits HDU with xi_p and xi_m data +col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) +col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) +col3 = fits.Column(name="ANGBIN", format="K", array=lst) +col4 = fits.Column(name="VALUE", format="D", array=gg.xip) +col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=gg.meanr) +coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) +xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") + + +col4 = fits.Column(name="VALUE", format="D", array=gg.xim) +coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) +ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") + +# append xi_p/xi_m header info +xip_dict = { + "2PTDATA": "T", + "QUANT1": "G+R", + "QUANT2": "G+R", + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", +} +for key in xip_dict: + xiplus_hdu.header[key] = xip_dict[key] + + +xim_dict = { + "2PTDATA": "T", + "QUANT1": "G-R", + "QUANT2": "G-R", + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", +} + +for key in xim_dict: + ximinus_hdu.header[key] = xim_dict[key] + +ximinus_hdu.writeto( + "%s/../data/" % script_dir + root + "/ximinus_" + root + ".fits", overwrite=True +) +xiplus_hdu.writeto( + "%s/../data/" % script_dir + root + "/xiplus_" + root + ".fits", overwrite=True +) + +print( + "Correlation functions written to {}".format( + "%s/../data/" % script_dir + root + "/xiplus_minus_" + root + ".fits" + ) +) diff --git a/cosmo_inference/scripts/xi_sys_psf.py b/cosmo_inference/scripts/xi_sys_psf.py new file mode 100644 index 00000000..631b9464 --- /dev/null +++ b/cosmo_inference/scripts/xi_sys_psf.py @@ -0,0 +1,53 @@ +import numpy as np +from astropy.io import fits +from cosmosis.datablock import option_section + + +# This file should be added to your cosmosis_standard_library following the path shear/xi_sys/xi_sys_psf.py +def setup(options): + filename = options.get_string(option_section, "data_file") + data = fits.open(filename) + rho_stats_name = options.get_string(option_section, "rho_stats_name") + samples_path = options.get_string(option_section, "samples") + + samples = np.load(samples_path) + mean = np.mean(samples, axis=0) + cov = np.cov(samples.T) + + rho_stats = data[rho_stats_name].data + + return mean, cov, rho_stats + + +def execute(block, config): + + mean, cov, rho_stats = config + + alpha, beta, eta = np.random.multivariate_normal(mean, cov) + block["xi_sys", "alpha"], block["xi_sys", "beta"], block["xi_sys", "eta"] = ( + alpha, + beta, + eta, + ) + + xi_sys_p = ( + alpha**2 * rho_stats["rho_0_p"] + + beta**2 * rho_stats["rho_1_p"] + + eta**2 * rho_stats["rho_3_p"] + + 2 * alpha * beta * rho_stats["rho_2_p"] + + 2 * beta * eta * rho_stats["rho_4_p"] + + 2 * alpha * eta * rho_stats["rho_5_p"] + ) + + xi_sys_m = ( + alpha**2 * rho_stats["rho_0_m"] + + beta**2 * rho_stats["rho_1_m"] + + eta**2 * rho_stats["rho_3_m"] + + 2 * alpha * beta * rho_stats["rho_2_m"] + + 2 * beta * eta * rho_stats["rho_4_m"] + + 2 * alpha * eta * rho_stats["rho_5_m"] + ) + + block["xi_sys", "xi_sys_vec"] = np.concatenate([xi_sys_p, xi_sys_m]) + + return 0 diff --git a/papers/bmodes/scripts/run_xi_sweep.py b/papers/bmodes/scripts/run_xi_sweep.py index b02283a5..49076435 100644 --- a/papers/bmodes/scripts/run_xi_sweep.py +++ b/papers/bmodes/scripts/run_xi_sweep.py @@ -2,8 +2,8 @@ Loops the [non-fiducial version list](sweep_versions.nonfiducial_versions) and runs the same ``run_2pcf.run_2pcf`` compute the fiducial two_point recipes call, -once per version, writing every version's ξ± text dump (+ ξ+/ξ- FITS) into one -lc ``{output}`` dir under run_2pcf's native, already-canonical name +once per version, writing every version's ξ± text dump into one lc ``{output}`` +dir under run_2pcf's native, already-canonical name ``{ver}_xi_minsep={min}_maxsep={max}_nbins={nbins}_npatch={npatch}.txt`` — the exact pattern ``cosebis_version_comparison._xi_integration`` reconstructs. @@ -72,11 +72,15 @@ def _from_cli(argv=None): versions = a.versions or nonfiducial_versions(config) for ver in versions: for grid in a.grids: + # The sweep consumes only the .txt dump (cosebis_version_comparison + # reconstructs it by binning). run_2pcf is born-as-SACC, so give its + # coarse part a grid-qualified name — the default {ver}_xi_coarse.sacc + # carries no binning, so the two grids per version would collide. run_2pcf( ver=ver, cat_config=a.cat_config, output_dir=a.out, - save_fits=True, + sacc_out=os.path.join(a.out, f"{ver}_xi_coarse_{grid}.sacc"), **GRIDS[grid], ) diff --git a/papers/realspace/S8_om_sigma8_whisker.py b/papers/realspace/S8_om_sigma8_whisker.py deleted file mode 100644 index 9f0aecae..00000000 --- a/papers/realspace/S8_om_sigma8_whisker.py +++ /dev/null @@ -1,549 +0,0 @@ -# -# This notebook plots the whisker plot of $S_8$, $\Omega_m$ and $\sigma_8$ - - -import os -import sys - -# Trick to plot with tex -os.environ["LD_LIBRARY_PATH"] = "" -os.environ["CONDA_PREFIX"] = "/home/guerrini/.conda/envs/sp_validation_3.11" - -import warnings - -import matplotlib.pyplot as plt -import numpy as np -import seaborn as sns -from getdist import plots - -sys.path.append("/home/guerrini/sp_validation/cosmo_inference/scripts") - -import chain_postprocessing as cp - -plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") - -plt.rc("text", usetex=True) - -sns.set_palette("husl") - -g = plots.get_subplot_plotter(width_inch=30) -g.settings.axes_fontsize = 60 -g.settings.axes_labelsize = 60 -g.settings.alpha_filled_add = 0.7 -g.settings.legend_fontsize = 60 - - -# SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE -root_dir = "/n09data/guerrini/output_chains/" -root_external = f"{root_dir}/ext_data/" -blind = "B" - -roots = [ - f"SP_v1.4.6.3_{blind}_fiducial_config", - f"SP_v1.4.6.3_leak_corr_{blind}", - "Planck18", - "DES Y6", - "KiDS-Legacy_bandpowers", - "KiDS-Legacy_cosebis", - "KiDS-Legacy_xipm", - "HSC_Y3", - "HSC_Y3_cell", - f"SP_v1.4.6.3_{blind}_small_scales_config", - f"SP_v1.4.6.3_{blind}_flat_alpha_beta_config", - f"SP_v1.4.6.3_{blind}_no_xi_sys_config", - f"SP_v1.4.6.3_{blind}_no_leak_corr_config", - f"SP_v1.4.6.3_{blind}_flat_delta_z_config", - f"SP_v1.4.6.3_{blind}_no_delta_z_config", - f"SP_v1.4.6.3_{blind}_flat_ia_config", - f"SP_v1.4.6.3_{blind}_no_ia_config", - f"SP_v1.4.6.3_{blind}_no_m_bias_config", - f"SP_v1.4.6.3_{blind}_unmasked_covmat_config", - f"SP_v1.4.6.3_{blind}_halofit_config", - f"SP_v1.4.6.3_{blind}_no_baryons_config", - f"SP_v1.4.6.3_{blind}_nautilus_config", - f"SP_v1.4.6.3_{blind}_planck_config", - f"SP_v1.4.6.3_{blind}_planck_desi_config", -] - -legend_labels = [ - r"UNIONS-3500 $\xi_{\pm}(\theta)$ (This work)", - r"UNIONS-3500 $C_\ell$ (Guerrini et al. 2026)", - r"$\textit{Planck}$ 2018", - r"DES Y6 $\xi_{\pm}$, NLA", - r"KiDS-Legacy Bandpowers ($C_{\rm E}$)", - r"KiDS-Legacy COSEBIs ($E_n$)", - r"KiDS-Legacy $\xi_{\pm}(\theta)$", - r"HSC-Y3 $\xi_{\pm}(\theta)$", - r"HSC-Y3 $C_\ell$", - r"$\xi_+$ small scales, $\theta$=[5,83] arcmin", - r"Flat $\alpha_{\rm{PSF}}$ and $\beta_{\rm{PSF}}$ priors", - r"No $\xi^{\rm sys}_{\pm}$", - r"No leakage correction", - r"Flat $\Delta z$ priors", - r"No $\Delta z$", - r"Flat $A_{\rm IA}$ prior", - r"No $A_{\rm IA}$", - r"No $m$ bias", - r"Unmasked covmat", - r"$\texttt{Halofit}$", - r"$\texttt{HMCode}$ no baryons", - r"Nautilus sampler", - r"UNIONS-3500 + $\textit{Planck}$", - r"UNIONS-3500 + $\textit{Planck}$ + DESI BAO", -] - -categories = [ - "configuration", - "harmonic", - "external", - "external", - "external", - "external", - "external", - "external", - "external", - "configuration", - "configuration", - "configuration", - "configuration", - "configuration", - "configuration", - "configuration", - "configuration", - "configuration", - "configuration", - "configuration", - "configuration", - "configuration", - "configuration", - "configuration", -] -colours = [ - "darkorange", - "royalblue", - "violet", - "black", - "black", - "black", - "black", - "black", - "black", - "forestgreen", - "forestgreen", - "forestgreen", - "forestgreen", - "forestgreen", - "forestgreen", - "forestgreen", - "forestgreen", - "forestgreen", - "forestgreen", - "forestgreen", - "forestgreen", - "forestgreen", - "forestgreen", - "forestgreen", -] - - -chains = [] -for i, root in enumerate(roots): - category = categories[i] - if root == "DES Y6": - continue - if category != "external": - if category == "configuration": - path_samples = os.path.join(root_dir, f"{root}/samples_{root}.txt") - path_getdist = os.path.join(root_dir, f"{root}/getdist_{root}") - elif category == "harmonic": - path_samples = os.path.join( - root_dir, f"{root}/{root}/samples_{root}_cell.txt" - ) - path_getdist = os.path.join(root_dir, f"{root}/{root}/getdist_{root}") - elif category == "external_compute_sample": - path_samples = os.path.join(root_dir, f"ext_data/{root}/samples_{root}.txt") - path_getdist = os.path.join(root_dir, f"ext_data/{root}/getdist_{root}") - else: - raise ValueError(f"The category, {category}, of {root} is not correct") - if "nautilus" not in root: - cp.load_samples_and_write_paramnames( - path_samples, path_getdist + ".paramnames" - ) - cp.write_samples_getdist_format(path_samples, path_getdist + ".txt") - else: - cp.load_samples_and_write_paramnames( - path_samples, path_getdist + ".paramnames", chain_type="nautilus" - ) - cp.write_samples_getdist_format( - path_samples, path_getdist + ".txt", chain_type="nautilus" - ) - chains.append(cp.load_chain(path_getdist, smoothing_scale=0.5)) - else: - path_getdist = os.path.join(root_dir, f"ext_data/{root}/getdist_{root}") - chains.append(cp.load_chain(path_getdist)) - - -name_list = [ - "OMEGA_M", - "ombh2", - "h0", - "n_s", - "SIGMA_8", - "S_8", - "s_8_input", - "logt_agn", - "a", - "m1", - "bias_1", -] -label_list = [ - r"\Omega_{\rm m}", - r"\omega_b h^2", - r"h_0", - r"n_s", - r"\sigma_8", - r"S_8", - r"S_8", - r"\log T_{\rm AGN}", - r"A_{\rm IA}", - r"m_1", - r"\Delta z_1", -] - -for i, chain in enumerate(chains): - print(legend_labels[i]) - param_names = chain.getParamNames() - for name, label in zip(name_list, label_list): - try: - param_names.parWithName(name).label = label - except Exception: - warnings.warn(f"Parameter {name} not found in chain {roots[i]}.") - - -# Micro management of external chains - -# Account for the missing parameter conventions - -idx = roots.index("KiDS-Legacy_xipm") -cp.derive_parameter_S8(chains[idx]) - -idx = roots.index("KiDS-Legacy_bandpowers") -cp.derive_parameter_S8(chains[idx]) - -idx = roots.index("KiDS-Legacy_cosebis") -cp.derive_parameter_S8(chains[idx]) - -# OMEGA_M not in HSC_Y3_cell -idx = roots.index("HSC_Y3_cell") -cp.adjust_paramname_chain(chains[idx], "omega_m", "OMEGA_M", r"\Omega_{\rm m}") - - -param_values = np.array( - [ - "# Expt", - "Colour", - "S8_Mean", - "S8_low", - "S8_high", - "sigma_8_Mean", - "sigma_8_low", - "sigma_8_high", - "Omega_m_Mean", - "Omega_m_low", - "Omega_m_high", - ] -) -escaped = np.char.replace(legend_labels, "\\", "\\\\") - -for i, root in enumerate(roots): - chain = chains[i] - if root == "DES Y6": - param_values = np.vstack( - ( - param_values, - [ - escaped[i], - colours[i], - 0.798, - 0.015, - 0.014, - 0.763, - 0.057, - 0.050, - 0.332, - 0.040, - 0.035, - ], - ) - ) - else: - best_fit_params = cp.extract_best_fit_params(chain, best_fit_method="2Dkde") - margestats = chain.getMargeStats() - - s8_stats = margestats.parWithName("S_8") - sigma8_stats = margestats.parWithName("SIGMA_8") - omegam_stats = margestats.parWithName("OMEGA_M") - - param_values = np.vstack( - ( - param_values, - [ - escaped[i], - colours[i], - best_fit_params["S_8"], - best_fit_params["S_8"] - s8_stats.limits[0].lower, - s8_stats.limits[0].upper - best_fit_params["S_8"], - best_fit_params["SIGMA_8"], - best_fit_params["SIGMA_8"] - sigma8_stats.limits[0].lower, - sigma8_stats.limits[0].upper - best_fit_params["SIGMA_8"], - best_fit_params["OMEGA_M"], - best_fit_params["OMEGA_M"] - omegam_stats.limits[0].lower, - omegam_stats.limits[0].upper - best_fit_params["OMEGA_M"], - ], - ) - ) -print(param_values) -np.savetxt( - f"{root_dir}/param_values.txt", - param_values, - fmt=["%s" for i in range(11)], - delimiter=";", -) - - -# Load the value of the parameters -cosmo = np.loadtxt( - f"{root_dir}/param_values.txt", - dtype={ - "names": ( - "Expt", - "colour", - "s8_mean", - "s8_low", - "s8_high", - "sigma8_mean", - "sigma8_low", - "sigma8_high", - "omegam_mean", - "omegam_low", - "omegam_high", - ), - "formats": ( - "U250", - "U20", - "U20", - "U20", - "U20", - "U20", - "U20", - "U20", - "U20", - "U20", - "U20", - ), - }, - skiprows=1, - delimiter=";", -) -expt = np.char.replace(cosmo["Expt"], "\\\\", "\\") -colours = cosmo["colour"] -s8_mean = cosmo["s8_mean"].astype(np.float64) -s8_low = cosmo["s8_low"].astype(np.float64) -s8_high = cosmo["s8_high"].astype(np.float64) -sigma8_mean = cosmo["sigma8_mean"].astype(np.float64) -sigma8_low = cosmo["sigma8_low"].astype(np.float64) -sigma8_high = cosmo["sigma8_high"].astype(np.float64) -omegam_mean = cosmo["omegam_mean"].astype(np.float64) -omegam_low = cosmo["omegam_low"].astype(np.float64) -omegam_high = cosmo["omegam_high"].astype(np.float64) - - -from matplotlib.gridspec import GridSpec - -fig = plt.figure(figsize=(13, 8)) -gs = GridSpec(1, 3, width_ratios=[1, 0.5, 0.5]) -ax1 = fig.add_subplot(gs[0]) -ax2 = fig.add_subplot(gs[1], sharey=ax1) -ax3 = fig.add_subplot(gs[2], sharey=ax1) - -axs = [ax1, ax2, ax3] - -params = [ - (s8_mean, s8_low, s8_high, r"$S_8$"), - (sigma8_mean, sigma8_low, sigma8_high, r"$\sigma_8$"), - (omegam_mean, omegam_low, omegam_high, r"$\Omega_{\rm m}$"), -] -reference = r"UNIONS-3500 $\xi_{\pm}(\theta)$ (This work)" - -separation_after = [ - r"UNIONS-3500 $C_\ell$ (Guerrini et al. 2026)", - r"HSC-Y3 $C_\ell$", - r"$\xi_+$ small scales, $\theta$=[5,83] arcmin", - r"Unmasked covmat", - r"$\texttt{HMCode}$ no baryons", - r"Nautilus sampler", -] -list_section_index = [r"(ii)", r"(iii)", r"(iv)", r"(v)", r"(vi)", r"(vii)"] - -preliminary_watermark = False -blind_axes = False -row_spacing = 0.2 - -index_ref = np.where(expt == reference)[0][0] - -y = np.arange(len(expt)) -for ax, param in zip(axs, params): - means, lows, highs, label = param - for i, mean, low, high, color in zip(y, means, lows, highs, colours): - ax.errorbar( - mean, - 0.05 + i * row_spacing, - xerr=np.array([low, high])[:, None], - fmt="o", - color=color, - ecolor=color, - elinewidth=2, - capsize=3, - ) - ax.set_xlabel(label, fontsize=14) - - ax.grid(False) - ax.tick_params(axis="y", left=False, labelleft=False) - if label == r"$S_8$": - ax.axvspan( - s8_mean[index_ref] - s8_low[index_ref], - s8_mean[index_ref] + s8_high[index_ref], - color=colours[index_ref], - alpha=0.2, - ) - ax.set_xlim(0.6, 1.35) - if blind_axes: - ref_tick = np.mean(s8_mean[:4]) - ax.set_xticks([ref_tick + i * 0.1 for i in range(-5, 5)], labels=[]) - elif label == r"$\sigma_8$": - ax.axvspan( - sigma8_mean[index_ref] - sigma8_low[index_ref], - sigma8_mean[index_ref] + sigma8_high[index_ref], - color=colours[index_ref], - alpha=0.2, - ) - ax.set_xlim(0.5, 1.35) - if blind_axes: - ref_tick = np.mean(sigma8_mean[:4]) - ax.set_xticks([ref_tick + i * 0.2 for i in range(-2, 2)], labels=[]) - elif label == r"$\Omega_{\rm m}$": - ax.axvspan( - omegam_mean[index_ref] - omegam_low[index_ref], - omegam_mean[index_ref] + omegam_high[index_ref], - color=colours[index_ref], - alpha=0.2, - ) - ax.set_xlim(0.1, 0.5) - if blind_axes: - ref_tick = np.mean(omegam_mean[:4]) - ax.set_xticks([ref_tick + i * 0.1 for i in range(-2, 3)], labels=[]) - - -ax1.set_yticks(0.01 + y * row_spacing) -ax1.set_yticklabels([]) -for label, color in zip(expt, colours): - if "This work" in label: - label_bold = ( - r"$\bf{UNIONS}$-$\bf{3500}$ $\xi_{\pm}(\theta)$ $\bf{(This\ work)}$" - ) - ax1.text( - -0.6, - 0.05 + row_spacing * np.where(expt == label)[0][0], - label_bold, - fontsize=12, - ha="left", - va="center", - color=color, - ) - else: - ax1.text( - -0.6, - 0.05 + row_spacing * np.where(expt == label)[0][0], - label, - fontsize=12, - ha="left", - va="center", - color=color, - ) - if label != reference: - index = np.where(expt == label)[0][0] - s8_tension = cp.get_sigma_tension( - s8_mean[index], - s8_low[index], - s8_high[index], - s8_mean[index_ref], - s8_low[index_ref], - s8_high[index_ref], - ) - sign_str = "+" if s8_tension > 0 else "-" - ax1.text( - 1.32, - 0.05 + row_spacing * index, - rf"${sign_str}{np.abs(s8_tension):.2f}" + r"\, \sigma$", - fontsize=10, - ha="right", - va="center", - color=color, - ) -# Add separation lines -for i, sep in enumerate(separation_after): - print(sep) - index_sep = np.where(expt == sep)[0][0] - ax2.axhline( - row_spacing * (index_sep + 1) - 0.07, - color="black", - linestyle="dotted", - linewidth=1, - ) - ax3.axhline( - row_spacing * (index_sep + 1) - 0.07, - color="black", - linestyle="dotted", - linewidth=1, - ) - ax1.axhline( - row_spacing * (index_sep + 1) - 0.07, - xmin=-1.8, - color="black", - linestyle="dotted", - linewidth=1, - clip_on=False, - ) - ax1.text( - -0.61, - row_spacing * (index_sep + 1) + 0.05, - list_section_index[i], - fontsize=12, - fontweight="bold", - va="center", - ha="right", - ) - - -# --- Add section label (i)) --- -ax1.text(-0.61, 0.05, r"(i)", fontsize=12, fontweight="bold", va="center", ha="right") - -if preliminary_watermark: - plt.figtext( - 0.5, - 0.5, - "PRELIMINARY", - fontsize=50, - color="gray", - ha="center", - va="center", - alpha=0.3, - rotation=330, - ) - -plt.gca().invert_yaxis() - -plt.tight_layout() - -# #Save pdf -plt.savefig("./../../results/S8_whisker_plot.pdf", bbox_inches="tight") diff --git a/papers/realspace/best_fit_xipm.py b/papers/realspace/best_fit_xipm.py deleted file mode 100644 index ebd66f27..00000000 --- a/papers/realspace/best_fit_xipm.py +++ /dev/null @@ -1,497 +0,0 @@ -import os -import sys - -sys.path.append("/home/guerrini/sp_validation/cosmo_inference/scripts") - -import chain_postprocessing as cp -import matplotlib.pyplot as plt -import matplotlib.scale as mscale -import numpy as np -import seaborn as sns -from astropy.io import fits -from getdist import plots - -plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") - -from sp_validation.rho_tau import SquareRootScale - -mscale.register_scale(SquareRootScale) - -plt.rcParams["text.usetex"] = True - -sns.set_palette("husl") - -g = plots.get_subplot_plotter(width_inch=30) -g.settings.axes_fontsize = 40 -g.settings.axes_labelsize = 40 -g.settings.alpha_filled_add = 0.7 -g.settings.legend_fontsize = 50 - -# Directory where the chains are located -root_dir = "/n09data/guerrini/output_chains" - -# THE BLIND TO USE FOR THE PLOTS -blind = "B" -catalog_version = "SP_v1.4.6.3" -fiducial_root_cell = f"SP_v1.4.6.3_leak_corr_{blind}" -label_fiducial_cell = r"UNIONS $C_{\ell}$" -fiducial_root_xi_data = f"SP_v1.4.6.3_leak_corr_{blind}_masked" -fiducial_root_xi_chains = f"SP_v1.4.6.3_{blind}_fiducial_config" -label_fiducial_xi = r"UNIONS $\xi_{\pm}$" - -# Path to the ini files used -path_ini_files = "/home/guerrini/sp_validation/cosmo_inference/cosmosis_config" -path_datavectors = "/home/guerrini/sp_validation/cosmo_inference/data/" -path_output_chains = "/n09data/guerrini/output_chains/" - - -data_cell = fits.open( - os.path.join( - path_datavectors, f"{fiducial_root_cell}/cosmosis_{fiducial_root_cell}.fits" - ) -) - -data_xi = fits.open( - os.path.join( - path_datavectors, - f"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}/cosmosis_{fiducial_root_xi_data}.fits", - ) -) - -path_samples_fiducial_cell = os.path.join( - path_output_chains, - fiducial_root_cell, - fiducial_root_cell, - f"samples_{fiducial_root_cell}_cell.txt", -) -path_gd_fiducial_cell = os.path.join( - path_output_chains, - fiducial_root_cell, - fiducial_root_cell, - f"getdist_{fiducial_root_cell}_cell", -) -cp.load_samples_and_write_paramnames( - path_samples_fiducial_cell, path_gd_fiducial_cell + ".paramnames" -) -cp.write_samples_getdist_format( - path_samples_fiducial_cell, path_gd_fiducial_cell + ".txt", chain_type="polychord" -) - -chain_fiducial_cell = cp.load_chain(path_gd_fiducial_cell, smoothing_scale=0.5) - -best_fit_params_fiducial_cell = cp.extract_best_fit_params( - chain_fiducial_cell, best_fit_method="2Dkde" -) - -cp.compute_best_fit( - path_ini_files, - best_fit_params_fiducial_cell, - fiducial_root_cell, - is_harmonic=True, - blind=blind, -) -path_samples_fiducial_xi = os.path.join( - path_output_chains, - fiducial_root_xi_chains, - f"samples_{fiducial_root_xi_chains}.txt", -) - -path_gd_fiducial_xi = os.path.join( - path_output_chains, fiducial_root_xi_chains, f"getdist_{fiducial_root_xi_chains}" -) -cp.load_samples_and_write_paramnames( - path_samples_fiducial_xi, path_gd_fiducial_xi + ".paramnames" -) -cp.write_samples_getdist_format( - path_samples_fiducial_xi, path_gd_fiducial_xi + ".txt", chain_type="polychord" -) - -chain_fiducial_xi = cp.load_chain(path_gd_fiducial_xi, smoothing_scale=0.5) - -best_fit_params_fiducial_xi = cp.extract_best_fit_params( - chain_fiducial_xi, best_fit_method="2Dkde" -) - -ini_file_root = os.path.join( - path_ini_files, - f"config_space_v1.4.6.3_fiducial/pipeline/blind_{blind}/fiducial.ini", -) -cp.compute_best_fit( - path_ini_files, - best_fit_params_fiducial_xi, - fiducial_root_xi_chains, - is_harmonic=False, - blind=blind, - ini_file_root=ini_file_root, -) - -root_to_plot = [ - fiducial_root_xi_chains, - fiducial_root_cell, -] - -labels = [ - r"UNIONS $\xi_\pm(\theta)$", - r"UNIONS $C_\ell$", -] - -line_args = [ - {"color": "royalblue", "linestyle": "-"}, - {"color": "orange", "linestyle": "-"}, -] - -properties = {} - -properties = cp.update_properties_w_roots( - properties, fiducial_root_cell, path_ini_files, with_configuration=False -) -properties = cp.update_properties_w_roots( - properties, - fiducial_root_xi_chains, - path_ini_files, - with_configuration=True, - path_to_this_ini=ini_file_root, -) - - -root_to_plot = [fiducial_root_cell, fiducial_root_xi_chains] -labels = [r"Best fit $C_\ell$", r"Best fit $\xi_\pm(\theta)$"] -path_best_fit_xi_theta = os.path.join( - path_output_chains, fiducial_root_xi_chains, "best_fit/shear_xi_plus/theta.txt" -) - -theta_rad = np.loadtxt(path_best_fit_xi_theta) -theta_min = 1 -theta_max = 250 - -cp.compute_best_fit_xi_from_cell( - path_output_chains, fiducial_root_cell, best_fit_params_fiducial_cell, theta_rad -) - -data = fits.open( - os.path.join( - path_datavectors, - f"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}/cosmosis_{fiducial_root_xi_data}.fits", - ) -) -bbox_to_anchor_xip = (0.685, 0.09) -bbox_to_anchor_xim = (0.3, 0.65) -xi_p_data = data["XI_PLUS"].data -xi_m_data = data["XI_MINUS"].data -cov_mat = data["COVMAT"].data - -# Plot hyperparameter -loc_legend = "lower center" - -fig, [ax, ax2] = plt.subplots(1, 2, figsize=(20, 8)) - -theta, xi_p, xi_m = xi_p_data["ANG"], xi_p_data["VALUE"], xi_m_data["VALUE"] -ax.errorbar( - theta, - theta * xi_p, - yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])), - fmt="o", - label=r"UNIONS $\xi_+$ data", - color="black", - capsize=2, -) -ax2.errorbar( - theta, - theta * xi_m, - yerr=theta - * np.sqrt( - np.diag(cov_mat[len(theta) : 2 * len(theta), len(theta) : 2 * len(theta)]) - ), - fmt="o", - label=r"UNIONS $\xi_-$ data", - color="black", - capsize=2, -) - -for idx, (label, root) in enumerate(zip(labels, root_to_plot)): - # Read the results - theta = ( - ( - np.loadtxt( - path_output_chains + "{}/best_fit/shear_xi_plus/theta.txt".format(root) - ) - ) - * 180 - / np.pi - * 60 - ) - xi_plus = np.loadtxt( - path_output_chains + "{}/best_fit/shear_xi_plus/bin_1_1.txt".format(root) - ) - xi_minus = np.loadtxt( - path_output_chains + "{}/best_fit/shear_xi_minus/bin_1_1.txt".format(root) - ) - if r"$C_\ell$" not in label: - xi_sys_plus = np.loadtxt( - path_output_chains + "{}/best_fit/xi_sys/shear_xi_plus.txt".format(root) - ) - xi_sys_minus = np.loadtxt( - path_output_chains + "{}/best_fit/xi_sys/shear_xi_minus.txt".format(root) - ) - theta_xi_sys = ( - np.loadtxt(path_output_chains + "{}/best_fit/xi_sys/theta.txt".format(root)) - * 180 - / np.pi - * 60 - ) - - xi_sys_plus = np.interp(theta, theta_xi_sys, xi_sys_plus) - xi_sys_minus = np.interp(theta, theta_xi_sys, xi_sys_minus) - xi_plus += xi_sys_plus - xi_minus += xi_sys_minus - - mask = (theta > theta_min) & (theta < theta_max) - theta = theta[mask] - ax.plot( - theta, - theta * xi_plus[mask], - label=r"Best fit $\xi_+(\theta)$", - **line_args[idx], - lw=2.5, - ) - ax.plot( - theta, - theta * xi_sys_plus[mask], - label=r"Best fit $\xi^{\rm sys}_{+}(\theta)$", - c="r", - ) - ax2.plot( - theta, - theta * xi_minus[mask], - label=r"Best fit $\xi_-(\theta)$", - **line_args[idx], - lw=2.5, - ) - ax2.plot( - theta, - theta * xi_sys_minus[mask], - label=r"Best fit $\xi^{\rm sys}_{-}(\theta)$", - c="r", - ) - - else: - mask = (theta > theta_min) & (theta < theta_max) - theta = theta[mask] - ax.plot(theta, theta * xi_plus[mask], label=label, **line_args[idx], lw=2.5) - ax2.plot(theta, theta * xi_minus[mask], label=label, **line_args[idx], lw=2.5) - -# XI PLUS PLOT SETTINGS - -# Plot the scale cuts for different k_max -ax.axvline(x=5, color="gray", linestyle="--", alpha=0.7) -ax.axhline(y=0, color="black", linestyle="--", alpha=0.7) - -ymin = ax.get_ylim()[0] -ymax = ax.get_ylim()[1] -# Shadowing cut scaled -ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color="gray", alpha=0.2) -ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color="gray", alpha=0.2) - -ax.set_ylim(ymin, ymax) - -# Add labels directly under the tick -ax.text( - 4.5, - 0.47e-4, - r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", - ha="center", - va="top", - fontsize=20, - rotation=90, -) - -ax.set_ylabel(r"$\theta \xi_\pm$", fontsize=26) -ax.set_xlabel(r"$\theta$ (arcmin)", fontsize=26) -ax.set_xlim([theta.min() - 0.1, theta.max() + 20]) -ax.set_title(r"$\xi_+(\theta)$", fontsize=26) -ax.set_xscale("log") -ax.set_xticks(np.array([1, 10, 100])) -ax.tick_params(axis="x", which="minor", length=2, width=0.8) -ax.tick_params(axis="both", which="major", labelsize=24) -ax.tick_params(axis="both", which="minor", labelsize=20) -ax.yaxis.get_offset_text().set_fontsize(24) -ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) -ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=20) - -# XI_MINUS PLOT SETTINGS - -# Plot the scale cuts for different k_max -ax2.axvline(x=50, color="gray", linestyle="--", alpha=0.7) -ax2.axhline(y=0, color="black", linestyle="--", alpha=0.7) - -ymin = ax2.get_ylim()[0] -ymax = ax2.get_ylim()[1] -# Shadowing cut scaled -ax2.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color="gray", alpha=0.2) -ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color="gray", alpha=0.2) - -ax2.set_ylim(ymin, ymax) - -# Add labels directly under the tick -ax2.text( - 45, - 1.15e-4, - r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", - ha="center", - va="top", - fontsize=20, - rotation=90, -) - -# ax2.set_ylabel(r'$\theta \xi_-$', fontsize=16) -ax2.set_xlabel(r"$\theta$ (arcmin)", fontsize=26) -ax2.set_xlim([theta.min() - 0.1, theta.max() + 20]) -ax2.set_xscale("log") -ax2.set_title(r"$\xi_-(\theta)$", fontsize=26) -ax2.set_xticks(np.array([1, 10, 100])) -ax2.tick_params(axis="x", which="minor", length=2, width=0.8) -ax2.tick_params(axis="both", which="major", labelsize=24) -ax2.tick_params(axis="both", which="minor", labelsize=20) -ax2.yaxis.get_offset_text().set_fontsize(24) -ax2.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) -ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=20) - -plt.savefig("./../../results/best_fit_xipm_SP_v1.4.6.3_B.pdf", bbox_inches="tight") - - -root_to_plot = [fiducial_root_xi_chains] -labels = [r"Best fit $\tau_{0,2}(\theta)$"] - -bbox_to_anchor_xip = (0.285, 0.7) -bbox_to_anchor_xim = (0.3, 0.65) -tau0_data = data["TAU_0_PLUS"].data -tau2_data = data["TAU_2_PLUS"].data -cov_mat = data["COVMAT"].data - -# Plot hyperparameter - -fig, [ax, ax2] = plt.subplots(1, 2, figsize=(20, 8)) - -theta, tau0, tau2 = tau0_data["ANG"], tau0_data["VALUE"], tau2_data["VALUE"] -ax.errorbar( - theta, - theta * tau0, - yerr=theta - * np.sqrt( - np.diag( - cov_mat[2 * len(theta) : 3 * len(theta), 2 * len(theta) : 3 * len(theta)] - ) - ), - fmt="o", - label=r"UNIONS $\tau_{0,+}$", - color="black", - capsize=2, -) -ax2.errorbar( - theta, - theta * tau2, - yerr=theta - * np.sqrt( - np.diag( - cov_mat[3 * len(theta) : 4 * len(theta), 3 * len(theta) : 4 * len(theta)] - ) - ), - fmt="o", - label=r"UNIONS $\tau_{2,+}$", - color="black", - capsize=2, -) - -for idx, (label, root) in enumerate(zip(labels, root_to_plot)): - # Read the results - theta = ( - ( - np.loadtxt( - path_output_chains + "{}/best_fit/tau_0_plus/theta.txt".format(root) - ) - ) - * 180 - / np.pi - * 60 - ) - tau0_plus = np.loadtxt( - path_output_chains + "{}/best_fit/tau_0_plus/bin_1_1.txt".format(root) - ) - tau2_plus = np.loadtxt( - path_output_chains + "{}/best_fit/tau_2_plus/bin_1_1.txt".format(root) - ) - - mask = (theta > theta_min) & (theta < theta_max) - theta = theta[mask] - ax.plot( - theta, - theta * tau0_plus[mask], - label=r"Best fit $\tau_{0,+}(\theta)$", - c="orange", - lw=2.5, - ) - ax2.plot( - theta, - theta * tau2_plus[mask], - label=r"Best fit $\tau_{2,+}(\theta)$", - c="orange", - lw=2.5, - ) - -# XI PLUS PLOT SETTINGS - -# Plot the scale cuts for different k_max -ax.axhline(y=0, color="black", linestyle="--", alpha=0.7) - -ymin = ax.get_ylim()[0] -ymax = ax.get_ylim()[1] - -ax.set_ylim(ymin, ymax) - -ax.set_ylabel(r"$\theta\tau_{0,2}$", fontsize=26) -ax.set_xlabel(r"$\theta$ (arcmin)", fontsize=26) -ax.set_xlim([theta.min() - 0.1, theta.max() + 20]) -ax.set_title(r"$\tau_{0,+}(\theta)$", fontsize=26) -ax.set_xscale("log") -ax.set_xticks(np.array([1, 10, 100])) -ax.tick_params(axis="x", which="minor", length=2, width=0.8) -ax.tick_params(axis="both", which="major", labelsize=24) -ax.tick_params(axis="both", which="minor", labelsize=20) -ax.yaxis.get_offset_text().set_fontsize(24) -ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) -ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=20) - -# XI_MINUS PLOT SETTINGS - -# Plot the scale cuts for different k_max -ax2.axhline(y=0, color="black", linestyle="--", alpha=0.7) - -ymin = ax2.get_ylim()[0] -ymax = ax2.get_ylim()[1] -# Shadowing cut scaled -ax2.fill_betweenx( - y=[ymin, ymax], - x1=0, - x2=12, - color="gray", - alpha=0.2, - label=r"$B$-mode informed scale cut", -) -ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color="gray", alpha=0.2) - -ax2.set_ylim(ymin, ymax) - -# ax2.set_ylabel(r'$\theta \xi_-$', fontsize=16) -ax2.set_xlabel(r"$\theta$ (arcmin)", fontsize=26) -ax2.set_xlim([theta.min() - 0.1, theta.max() + 20]) -ax2.set_xscale("log") -ax2.set_title(r"$\tau_{2,+}(\theta)$", fontsize=26) -ax2.set_xticks(np.array([1, 10, 100])) -ax2.tick_params(axis="x", which="minor", length=2, width=0.8) -ax2.tick_params(axis="both", which="major", labelsize=24) -ax2.tick_params(axis="both", which="minor", labelsize=20) -ax2.yaxis.get_offset_text().set_fontsize(24) -ax2.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) -ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=20) - -plt.savefig("./../../results/best_fit_tau_02_SP_v1.4.6.3_B.pdf", bbox_inches="tight") diff --git a/papers/realspace/contours.py b/papers/realspace/contours.py deleted file mode 100644 index 78b988f4..00000000 --- a/papers/realspace/contours.py +++ /dev/null @@ -1,745 +0,0 @@ -# # 2D contour plots -# -# This notebook produces the plots for all the 2D contours in the results section. - - -import os.path - -import matplotlib.pyplot as plt -import numpy as np -import seaborn as sns -from astropy.io import fits -from getdist import plots - -plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") - -plt.rcParams["text.usetex"] = True - -sns.set_palette("husl") -g = plots.get_subplot_plotter(width_inch=30) -g.settings.axes_fontsize = 70 -g.settings.axes_labelsize = 80 -g.settings.alpha_filled_add = 0.7 -g.settings.legend_fontsize = 70 - - -# SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE - -root_dir = "/n09data/guerrini/output_chains/" -path_datavectors = "/home/guerrini/sp_validation/cosmo_inference/data/" -path_output_chains = "/n09data/guerrini/output_chains/" - -data = fits.open( - os.path.join( - path_datavectors, - "SP_v1.4.6.3_config/SP_v1.4.6.3_B/cosmosis_SP_v1.4.6.3_leak_corr_B_masked.fits", - ) -) - -roots_fid = { - "SP_v1.4.6.3_leak_corr_B": r"UNIONS-3500 $C_\ell$", - "SP_v1.4.6.3_B_fiducial_config": r"UNIONS-3500 $\xi_\pm$ (This work) ", - "KiDS-Legacy_xipm": r"KiDS-Legacy $\xi_\pm$", - "HSC_Y3": r"HSC-Y3 $\xi_\pm$", - "Planck18": r"$\textit{Planck}$ 2018", -} - -roots_full = { - "SP_v1.4.6.3_B_fiducial_config": r"UNIONS-3500 $\xi_\pm$ (This work) ", -} - -roots_ia = { - "SP_v1.4.6.3_B_fiducial_config": r"Gaussian $A_{\rm{IA}}$ prior", - "SP_v1.4.6.3_B_flat_ia_config": r"Flat $A_{\rm{IA}}$ prior", - "SP_v1.4.6.3_B_no_ia_config": r"No IA", -} - -roots_ext = { - "SP_v1.4.6.3_B_fiducial_config": r"UNIONS-3500 $\xi_\pm$", - "SP_v1.4.6.3_B_planck_config": r"UNIONS-3500 $\xi_\pm$ + CMB", - "SP_v1.4.6.3_B_planck_desi_config": r"UNIONS-3500 $\xi_\pm$ + CMB + BAO", - "Planck18": r"$\textit{Planck}$ 2018", -} - -roots_dz = { - "SP_v1.4.6.3_B_fiducial_config": r"Gaussian $\Delta z$ prior", - "SP_v1.4.6.3_B_flat_delta_z_config": r"Flat $\Delta z$ prior", - "SP_v1.4.6.3_B_no_delta_z_config": r"No $\Delta z$ modelling", -} - -roots_psf = { - "SP_v1.4.6.3_B_flat_alpha_beta_config": r"Flat $\alpha$ and $\beta$ priors", - "SP_v1.4.6.3_B_fiducial_config": r"Gaussian $\alpha$ and $\beta$ priors", - "SP_v1.4.6.3_B_no_xi_sys_config": r"No $\xi^{\rm sys}$ included", - "SP_v1.4.6.3_B_no_leak_corr_config": r"No object-wise leakage correction", -} - -roots_scale = { - "SP_v1.4.6.3_B_fiducial_config": r"$\xi_+$: $\theta=[12,83]$", - "SP_v1.4.6.3_B_small_scales_config": r"$\xi_+$: $\theta=[5,83]$", -} - -roots_nonlin = { - "SP_v1.4.6.3_B_fiducial_config": r"Fiducial (\texttt{HMCode2020}, $\log(T_{\rm AGN})$)", - "SP_v1.4.6.3_B_no_baryons_config": r"\texttt{HMCode2020} no baryons", - "SP_v1.4.6.3_B_halofit_config": r"\texttt{Halofit}", -} -roots = roots_ext - - -# ## Retrieve the chains - - -# READ CHAIN - -chains = [] - -for i, root in enumerate(list(roots.keys())): - burnin = 0 - if "SP" not in root: - chain = g.samples_for_root( - root_dir + "ext_data/{}/getdist_{}".format(root, root), - cache=False, - settings={ - "ignore_rows": burnin, - # 'smooth_scale_2D':0.2, - # 'smooth_scale_1D':0.2 - }, - ) - p = chain.getParams() - if hasattr(p, "S_8") == False: - omega_m = chain.getParams().OMEGA_M - sigma_8 = chain.getParams().SIGMA_8 - - s_8 = sigma_8 * (omega_m / 0.3) ** 0.5 - - chain.addDerived(s_8, name="S_8", label=r"S_8") - - p = chain.paramNames.parWithName("S_8") - - elif "config" in root: - if os.path.isfile(root_dir + "{}/getdist_{}.txt".format(root, root)) == False: - samples = np.loadtxt(root_dir + "{}/samples_{}.txt".format(root, root)) - - if "nautilus" in root: - weights = np.exp(samples[:, -3]) - neglogL = samples[:, -2] - samples[:, -1] - - samples = np.column_stack((weights, neglogL, samples[:, 0:-3])) - elif "mh" in root: - samples = np.column_stack( - ( - np.ones_like(samples[:, -1]), - np.log(samples[:, -1]) - np.log(samples[:, -2]), - samples[:, 0:-2], - ) - ) - burnin = 0.3 - else: - samples = np.column_stack( - (samples[:, -1], samples[:, -3], samples[:, 0:-4]) - ) - - np.savetxt(root_dir + "{}/getdist_{}.txt".format(root, root), samples) - - chain = g.samples_for_root( - root_dir + "{}/getdist_{}".format(root, root), - cache=False, - settings={ - "ignore_rows": burnin, - # 'smooth_scale_2D':0.2, - # 'smooth_scale_1D':0.2 - }, - ) - else: - if ( - os.path.isfile( - root_dir + "{}/{}/getdist_{}_cell.txt".format(root, root, root) - ) - == False - ): - samples = np.loadtxt( - root_dir + "{}/{}/samples_{}_cell.txt".format(root, root, root) - ) - - if "nautilus" in root: - weights = np.exp(samples[:, -3]) - neglogL = samples[:, -2] - samples[:, -1] - - samples = np.column_stack((weights, neglogL, samples[:, 0:-3])) - elif "mh" in root: - samples = np.column_stack( - ( - np.ones_like(samples[:, -1]), - np.log(samples[:, -1]) - np.log(samples[:, -2]), - samples[:, 0:-2], - ) - ) - burnin = 0.3 - else: - samples = np.column_stack( - (samples[:, -1], samples[:, -3], samples[:, 0:-4]) - ) - - np.savetxt( - root_dir + "{}/{}/getdist_{}_cell.txt".format(root, root, root), samples - ) - - chain = g.samples_for_root( - root_dir + "{}/{}/getdist_{}_cell".format(root, root, root), - cache=False, - settings={ - "ignore_rows": burnin, - # 'smooth_scale_2D':0.2, - # 'smooth_scale_1D':0.2 - }, - ) - p = chain.getParams() - - chains.append(chain) - - -name_list = [ - "OMEGA_M", - "ombh2", - "h0", - "n_s", - "SIGMA_8", - "S_8", - "logt_agn", - "a", - "m1", - "bias_1", - "alpha", - "beta", - "omch2", -] -label_list = [ - r"\Omega_{\rm m}", - r"\omega_{\rm b}", - r"h", - r"n_{\rm s}", - r"\sigma_8", - r"S_8", - r"\log T_{\rm AGN}", - r"A_{\rm IA}", - r"m_1", - r"\Delta z", - r"\alpha_{\rm PSF}", - r"\beta_{\rm PSF}", - r"\omega_{\rm c}", -] - -for chain in chains: - param_names = chain.getParamNames() - p = chain.getParams() - for name, label in zip(name_list, label_list): - if hasattr(p, name): - param_names.parWithName(name).label = label - -legend_labels = list(roots.values()) - - -# ## Plot the chains - - -# ### FIDUCIAL PLOT - - -colours = [ - "royalblue", - "orange", - "crimson", - "forestgreen", - "indigo", -] - -linestyle = ["solid", "solid", "solid", "solid", "solid"] - -line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] - -# FIDUCIAL PLOT -g.triangle_plot( - chains, - ["SIGMA_8", "S_8", "OMEGA_M"], # - legend_labels=legend_labels, - line_args=line_args, - contour_colors=colours, - label_order=[1, 0, 2, 3, 4], - filled=[True, True, False, False, True], -) - -g.export("./../../results/SP_v1.4.6.3_B_fiducial_config_contour_plot.pdf") - - -# ### FULL PLOT - - -g.settings.axes_fontsize = 40 -g.settings.axes_labelsize = 50 - -colours = [ - "orange", -] - -linestyle = [ - "solid", -] - -line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] - -# FIDUCIAL PLOT -g.triangle_plot( - chains, - [ - "OMEGA_M", - "ombh2", - "h0", - "n_s", - "SIGMA_8", - "S_8", - "logt_agn", - "a", - "m1", - "bias_1", - ], - legend_labels=legend_labels, - line_args=line_args, - contour_colors=colours, - filled=True, -) - -g.export("./../../results/SP_v1.4.6.3_B_fiducial_config_contour_plot_full.pdf") - - -# ### IA PLOT - - -colours = [ - "orange", - "royalblue", - "forestgreen", -] - -linestyle = [ - "solid", - "solid", - "solid", -] - -line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] - -g.triangle_plot( - chains, - ["S_8", "OMEGA_M", "a"], # - legend_labels=legend_labels, - line_args=line_args, - contour_args={"alpha": 0.6}, - contour_colors=colours, - filled=[True, False, True], -) - -g.export("./../../results/SP_v1.4.6.3_B_fiducial_config_contour_plot_ia.pdf") - - -# ### PSF PLOT - - -colours = [ - "royalblue", - "orange", - "hotpink", - "slategray", -] - -linestyle = [ - "solid", - "solid", - "solid", - "solid", -] - -line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] - -g.triangle_plot( - chains, - ["S_8", "OMEGA_M", "alpha", "beta"], # - legend_labels=legend_labels, - line_args=line_args, - contour_args=[{"alpha": 1}, {"alpha": 0.6}, {"alpha": 0.8}, {"alpha": 0.8}], - contour_colors=colours, - legend_loc="upper right", - label_order=[1, 0, 2, 3], - filled=[False, True, True, True], -) - -g.subplots[3, 2].scatter( - 0.005, 0.81, color="k", marker="X", s=400, label="Fiducial config best-fit" -) -g.subplots[3, 2].scatter( - 0.022, 0.798, color="k", marker="P", s=400, label="Fiducial config best-fit" -) - -g.export("./../../results/SP_v1.4.6.3_B_fiducial_config_contour_plot_psf.pdf") - - -# ### DELTA Z PLOT - - -colours = [ - "orange", - "royalblue", - "indigo", -] - -linestyle = [ - "solid", - "solid", - "solid", -] - -line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] -g.triangle_plot( - chains, - ["S_8", "OMEGA_M", "bias_1"], # - legend_labels=legend_labels, - line_args=line_args, - contour_args=[{"alpha": 1.0}, {"alpha": 0.9}, {"alpha": 0.5}], - contour_colors=colours, - filled=[True, False, True], -) - -g.export("./../../results/SP_v1.4.6.3_B_fiducial_config_contour_plot_dz.pdf") - - -# ### EXTERNAL DATA - - -colours = [ - "orange", - "royalblue", - "crimson", - "forestgreen", -] - -linestyle = [ - "solid", - "solid", - "solid", - "solid", - "solid", -] - -line_args = [dict(color=col, ls=ls) for col, ls in zip(colours, linestyle)] - -g = plots.get_subplot_plotter(width_inch=10) -g.settings.axes_fontsize = 25 -g.settings.axes_labelsize = 25 -g.settings.legend_fontsize = 22 - -g.plot_2d( - chains, - ["S_8", "OMEGA_M", "SIGMA_8"], # - line_args=line_args, - contour_colors=colours, - legend_labels=legend_labels, - alphas=[0.7, 1.0, 1.0, 1.0], - filled=[True, True, True, False], -) - -g.add_y_bands(0.2975, 0.0086, alpha2=0, color="k", label="BAO") -g.add_legend(legend_labels, legend_loc="upper right") - -g.export("./../../results/SP_v1.4.6.3_B_fiducial_config_contour_plot_ext.pdf") - - -# ### Small scales - - -colours = [ - "orange", - "dodgerblue", -] - -linestyle = [ - "solid", - "solid", -] - -line_args = [dict(color=col, ls=ls) for col, ls in zip(colours, linestyle)] - -g = plots.get_subplot_plotter(width_inch=9) -g.settings.axes_fontsize = 25 -g.settings.axes_labelsize = 25 -g.settings.alpha_filled_add = 0.7 -g.settings.legend_fontsize = 30 - -g.plot_2d( - chains, - ["S_8", "OMEGA_M"], # - line_args=line_args, - contour_args=[{"alpha": 0.7}, {"alpha": 1.0}], - contour_colors=colours, - filled=[True, True], -) -g.add_legend(legend_labels, legend_loc="upper right") - -g.export("./../../results/SP_v1.4.6.3_B_fiducial_config_contour_plot_scales.pdf") - - -# ### BBN Prior - - -from getdist.gaussian_mixtures import Gaussian1D - -colours = [ - "orange", - "royalblue", -] - -linestyle = [ - "solid", - "solid", -] - -line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] - -# BBN PRIOR -bbn_prior = Gaussian1D( - mean=0.02218, - sigma=0.00055, - name="ombh2", - labels=[r"\omega_{\rm b}"], - label="BBN prior", -) -bbn_chain = bbn_prior.MCSamples(3000, label="BBN prior") - -g.triangle_plot( - chains + [bbn_chain], - name_list, - legend_labels=legend_labels, - line_args=line_args, - contour_colors=colours, - filled=[True, False], -) - - -# ## Plot the best-fit $\xi_\pm$ - - -xi_p_data = data["XI_PLUS"].data -xi_m_data = data["XI_MINUS"].data -cov_mat = data["COVMAT"].data - -labels = roots_scale.values() - -bbox_to_anchor_xip = (0.685, 0.09) -bbox_to_anchor_xim = (0.3, 0.65) -theta_min = 1.0 -theta_max = 250.0 -loc_legend = "lower center" - - -colours = [ - "orange", - "dodgerblue", -] - -linestyle = [ - "solid", - "solid", -] - -line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] - -labels = roots_scale.values() - -fig, ax = plt.subplots(1, 1, figsize=(11, 7)) - -theta, xi_p, xi_m = xi_p_data["ANG"], xi_p_data["VALUE"], xi_m_data["VALUE"] -ax.errorbar( - theta, - theta * xi_p, - yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])), - fmt="o", - color="black", - capsize=2, -) - -for idx, (label, root) in enumerate(zip(labels, roots_scale)): - # Read the results - theta = ( - ( - np.loadtxt( - path_output_chains + "{}/best_fit/shear_xi_plus/theta.txt".format(root) - ) - ) - * 180 - / np.pi - * 60 - ) - xi_plus = np.loadtxt( - path_output_chains + "{}/best_fit/shear_xi_plus/bin_1_1.txt".format(root) - ) - xi_minus = np.loadtxt( - path_output_chains + "{}/best_fit/shear_xi_minus/bin_1_1.txt".format(root) - ) - xi_sys_plus = np.loadtxt( - path_output_chains + "{}/best_fit/xi_sys/shear_xi_plus.txt".format(root) - ) - xi_sys_minus = np.loadtxt( - path_output_chains + "{}/best_fit/xi_sys/shear_xi_minus.txt".format(root) - ) - theta_xi_sys = ( - np.loadtxt(path_output_chains + "{}/best_fit/xi_sys/theta.txt".format(root)) - * 180 - / np.pi - * 60 - ) - - xi_sys_plus = np.interp(theta, theta_xi_sys, xi_sys_plus) - xi_sys_minus = np.interp(theta, theta_xi_sys, xi_sys_minus) - xi_plus += xi_sys_plus - xi_minus += xi_sys_minus - - mask = (theta > theta_min) & (theta < theta_max) - theta = theta[mask] - ax.plot(theta, theta * xi_plus[mask], label=label, **line_args[idx]) - -ymin = ax.get_ylim()[0] -ymax = ax.get_ylim()[1] - -ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color="gray", alpha=0.2) -ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=5, color="gray", alpha=0.7) -ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=300, color="gray", alpha=0.2) - -ax.set_ylim(ymin, ymax) - -ax.set_ylabel(r"$\theta \xi_\pm$", fontsize=26) -ax.set_xlabel(r"$\theta$ (arcmin)", fontsize=26) -ax.set_xlim([theta.min() - 0.1, theta.max() + 20]) -ax.set_title(r"$\xi_+(\theta)$", fontsize=26) -ax.set_xscale("log") -ax.set_xticks(np.array([1, 10, 100])) -ax.tick_params(axis="x", which="minor", length=2, width=0.8) -ax.tick_params(axis="both", which="major", labelsize=24) -ax.tick_params(axis="both", which="minor", labelsize=20) -ax.yaxis.get_offset_text().set_fontsize(24) -ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) -ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=20) - - -plt.savefig("./../../results/scale_cut_xipm_SP_v1.4.6.3_B.pdf", bbox_inches="tight") - - -labels = roots_nonlin.values() - -colours = ["orange", "hotpink", "teal"] - -linestyle = ["solid", "solid", "dashed"] - -line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] - -fig, [ax, ax2] = plt.subplots(2, 1, figsize=(11, 14)) - -theta, xi_p, xi_m = xi_p_data["ANG"], xi_p_data["VALUE"], xi_m_data["VALUE"] -ax.errorbar( - theta, - theta * xi_p, - yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])), - fmt="o", - color="black", - capsize=2, -) -ax2.errorbar( - theta, - theta * xi_m, - yerr=theta - * np.sqrt( - np.diag(cov_mat[len(theta) : 2 * len(theta), len(theta) : 2 * len(theta)]) - ), - fmt="o", - color="black", - capsize=2, -) - -for idx, (label, root) in enumerate(zip(labels, roots_nonlin)): - # Read the results - theta = ( - ( - np.loadtxt( - path_output_chains + "{}/best_fit/shear_xi_plus/theta.txt".format(root) - ) - ) - * 180 - / np.pi - * 60 - ) - xi_plus = np.loadtxt( - path_output_chains + "{}/best_fit/shear_xi_plus/bin_1_1.txt".format(root) - ) - xi_minus = np.loadtxt( - path_output_chains + "{}/best_fit/shear_xi_minus/bin_1_1.txt".format(root) - ) - xi_sys_plus = np.loadtxt( - path_output_chains + "{}/best_fit/xi_sys/shear_xi_plus.txt".format(root) - ) - xi_sys_minus = np.loadtxt( - path_output_chains + "{}/best_fit/xi_sys/shear_xi_minus.txt".format(root) - ) - theta_xi_sys = ( - np.loadtxt(path_output_chains + "{}/best_fit/xi_sys/theta.txt".format(root)) - * 180 - / np.pi - * 60 - ) - - xi_sys_plus = np.interp(theta, theta_xi_sys, xi_sys_plus) - xi_sys_minus = np.interp(theta, theta_xi_sys, xi_sys_minus) - xi_plus += xi_sys_plus - xi_minus += xi_sys_minus - - mask = (theta > theta_min) & (theta < theta_max) - theta = theta[mask] - ax.plot(theta, theta * xi_plus[mask], label=label, **line_args[idx]) - ax2.plot(theta, theta * xi_minus[mask], label=label, **line_args[idx]) - -ymin = ax.get_ylim()[0] -ymax = ax.get_ylim()[1] -ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color="gray", alpha=0.2) -ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=300, color="gray", alpha=0.2) - -ax.set_ylim(ymin, ymax) - -ax.set_ylabel(r"$\theta \xi_\pm$", fontsize=26) -ax.set_xlabel(r"$\theta$ (arcmin)", fontsize=26) -ax.set_xlim([theta.min() - 0.1, theta.max() + 20]) -ax.set_title(r"$\xi_+(\theta)$", fontsize=26) -ax.set_xscale("log") -ax.set_xticks(np.array([1, 10, 100])) -ax.tick_params(axis="x", which="minor", length=2, width=0.8) -ax.tick_params(axis="both", which="major", labelsize=24) -ax.tick_params(axis="both", which="minor", labelsize=20) -ax.yaxis.get_offset_text().set_fontsize(24) -ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) - - -ymin = ax2.get_ylim()[0] -ymax = ax2.get_ylim()[1] -ax2.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color="gray", alpha=0.2) -ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=3000, color="gray", alpha=0.2) - -ax2.set_ylim(ymin, ymax) -ax2.set_xlabel(r"$\theta$ (arcmin)", fontsize=26) -ax2.set_xlim([theta.min() - 0.1, theta.max()]) -ax2.set_xscale("log") -ax2.set_title(r"$\xi_-(\vartheta)$", fontsize=26) -ax2.set_xticks(np.array([1, 10, 100])) -ax2.tick_params(axis="x", which="minor", length=2, width=0.8) -ax2.tick_params(axis="both", which="major", labelsize=24) -ax2.tick_params(axis="both", which="minor", labelsize=20) -ax2.yaxis.get_offset_text().set_fontsize(24) -ax2.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) -ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=20) - -plt.savefig("./../../results/nonlin_xipm_SP_v1.4.6.3_B.pdf", bbox_inches="tight") diff --git a/papers/realspace/cov_masking.py b/papers/realspace/cov_masking.py deleted file mode 100644 index 210638cb..00000000 --- a/papers/realspace/cov_masking.py +++ /dev/null @@ -1,82 +0,0 @@ -# # Covmat mask analysis -# -# This notebook creates the plots to look at the ratio of the covaraiance matrices when applying the mask or not - - -import os - -import healpy as hp -import matplotlib.pyplot as plt -import numpy as np -import seaborn as sns - -plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") - -plt.rcParams["axes.labelsize"] = 18 -plt.rcParams["xtick.labelsize"] = 18 -plt.rcParams["ytick.labelsize"] = 18 - -plt.rcParams["text.usetex"] = True -sns.set_palette("husl") - -cat_dir = "/n17data/UNIONS/WL/v1.4.x/" -catalog_ver = "v1.4.6.3" -blind = "B" - -nside = 8192 -npix = hp.nside2npix(nside) - -data_dir = "/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/data/" -curr_dir = os.getcwd() - - -# PLOT 2D MAP OF COVMAT masked vs unmasked RATIOS -nbins = 20 -ndata = nbins * 2 -full_ratio = np.zeros((ndata, ndata)) - -cov = np.loadtxt(data_dir + f"/covs/cov_SP_{catalog_ver}_{blind}.txt") -cov_masked = np.loadtxt(data_dir + f"/covs/cov_masked_SP_{catalog_ver}_{blind}.txt") - -for i in range(ndata): - for j in range(ndata): - full_ratio[i][j] = cov_masked[i][j] / cov[i][j] - -fig = plt.figure() -ax = fig.add_subplot(1, 1, 1) -extent = (0, ndata, ndata, 0) - -vmin, vmax = np.percentile(full_ratio, [1, 99]) - -im3 = ax.imshow(full_ratio, cmap="RdBu_r", vmin=vmin, vmax=vmax, extent=extent) - -cbar = fig.colorbar(im3, ax=ax, fraction=0.046, pad=0.04) - -ax.text(int(ndata / 4), ndata + 5, r"$\xi_+$", fontsize=15) -ax.text(3 * int(ndata / 4), ndata + 5, r"$\xi_-$", fontsize=15) -ax.text(-8, int(ndata / 4), r"$\xi_+$", fontsize=15, rotation=90) -ax.text(-8, 3 * int(ndata / 4), r"$\xi_-$", fontsize=15, rotation=90) -ax.set_xticks([0, 10, 20, 30, 40]) -ax.set_yticks([0, 10, 20, 30, 40]) -ax.set_yticklabels(["1'", "125'", "250'", "125'", "250'"]) -ax.set_xticklabels(["1'", "125'", "250'", "125'", "250'"]) -plt.axvline(x=int(ndata / 2), color="white", linewidth=1.0) -plt.axhline(y=int(ndata / 2), color="white", linewidth=1.0) - -plt.savefig( - f"./../../results/covmat_masked_unmasked_ratio_{catalog_ver}_{blind}.pdf", - bbox_inches="tight", -) - - -theta = np.linspace(1, 250, 20) -plt.axhline(y=1, color="k", ls="--") -plt.plot(theta, np.diag(cov_masked)[:20] / np.diag(cov)[:20], label=r"$\xi_+$") -plt.plot(theta, np.diag(cov_masked)[20:] / np.diag(cov)[20:], label=r"$\xi_-$") - -plt.xlabel(r"$\theta$ (arcmin)") -plt.ylabel("Cov masked / Cov unmasked") -plt.legend(fontsize=20) -plt.savefig( - "./../../results/covmat_masked_unmasked_ratio_diag.pdf", bbox_inches="tight" -) diff --git a/papers/realspace/get_chi2.py b/papers/realspace/get_chi2.py deleted file mode 100644 index c87a33da..00000000 --- a/papers/realspace/get_chi2.py +++ /dev/null @@ -1,564 +0,0 @@ -import configparser -import os -import re -import subprocess -import sys - -import matplotlib.pyplot as plt -import numpy as np -import scipy.stats as stats -from astropy.io import fits -from getdist import plots -from scipy.interpolate import interp1d - -sys.path.append("/home/guerrini/sp_validation/cosmo_inference/scripts") - -import chain_postprocessing - -plt.rc("mathtext", fontset="stix") -plt.rc("font", family="sans-serif") - -g = plots.get_subplot_plotter(width_inch=30) -g.settings.axes_fontsize = 30 -g.settings.axes_labelsize = 30 -g.settings.alpha_filled_add = 0.7 -g.settings.legend_fontsize = 40 - -# #SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE -root_dir = "/n09data/guerrini/output_chains/" -blind = "B" - -roots = [ - f"SP_v1.4.6.3_{blind}_fiducial_config", - f"SP_v1.4.6.3_{blind}_small_scales_config", - f"SP_v1.4.6.3_{blind}_flat_alpha_beta_config", - f"SP_v1.4.6.3_{blind}_no_xi_sys_config", - f"SP_v1.4.6.3_{blind}_no_leak_corr_config", - f"SP_v1.4.6.3_{blind}_flat_delta_z_config", - f"SP_v1.4.6.3_{blind}_no_delta_z_config", - f"SP_v1.4.6.3_{blind}_flat_ia_config", - f"SP_v1.4.6.3_{blind}_no_ia_config", - f"SP_v1.4.6.3_{blind}_no_m_bias_config", - f"SP_v1.4.6.3_{blind}_unmasked_covmat_config", - f"SP_v1.4.6.3_{blind}_halofit_config", - f"SP_v1.4.6.3_{blind}_no_baryons_config", - f"SP_v1.4.6.3_{blind}_nautilus_config", - f"SP_v1.4.6.3_{blind}_planck_config", - f"SP_v1.4.6.3_{blind}_planck_desi_config", -] - -catalog_versions = [ - f"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}", -] - -catalog_sub_versions = [ - f"SP_v1.4.6.3_leak_corr_{blind}_masked", - f"SP_v1.4.6.3_leak_corr_{blind}_masked", - f"SP_v1.4.6.3_leak_corr_{blind}_masked", - f"SP_v1.4.6.3_leak_corr_{blind}_masked", - f"SP_v1.4.6.3_{blind}_masked", - f"SP_v1.4.6.3_leak_corr_{blind}_masked", - f"SP_v1.4.6.3_leak_corr_{blind}_masked", - f"SP_v1.4.6.3_leak_corr_{blind}_masked", - f"SP_v1.4.6.3_leak_corr_{blind}_masked", - f"SP_v1.4.6.3_leak_corr_{blind}_masked", - f"SP_v1.4.6.3_leak_corr_{blind}", - f"SP_v1.4.6.3_leak_corr_{blind}_masked", - f"SP_v1.4.6.3_leak_corr_{blind}_masked", - f"SP_v1.4.6.3_leak_corr_{blind}_masked", - f"SP_v1.4.6.3_leak_corr_{blind}_masked", - f"SP_v1.4.6.3_leak_corr_{blind}_masked", -] -output_folder = "/n09data/guerrini/output_chains/" - -path_ini_files = "/home/guerrini/sp_validation/cosmo_inference/cosmosis_config/" - - -ini_roots = [ - f"blind_{blind}/fiducial", - f"blind_{blind}/small_scales", - f"blind_{blind}/flat_alpha_beta", - f"blind_{blind}/no_xi_sys", - f"blind_{blind}/no_leak_corr", - f"blind_{blind}/flat_delta_z", - f"blind_{blind}/no_delta_z", - f"blind_{blind}/flat_ia", - f"blind_{blind}/no_ia", - f"blind_{blind}/no_m_bias", - f"blind_{blind}/unmasked_covmat", - f"blind_{blind}/halofit", - f"blind_{blind}/no_baryons", - f"blind_{blind}/nautilus", - f"blind_{blind}/planck", - f"blind_{blind}/planck_desi", -] - -properties = {} - -for i, root in enumerate(roots): - print(root) - config = configparser.ConfigParser() - config.optionxform = str # Preserve case sensitivity of option names - config.read( - path_ini_files - + "config_space_v1.4.6.3_fiducial/pipeline/" - + ini_roots[i] - + ".ini" - ) - add_xi_sys = config["2pt_like"]["add_xi_sys"] - lower_bound_xi_plus, upper_bound_xi_plus = map( - float, config["2pt_like"]["angle_range_XI_PLUS_1_1"].split() - ) - lower_bound_xi_minus, upper_bound_xi_minus = map( - float, config["2pt_like"]["angle_range_XI_MINUS_1_1"].split() - ) - - properties[root] = { - "add_xi_sys": add_xi_sys, - "lower_bound_xi_plus": lower_bound_xi_plus, - "upper_bound_xi_plus": upper_bound_xi_plus, - "lower_bound_xi_minus": lower_bound_xi_minus, - "upper_bound_xi_minus": upper_bound_xi_minus, - } - - -# ## Retrieve the chains - - -# READ CHAIN - -chains = [] - -for i, root in enumerate(roots): - burnin = 0 - - if os.path.isfile(root_dir + "{}/getdist_{}.txt".format(root, root)) == False: - samples = np.loadtxt(root_dir + "{}/samples_{}.txt".format(root, root)) - - if "nautilus" in root: - samples = np.column_stack( - ( - np.exp(samples[:, -3]), - samples[:, -1] - samples[:, -2], - samples[:, 0:-3], - ) - ) - elif "mh" in root: - samples = np.column_stack( - ( - np.ones_like(samples[:, -1]), - np.log(samples[:, -1]) - np.log(samples[:, -2]), - samples[:, 0:-2], - ) - ) - burnin = 0.3 - else: - samples = np.column_stack( - (samples[:, -1], samples[:, -3], samples[:, 0:-4]) - ) - - np.savetxt(root_dir + "{}/getdist_{}.txt".format(root, root), samples) - - chain = g.samples_for_root( - root_dir + "{}/getdist_{}".format(root, root), - cache=False, - settings={ - "ignore_rows": burnin, - "smooth_scale_2D": 0.5, - "smooth_scale_1D": 0.5, - }, - ) - p = chain.getParams() - - chains.append(chain) - - -param_list = [ - "OMEGA_M", - "ombh2", - "h0", - "n_s", - "SIGMA_8", - "s_8_input", - "logt_agn", - "a", - "m1", - "bias_1", - "alpha", - "beta", - "omch2", - "m", - "a_planck", -] -label_list = [ - r"\Omega_m", - r"\omega_b", - "h_0", - "n_s", - r"\sigma_8", - "S_8", - "log T_{AGN}", - "A_{IA}", - "m_1", - r"\Delta z_1", - "\\alpha_{PSF}", - "\\beta_{PSF}", - r"\omega_c", - "M", - "A_{\rm Planck}", -] - -for chain in chains: - param_names = chain.getParamNames() - for name, label in zip(param_list, label_list): - if param_names.parWithName(name) is not None: - param_names.parWithName(name).label = label - - -# ## Extract the best fit parameters - - -best_fit = {} - -for root, chain in zip(roots, chains): - print(root) - p = chain.getParams() - - best_fit[root] = chain_postprocessing.extract_best_fit_params( - chain, best_fit_method="2Dkde" - ) - - for param_name in best_fit[root].keys(): - high_68, low_68, high_95, low_95 = chain_postprocessing.compute_limits( - chain, param_name - ) - if param_name == "S_8": - print(f"{best_fit[root][param_name]}") - - -# ## Run `Cosmosis` in test mode to get the data vectors - - -if not os.path.exists(path_ini_files + "/values_empty.ini"): - content = """[cosmological_parameters] - -tau = 0.0544 -w = -1.0 -mnu = 0.06 -omega_k = 0.0 -wa = 0.0 - -[halo_model_parameters] - -[intrinsic_alignment_parameters] - -[shear_calibration_parameters] - -[nofz_shifts] - -[psf_leakage_parameters] -""" - - with open(path_ini_files + "/values_empty.ini", "w") as f: - f.write(content) - f.close() - - print("File created successfully") - - -section_map = { - "omch2": "cosmological_parameters", - "ombh2": "cosmological_parameters", - "h0": "cosmological_parameters", - "n_s": "cosmological_parameters", - "tau": "cosmological_parameters", - "s_8_input": "cosmological_parameters", - "logt_agn": "halo_model_parameters", - "a": "intrinsic_alignment_parameters", - "m1": "shear_calibration_parameters", - "bias_1": "nofz_shifts", - "alpha": "psf_leakage_parameters", - "beta": "psf_leakage_parameters", - "m": "supernova_params", - "a_planck": "planck", -} - -best_fit["SP_v1.4.6.3_B_no_ia_config"]["a"] = 0 - - -env = os.environ.copy() -env["LD_LIBRARY_PATH"] = ( - "/home/guerrini/.conda/envs/sp_validation/lib/python3.9/site-packages/cosmosis/datablock:" - + env.get("LD_LIBRARY_PATH", "") -) - -for i, root in enumerate(roots): - print(root) - config = configparser.ConfigParser() - config.optionxform = str # Preserve case sensitivity of option names - - for param, section in section_map.items(): - # Check if this parameter exists for the current root - if param in best_fit[root]: - value = best_fit[root][param] - - if section not in config: - config.add_section(section) - - config[section][param] = str(value) - - with open(path_ini_files + "/values_empty.ini", "w") as configfile: - config.write(configfile) - - # Modify the ini file to run in test mode at the best fit - config = configparser.ConfigParser() - config.optionxform = str # Preserve case sensitivity of option names - - ini_file = path_ini_files + "config_space_v1.4.6.3_fiducial/pipeline/{}.ini".format( - ini_roots[i] - ) - config.read(ini_file) - - sampler = config["runtime"]["sampler"] - config["runtime"]["sampler"] = "test" - values = config["pipeline"]["values"] - config["pipeline"]["values"] = path_ini_files + "/values_empty.ini" - config["DEFAULT"]["FITS_FILE"] = ( - f"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_versions[0]}/cosmosis_{catalog_sub_versions[i]}.fits" - ) - config["test"]["save_dir"] = root_dir + "{}/best_fit".format(root) - - with open(ini_file, "w") as configfile: - config.write(configfile) - - # Run cosmosis - result = subprocess.run( - ["cosmosis", ini_file], env=env, capture_output=True, text=True - ) - print(f"STDOUT:\n{result.stdout}") - print(f"STDERR:\n{result.stderr}") - - # Modify the ini file to the previous one - config["pipeline"]["values"] = values - config["runtime"]["sampler"] = sampler - - with open(ini_file, "w") as configfile: - config.write(configfile) - - -# ## Compute the $\chi^2$ - - -metrics = {} - -for idx, root in enumerate(roots): - print(root) - match = re.search(r"corr_([A-Za-z])", root) - if match: - blind = match.group(1) - - add_xi_sys = properties[root]["add_xi_sys"] - print(f"add_xi_sys: {add_xi_sys}") - lower_bound_xi_plus = properties[root]["lower_bound_xi_plus"] - upper_bound_xi_plus = properties[root]["upper_bound_xi_plus"] - lower_bound_xi_minus = properties[root]["lower_bound_xi_minus"] - upper_bound_xi_minus = properties[root]["upper_bound_xi_minus"] - - # Read the results - theta = np.loadtxt( - output_folder + "{}/best_fit/shear_xi_plus/theta.txt".format(root) - ) - theta_arcmin = theta * 180 * 60 / np.pi - shear_xi_plus = np.loadtxt( - output_folder + "{}/best_fit/shear_xi_plus/bin_1_1.txt".format(root) - ) - shear_xi_minus = np.loadtxt( - output_folder + "{}/best_fit/shear_xi_minus/bin_1_1.txt".format(root) - ) - - if add_xi_sys == "T": - xi_sys_plus = np.loadtxt( - output_folder + "{}/best_fit/xi_sys/shear_xi_plus.txt".format(root) - ) - xi_sys_minus = np.loadtxt( - output_folder + "{}/best_fit/xi_sys/shear_xi_minus.txt".format(root) - ) - - theta_tau = np.loadtxt( - output_folder + "{}/best_fit/tau_0_plus/theta.txt".format(root) - ) - theta_tau_arcmin = theta_tau * 180 * 60 / np.pi - tau_0_model = np.loadtxt( - output_folder + "{}/best_fit/tau_0_plus/bin_1_1.txt".format(root) - ) - tau_2_model = np.loadtxt( - output_folder + "{}/best_fit/tau_2_plus/bin_1_1.txt".format(root) - ) - - data = fits.open( - f"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_versions[0]}/cosmosis_{catalog_sub_versions[idx]}.fits" - ) - - tau_0_data = data["TAU_0_PLUS"].data["VALUE"] - tau_2_data = data["TAU_2_PLUS"].data["VALUE"] - - theta_data = data["XI_PLUS"].data["ANG"] - xi_plus_data = data["XI_PLUS"].data["VALUE"] - xi_minus_data = data["XI_MINUS"].data["VALUE"] - - # Load the covariance - cov = data["COVMAT"].data - cov_xi = cov[0 : 2 * len(xi_plus_data), 0 : 2 * len(xi_plus_data)] - cov_tau = cov[2 * len(xi_plus_data) :, 2 * len(xi_plus_data) :] - - # interpolate the model - interp_xi_plus = interp1d( - theta_arcmin, shear_xi_plus, kind="cubic", fill_value="extrapolate" - ) - interp_xi_minus = interp1d( - theta_arcmin, shear_xi_minus, kind="cubic", fill_value="extrapolate" - ) - - xi_plus_model = interp_xi_plus(theta_data) - if add_xi_sys: - xi_plus_model += xi_sys_plus - xi_minus_model = interp_xi_minus(theta_data) - if add_xi_sys: - xi_minus_model += xi_sys_minus - - # Concatenate the data vector - xi_data = np.concatenate((xi_plus_data, xi_minus_data)) - xi_model = np.concatenate((xi_plus_model, xi_minus_model)) - - tau_data = np.concatenate((tau_0_data, tau_2_data)) - tau_model = np.concatenate((tau_0_model, tau_2_model)) - - # Apply scale cuts - mask_xi_plus = (theta_data > lower_bound_xi_plus) & ( - theta_data < upper_bound_xi_plus - ) - mask_xi_minus = (theta_data > lower_bound_xi_minus) & ( - theta_data < upper_bound_xi_minus - ) - mask = np.concatenate((mask_xi_plus, mask_xi_minus)) - - xi_data = xi_data[mask] - xi_model = xi_model[mask] - cov_xi = cov_xi[mask][:, mask] - - cov_xi_plus = cov[0 : len(xi_plus_data), 0 : len(xi_plus_data)] - cov_xi_plus = cov_xi_plus[mask_xi_plus][:, mask_xi_plus] - cov_xi_minus = cov[ - len(xi_plus_data) : 2 * len(xi_minus_data), - len(xi_plus_data) : 2 * len(xi_minus_data), - ] - cov_xi_minus = cov_xi_minus[mask_xi_minus][:, mask_xi_minus] - - xi_plus_chi2 = np.dot( - (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]), - np.dot( - np.linalg.inv(cov_xi_plus), - (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]), - ), - ) - xi_minus_chi2 = np.dot( - (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]), - np.dot( - np.linalg.inv(cov_xi_minus), - (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]), - ), - ) - xi_chi2 = np.dot( - (xi_model - xi_data), np.dot(np.linalg.inv(cov_xi), (xi_model - xi_data)) - ) - tau_chi2 = np.dot( - (tau_model - tau_data), np.dot(np.linalg.inv(cov_tau), (tau_model - tau_data)) - ) - n_dof_xi_plus = np.sum(mask_xi_plus) - n_dof_xi_minus = np.sum(mask_xi_minus) - n_dof_tau = len(tau_0_data) + len(tau_2_data) - p_value_xi_plus = 1 - stats.chi2.cdf(xi_plus_chi2, n_dof_xi_plus) - p_value_xi_minus = 1 - stats.chi2.cdf(xi_minus_chi2, n_dof_xi_minus) - p_value_xi = 1 - stats.chi2.cdf(xi_chi2, n_dof_xi_plus + n_dof_xi_minus) - p_value_tau = 1 - stats.chi2.cdf(tau_chi2, n_dof_tau) - chi2_tot = xi_plus_chi2 + xi_minus_chi2 + tau_chi2 - n_dof_tot = n_dof_xi_plus + n_dof_xi_minus + n_dof_tau - p_value_tot = 1 - stats.chi2.cdf(chi2_tot, n_dof_tot) - - metrics[root] = { - "chi2_xi_plus": xi_plus_chi2, - "n_dof_xi_plus": n_dof_xi_plus, - "p_value_xi_plus": p_value_xi_plus, - "chi2_xi_minus": xi_minus_chi2, - "n_dof_xi_minus": n_dof_xi_minus, - "p_value_xi_minus": p_value_xi_minus, - "chi2_xi": xi_chi2, - "p_value_xi": p_value_xi, - "chi2_tau": tau_chi2, - "n_dof_tau": n_dof_tau, - "p_value_tau": p_value_tau, - "chi2_tot": chi2_tot, - "n_dof_tot": n_dof_tot, - "p_value_tot": p_value_tot, - } - print("Done!") - - -def get_latex_table(metrics): - latex_lines = [ - r"\begin{tabular}{lccc|ccc|ccc}", - r"\hline", - r"Root & $\chi^2_{\xi^+}$/dof & $p_{\xi^+}$ & $\chi^2_{\xi^-}$/dof & $p_{\xi^+}$ & $\chi^2_{\xi}$/dof & $p_{\xi}$ &" - r"$\chi^2_\tau$/dof & $p_\tau$ & $\chi^2_{\text{tot}}$/dof & $p_{\text{tot}}$ \\", - r"\hline", - ] - - for root, vals in metrics.items(): - escaped = root.replace("_", r"\_") - line = ( - f"{escaped} & " - f"{vals['chi2_xi_plus']:.2f}/{vals['n_dof_xi_plus']} & {vals['p_value_xi_plus']:.3g} & " - f"{vals['chi2_xi_minus']:.2f}/{vals['n_dof_xi_minus']} & {vals['p_value_xi_minus']:.3g} & " - f"{vals['chi2_xi']:.2f}/{vals['n_dof_xi_plus'] + vals['n_dof_xi_minus']} & {vals['p_value_xi']:.3g} &" - f"{vals['chi2_tau']:.2f}/{vals['n_dof_tau']} & {vals['p_value_tau']:.3g} & " - f"{vals['chi2_tot']:.2f}/{vals['n_dof_tot']} & {vals['p_value_tot']:.3g} \\\\" - ) - latex_lines.append(line) - - latex_lines.append(r"\hline") - latex_lines.append(r"\end{tabular}") - - # Print LaTeX table - print("\n".join(latex_lines)) - - -get_latex_table(metrics) - - -def display_markdown(metrics): - # Build Markdown table - header = ( - "| Root | $\\chi^2$ (ξ⁺) / dof | p-val (ξ⁺) |$\\chi^2$ (ξ-) / dof | p-val (ξ-) | $\\chi^2$ (ξ) / dof | p-val (ξ) | $\\chi^2$ (τ) / dof | p-val (τ) | $\\chi^2$ (tot) / dof | p-val (tot) |\n" - "|------|----------------|------------|----------------|------------|------------|---------------|------------|------------|------------------|--------------|\n" - ) - - rows = [] - for root, vals in metrics.items(): - row = f"| `{root}` " - row += f"| {vals['chi2_xi_plus']:.2f} / {vals['n_dof_xi_plus']} " - row += f"| {vals['p_value_xi_plus']:.5f} " - row += f"| {vals['chi2_xi_minus']:.2f} / {vals['n_dof_xi_minus']} " - row += f"| {vals['p_value_xi_minus']:.5f} " - row += f"| {vals['chi2_xi']:.2f} / {vals['n_dof_xi_minus'] + vals['n_dof_xi_plus']} " - row += f"| {vals['p_value_xi']:.5f} " - row += f"| {vals['chi2_tau']:.2f} / {vals['n_dof_tau']} " - row += f"| {vals['p_value_tau']:.5f} " - row += f"| {vals['chi2_tot']:.2f} / {vals['n_dof_tot']} " - row += f"| {vals['p_value_tot']:.5f} |" - rows.append(row) - - # Display in Jupyter - return header + "\n".join(rows) - - -markdown_source = display_markdown(metrics) diff --git a/papers/realspace/get_chi2_glass_mock.py b/papers/realspace/get_chi2_glass_mock.py deleted file mode 100644 index 596da161..00000000 --- a/papers/realspace/get_chi2_glass_mock.py +++ /dev/null @@ -1,468 +0,0 @@ -import configparser -import os -import subprocess -import sys - -import matplotlib.pyplot as plt -import numpy as np - -# Make the plot -import seaborn as sns -from astropy.io import fits -from getdist import plots -from scipy.interpolate import interp1d -from scipy.stats import chi2 - -sys.path.append("/home/guerrini/sp_validation/cosmo_inference/scripts") - -import chain_postprocessing - -plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") - -plt.rcParams["axes.labelsize"] = 18 -plt.rcParams["xtick.labelsize"] = 18 -plt.rcParams["ytick.labelsize"] = 18 - -plt.rcParams["text.usetex"] = True - -g = plots.get_subplot_plotter(width_inch=30) -g.settings.axes_fontsize = 30 -g.settings.axes_labelsize = 30 -g.settings.alpha_filled_add = 0.7 -g.settings.legend_fontsize = 40 - -# #SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE - -root_dir = "/n09data/guerrini/glass_mock_chains/" - -# Version of the glass mock chain run -chain_version = "v6" - -# Path to the glass mock data vectors -root_glass_dv = ( - f"/home/guerrini/sp_validation/cosmo_inference/data/glass_mocks/{chain_version}/" -) - -# Choose the best-fit method -best_fit_method = "2Dkde" - -# Create the list of mocks -max_sim = 350 -failed_simulations = [82, 83, 281, 282, 283, 284, 285, 286, 287] -roots = [f"glass_mock_{chain_version}_{str(i).zfill(5)}" for i in range(1, max_sim + 1)] -roots = [root for root in roots if int(root.split("_")[-1]) not in failed_simulations] - -catalog_versions = [ - "SP_v1.4.6.3_config/SP_v1.4.6.3_A", -] - -output_folder_chains = "/n23data1/n06data/lgoh/scratch/temp/" -path_ini_files = "/home/xguerrini/sp_validation/cosmo_inference/cosmosis_config/" - -ini_root = "blind_A/fiducial" - -lower_bound_xi = 12 -upper_bound_xi = 83 - -# ## Retrieve the chains - - -# READ CHAIN - -chains = [] -best_fit = {} - -for i, root in enumerate(roots): - burnin = 0 - - if os.path.isfile(f"{root_dir}/{root}/{root}/getdist_{root}.txt") == True: - chain = g.samples_for_root( - f"{root_dir}/{root}/{root}/getdist_{root}", - cache=False, - settings={ - "ignore_rows": burnin, - "smooth_scale_2D": 0.5, - "smooth_scale_1D": 0.5, - }, - ) - p = chain.getParams() - - best_fit[root] = chain_postprocessing.extract_best_fit_params( - chain, best_fit_method="2Dkde" - ) - - -param_list = [ - "OMEGA_M", - "ombh2", - "h0", - "n_s", - "SIGMA_8", - "s_8_input", - "logt_agn", - "a", - "m1", - "bias_1", - "alpha", - "beta", - "omch2", - "m", - "a_planck", -] -label_list = [ - r"\Omega_m", - r"\omega_b", - "h_0", - "n_s", - r"\sigma_8", - "S_8", - "log T_{AGN}", - "A_{IA}", - "m_1", - r"\Delta z_1", - "\\alpha_{PSF}", - "\\beta_{PSF}", - r"\omega_c", - "M", - "A_{\rm Planck}", -] - - -# ## Run `Cosmosis` in test mode to get the data vectors - - -if not os.path.exists(path_ini_files + "/values_empty.ini"): - content = """[cosmological_parameters] - -tau = 0.0544 -w = -1.0 -mnu = 0.06 -omega_k = 0.0 -wa = 0.0 - -[halo_model_parameters] - -[intrinsic_alignment_parameters] - -[shear_calibration_parameters] - -[nofz_shifts] - -[psf_leakage_parameters] -""" - - with open(path_ini_files + "/values_empty.ini", "w") as f: - f.write(content) - f.close() - - print("File created successfully") - - -section_map = { - "omch2": "cosmological_parameters", - "ombh2": "cosmological_parameters", - "h0": "cosmological_parameters", - "n_s": "cosmological_parameters", - "s_8_input": "cosmological_parameters", - "logt_agn": "halo_model_parameters", - "a": "intrinsic_alignment_parameters", - "m1": "shear_calibration_parameters", - "bias_1": "nofz_shifts", - "alpha": "psf_leakage_parameters", - "beta": "psf_leakage_parameters", -} - - -env = os.environ.copy() -env["LD_LIBRARY_PATH"] = ( - "/home/guerrini/.conda/envs/sp_validation/lib/python3.9/site-packages/cosmosis/datablock:" - + env.get("LD_LIBRARY_PATH", "") -) -for i, root in enumerate(roots): - print(root) - config = configparser.ConfigParser() - config.optionxform = str # Preserve case sensitivity of option names - - for param, section in section_map.items(): - # Check if this parameter exists for the current root - if param in best_fit[root]: - value = best_fit[root][param] - - if section not in config: - config.add_section(section) - - config[section][param] = str(value) - - with open(path_ini_files + "/values_empty.ini", "w") as configfile: - config.write(configfile) - - # Modify the ini file to run in test mode at the best fit - config = configparser.ConfigParser() - config.optionxform = str # Preserve case sensitivity of option names - - ini_file = ( - path_ini_files + f"config_space_v1.4.6.3_fiducial/pipeline/{ini_root}.ini" - ) - config.read(ini_file) - - sampler = config["runtime"]["sampler"] - config["runtime"]["sampler"] = "test" - values = config["pipeline"]["values"] - config["pipeline"]["values"] = path_ini_files + "/values_empty.ini" - config["DEFAULT"]["FITS_FILE"] = ( - f"{root_glass_dv}/glass_mock_{root[-5:]}/cosmosis_glass_mock_v6_{root[-5:]}.fits" - ) - config["test"]["save_dir"] = output_folder_chains + f"{root}/best_fit_config" - - with open(ini_file, "w") as configfile: - config.write(configfile) - - # Run cosmosis - result = subprocess.run( - ["cosmosis", ini_file], env=env, capture_output=True, text=True - ) - # print(f"STDOUT:\n{result.stdout}") - # print(f"STDERR:\n{result.stderr}") - - # Modify the ini file to the previous one - config["pipeline"]["values"] = values - config["runtime"]["sampler"] = sampler - - with open(ini_file, "w") as configfile: - config.write(configfile) - - -xi_plus_chi2s = np.array([]) -xi_minus_chi2s = np.array([]) -xi_chi2s = np.array([]) -tau_chi2s = np.array([]) -chi2_tots = np.array([]) - - -for idx, root in enumerate(roots): - print(root) - - data = fits.open( - f"{root_glass_dv}/glass_mock_{root[-5:]}/cosmosis_glass_mock_v6_{root[-5:]}.fits" - ) - - tau_0_data = data["TAU_0_PLUS"].data["VALUE"] - tau_2_data = data["TAU_2_PLUS"].data["VALUE"] - - theta_data = data["XI_PLUS"].data["ANG"] - xi_plus_data = data["XI_PLUS"].data["VALUE"] - xi_minus_data = data["XI_MINUS"].data["VALUE"] - xi_data = np.concatenate((xi_plus_data, xi_minus_data)) - - tau_data = np.concatenate((tau_0_data, tau_2_data)) - - # Apply scale cuts - mask_xi_plus = (theta_data > lower_bound_xi) & (theta_data < upper_bound_xi) - mask_xi_minus = (theta_data > lower_bound_xi) & (theta_data < upper_bound_xi) - mask = np.concatenate((mask_xi_plus, mask_xi_minus)) - # Load the covariance - cov = data["COVMAT"].data - cov_xi = cov[0 : 2 * len(xi_plus_data), 0 : 2 * len(xi_plus_data)] - cov_tau = cov[ - 2 * len(xi_plus_data) : 4 * len(xi_plus_data), - 2 * len(xi_plus_data) : 4 * len(xi_plus_data), - ] - xi_data = xi_data[mask] - cov_xi = cov_xi[mask][:, mask] - - cov_xi_plus = cov[0 : len(xi_plus_data), 0 : len(xi_plus_data)] - cov_xi_plus = cov_xi_plus[mask_xi_plus][:, mask_xi_plus] - cov_xi_minus = cov[ - len(xi_plus_data) : 2 * len(xi_minus_data), - len(xi_plus_data) : 2 * len(xi_minus_data), - ] - cov_xi_minus = cov_xi_minus[mask_xi_minus][:, mask_xi_minus] - - # Read the results - theta = np.loadtxt( - output_folder_chains + f"{root}/best_fit_config/shear_xi_plus/theta.txt" - ) - theta_arcmin = theta * 180 * 60 / np.pi - shear_xi_plus = np.loadtxt( - output_folder_chains + f"{root}/best_fit_config/shear_xi_plus/bin_1_1.txt" - ) - shear_xi_minus = np.loadtxt( - output_folder_chains + f"{root}/best_fit_config/shear_xi_minus/bin_1_1.txt" - ) - - xi_sys_plus = np.loadtxt( - output_folder_chains + f"{root}/best_fit_config/xi_sys/shear_xi_plus.txt" - ) - xi_sys_minus = np.loadtxt( - output_folder_chains + f"{root}/best_fit_config/xi_sys/shear_xi_minus.txt" - ) - - theta_tau = np.loadtxt( - output_folder_chains + f"{root}/best_fit_config/tau_0_plus/theta.txt" - ) - theta_tau_arcmin = theta_tau * 180 * 60 / np.pi - tau_0_model = np.loadtxt( - output_folder_chains + f"{root}/best_fit_config/tau_0_plus/bin_1_1.txt" - ) - tau_2_model = np.loadtxt( - output_folder_chains + f"{root}/best_fit_config/tau_2_plus/bin_1_1.txt" - ) - - # interpolate the model - interp_xi_plus = interp1d( - theta_arcmin, shear_xi_plus, kind="cubic", fill_value="extrapolate" - ) - interp_xi_minus = interp1d( - theta_arcmin, shear_xi_minus, kind="cubic", fill_value="extrapolate" - ) - - xi_plus_model = interp_xi_plus(theta_data) - xi_plus_model += xi_sys_plus - xi_minus_model = interp_xi_minus(theta_data) - xi_minus_model += xi_sys_minus - - xi_model = np.concatenate((xi_plus_model, xi_minus_model)) - tau_model = np.concatenate((tau_0_model, tau_2_model)) - xi_model = xi_model[mask] - - xi_plus_chi2 = np.dot( - (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]), - np.dot( - np.linalg.inv(cov_xi_plus), - (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]), - ), - ) - xi_minus_chi2 = np.dot( - (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]), - np.dot( - np.linalg.inv(cov_xi_minus), - (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]), - ), - ) - xi_chi2 = np.dot( - (xi_model - xi_data), np.dot(np.linalg.inv(cov_xi), (xi_model - xi_data)) - ) - tau_chi2 = np.dot( - (tau_model - tau_data), np.dot(np.linalg.inv(cov_tau), (tau_model - tau_data)) - ) - chi2_tot = xi_plus_chi2 + xi_minus_chi2 + tau_chi2 - - xi_plus_chi2s = np.append(xi_plus_chi2s, xi_plus_chi2) - xi_minus_chi2s = np.append(xi_minus_chi2s, xi_minus_chi2) - xi_chi2s = np.append(xi_chi2s, xi_chi2) - tau_chi2s = np.append(tau_chi2s, tau_chi2) - chi2_tots = np.append(chi2_tots, chi2_tot) - - -fig, [ax1, ax2] = plt.subplots(2, 1, figsize=(7, 10)) -chi2_fiducial = -2 * -37.560916821678894 -dof, loc, scale = chi2.fit(chi2_tots, floc=0) - -print(f"Best-fit dof: {dof:.3e}") -counts, bin_edges = np.histogram(chi2_tots, bins=25, density=True) - -sns.histplot( - chi2_tots, - ax=ax1, - kde=False, - bins=bin_edges, - stat="density", - label=r"$\chi^2$ for \texttt{GLASS} mocks best-fits", - color="green", - alpha=0.3, -) - -# Compute the p-value - -# 1. Get in which bin the chi2 of the fiducial falls -bin_index = np.digitize(chi2_fiducial, bin_edges) - -# 2. Compute the p-value as the integral of the tail of the histogram -p_value = np.sum(counts[bin_index:]) * np.diff(bin_edges)[0] - -print(f"P-value: {p_value}") - -ax1.axvline(chi2_fiducial, color="red", label=r"$\chi^2$ of the fiducial", lw=2) - -mantissa, exponent = np.frexp(p_value) -pte_string = rf"${{\rm PTE}} = {p_value:.4f}$" -print(f"mantissa: {mantissa}, exponent: {exponent}") -x_text = 78 -y_text = max(counts) * 0.95 -ax1.text( - x_text, - y_text, - pte_string, - fontsize=15, - bbox=dict(facecolor="wheat", alpha=0.8, edgecolor="black"), -) - -chi2_string = rf"${{\rm Eff. dof}}= {dof:.1f}$" -y_text = max(counts) * 0.85 -ax1.text( - x_text, - y_text, - chi2_string, - fontsize=15, - bbox=dict(facecolor="wheat", alpha=0.8, edgecolor="black"), -) - -ax1.set_xlabel(r"$\chi^2_{\rm tot}$") -ax1.set_ylabel("Density") - -chi2_fiducial = 9.5 -dof, loc, scale = chi2.fit(xi_chi2s, floc=0) - -print(f"Best-fit dof: {dof:.3e}") -counts, bin_edges = np.histogram(xi_chi2s, bins=25, density=True) - -sns.histplot( - xi_chi2s, - ax=ax2, - kde=False, - bins=bin_edges, - stat="density", - label=r"$\chi^2$ for \texttt{GLASS} mocks best-fits", - color="pink", - alpha=0.5, -) - -# Compute the p-value - -# 1. Get in which bin the chi2 of the fiducial falls -bin_index = np.digitize(chi2_fiducial, bin_edges) - -# 2. Compute the p-value as the integral of the tail of the histogram -p_value = np.sum(counts[bin_index:]) * np.diff(bin_edges)[0] - -print(f"P-value: {p_value}") - -ax2.axvline(chi2_fiducial, color="red", label=r"$\chi^2$ of the fiducial", lw=2) - -mantissa, exponent = np.frexp(p_value) -print(f"mantissa: {mantissa}, exponent: {exponent}") -pte_string = rf"${{\rm PTE}} = {p_value:.4f}$" -# rf"${{\rm PTE}} = {mantissa:.2f} \times 10^{{{exponent}}}$" if exponent != 0 else -x_text = 17.5 -y_text = max(counts) * 0.95 -ax2.text( - x_text, - y_text, - pte_string, - fontsize=15, - bbox=dict(facecolor="wheat", alpha=0.8, edgecolor="black"), -) - -chi2_string = rf"${{\rm Eff. dof}}= {dof:.1f}$" -y_text = max(counts) * 0.85 -ax2.text( - x_text, - y_text, - chi2_string, - fontsize=15, - bbox=dict(facecolor="wheat", alpha=0.8, edgecolor="black"), -) - -ax2.set_xlabel(r"$\chi^2 (\xi_\pm)$") -ax2.set_ylabel("Density") -fig.savefig("./../../results/chi2_glass_mocks_p_value_xi_tau.pdf") diff --git a/papers/realspace/get_prior_psf_leakage.py b/papers/realspace/get_prior_psf_leakage.py deleted file mode 100644 index 4d335085..00000000 --- a/papers/realspace/get_prior_psf_leakage.py +++ /dev/null @@ -1,163 +0,0 @@ -# # Covariance matrix and PSF leakage -# -# This notebook plots the combined covariance matrix, and samples and plots the 2D marginalised posteriors of the PSF leakage parameters $\alpha$ and $\beta$. - - -import matplotlib.pyplot as plt -import numpy as np -import seaborn as sns -from astropy.io import fits -from getdist import MCSamples, plots -from shear_psf_leakage.rho_tau_stat import PSFErrorFit, RhoStat, TauStat - -# Use paper style and seaborn with husl palette -plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") -# Set default palette - will be updated per plot as needed -sns.set_palette("husl") - -g = plots.get_subplot_plotter(width_inch=30) -g.settings.axes_fontsize = 30 -g.settings.axes_labelsize = 30 -g.settings.alpha_filled_add = 0.7 -g.settings.legend_fontsize = 25 - -ver = "v1.4.6.3" -blind = "B" - - -data_path = f"/home/guerrini/sp_validation/cosmo_inference/data/SP_{ver}_config/" - -path_cosmo_val = "/home/guerrini/sp_validation/cosmo_val/output/" - -roots = [f"SP_{ver}_{blind}", f"SP_{ver}_leak_corr_{blind}"] - -labels = [f"SP_{ver}_{blind}", f"SP_{ver}_leak_corr_{blind}"] - - -data_vectors = [] - -for root in roots: - data_vectors.append( - fits.open(data_path + f"SP_{ver}_{blind}/cosmosis_{root}_masked.fits") - ) - - -def cov_to_corr(cov): - """Convert a covariance matrix to a correlation matrix.""" - d = np.sqrt(np.diag(cov)) - corr = cov / np.outer(d, d) - corr[cov == 0] = 0 - return corr - - -# Print the covariance matrix for each root -for i, root in enumerate(roots): - print(f"Covariance matrix for {labels[i]}:") - cov = data_vectors[i]["COVMAT"].data - - n_bins = cov.shape[0] // 4 - - fig, ax = plt.subplots(figsize=(10, 8)) - - im = ax.imshow(cov_to_corr(cov), vmin=-1, vmax=1, cmap="seismic") - ax.set_aspect("equal") - ax.set_yticks(np.array([10, 30, 50, 70])) - ax.set_yticklabels( - [ - r"$\xi_+(\vartheta)$", - r"$\xi_-(\vartheta)$", - r"$\tau_0(\vartheta)$", - r"$\tau_2(\vartheta)$", - ] - ) - ax.set_xticks(np.array([10, 30, 50, 70])) - ax.set_xticklabels( - [ - r"$\xi_+(\vartheta)$", - r"$\xi_-(\vartheta)$", - r"$\tau_0(\vartheta)$", - r"$\tau_2(\vartheta)$", - ], - rotation=45, - ) - fig.colorbar(im, ax=ax) - - plt.savefig(f"./../../results/cov_matrix_{root}.png", bbox_inches="tight", dpi=300) - - -# Create dummy rho and tau stat handler. - -# Inference of the xi_sys parameters -sep_units = "arcmin" -coord_units = "degrees" -theta_min = 1.0 -theta_max = 250 -nbins = 20 - - -TreeCorrConfig_xi = { - "ra_units": coord_units, - "dec_units": coord_units, - "min_sep": theta_min, - "max_sep": theta_max, - "sep_units": sep_units, - "nbins": nbins, - "var_method": "jackknife", -} - -rho_stats_handler = RhoStat(output=".", treecorr_config=TreeCorrConfig_xi, verbose=True) - -tau_stats_handler = TauStat( - catalogs=rho_stats_handler.catalogs, - output=".", - treecorr_config=TreeCorrConfig_xi, - verbose=True, -) - - -# Create a PSFErrorFit instance -psf_fitter = PSFErrorFit( - rho_stats_handler, - tau_stats_handler, - path_cosmo_val + "rho_tau_stats/", - use_eta=False, -) - -g = plots.get_subplot_plotter(width_inch=30) - -g.settings.axes_fontsize = 30 -g.settings.axes_labelsize = 30 -g.settings.alpha_filled_add = 0.7 -g.settings.legend_fontsize = 40 - -chains = [] - -# Load rho-, tau-statistics, and cov_tau from the data_vector -for i, root in enumerate(roots): - print("Sampling PSF parameters for ", labels[i]) - path_rho = f"rho_stats_{root}.fits" - path_tau = f"tau_stats_{root}.fits" - path_cov_rho = f"cov_rho_{root}.npy" - path_cov_tau = f"cov_tau_{root}_th.npy" - psf_fitter.load_rho_stat(path_rho) - psf_fitter.load_tau_stat(path_tau) - psf_fitter.load_covariance(path_cov_rho, cov_type="rho") - psf_fitter.load_covariance(path_cov_tau, cov_type="tau") - samples_lq, _, _ = psf_fitter.get_least_squares_params_samples( - npatch=None, apply_debias=False - ) - - samples_gd = MCSamples( - samples=samples_lq, names=[r"\alpha", r"\beta"], labels=[r"\alpha", r"\beta"] - ) - - chains.append(samples_gd) - -g.triangle_plot( - chains, - filled=True, - legend_labels=labels, - legend_loc="upper right", -) - -plt.savefig("./../../results/psf_leakage_params.png", bbox_inches="tight", dpi=300) diff --git a/papers/realspace/glass_mock_hist.py b/papers/realspace/glass_mock_hist.py deleted file mode 100644 index 97b34f1d..00000000 --- a/papers/realspace/glass_mock_hist.py +++ /dev/null @@ -1,458 +0,0 @@ -import os - -import matplotlib.pyplot as plt -import numpy as np -import seaborn as sns -from getdist import plots -from tqdm import tqdm - -g = plots.get_subplot_plotter(width_inch=7) -g.settings.axes_fontsize = 15 -g.settings.axes_labelsize = 15 -g.settings.alpha_filled_add = 0.7 -g.settings.legend_fontsize = 15 - -if os.path.exists("/home/guerrini/matplotlib_config/paper.mplstyle"): - plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") - -# Set default palette - will be updated per plot as needed -sns.set_palette("husl") - -root_dir = "/n09data/guerrini/glass_mock_chains/" -chain_version = "v6" -num_sims = 350 - -roots = [f"glass_mock_{chain_version}_{i + 1:05d}" for i in range(num_sims)] - - -# -def load_samples_and_write_paramames(root_dir, root, chain_type="configuration"): - assert chain_type in ["configuration", "harmonic"], ( - "chain_type must be 'configuration' or 'harmonic'" - ) - - if chain_type == "configuration": - path_samples = root_dir + "{}/{}/samples_{}.txt".format("/" + root, root, root) - path_paramnames = root_dir + "{}/{}/getdist_{}.paramnames".format( - "/" + root, root, root - ) - else: - path_samples = root_dir + "{}/{}/samples_{}_cell.txt".format( - "/" + root, root, root - ) - path_paramnames = root_dir + "{}/{}/getdist_{}_cell.paramnames".format( - "/" + root, root, root - ) - - with open(path_samples, "r") as file: - params = file.readline()[1:].split("\t")[:-4] - file.close() - - with open(path_paramnames, "w") as file: - for i in range(len(params)): - if len(params[i].split("--")) > 1: - file.write(params[i].split("--")[1] + "\n") - else: - file.write(params[i].split("--")[0] + "\n") - file.close() - - -def write_samples_getdist_format(root_dir, root, chain_type="configuration"): - assert chain_type in ["configuration", "harmonic"], ( - "chain_type must be 'configuration' or 'harmonic'" - ) - - if chain_type == "configuration": - path_samples = root_dir + "{}/{}/samples_{}.txt".format("/" + root, root, root) - path_gd_samples = root_dir + "{}/{}/getdist_{}.txt".format( - "/" + root, root, root - ) - path_gd = root_dir + "{}/{}/getdist_{}".format(root, root, root) - else: - path_samples = root_dir + "{}/{}/samples_{}_cell.txt".format( - "/" + root, root, root - ) - path_gd_samples = root_dir + "{}/{}/getdist_{}_cell.txt".format( - "/" + root, root, root - ) - path_gd = root_dir + "{}/{}/getdist_{}_cell".format(root, root, root) - - samples = np.loadtxt( - path_samples, - ) - if "nautilus" in root: - samples = np.column_stack( - (np.exp(samples[:, -3]), samples[:, -1] - samples[:, -2], samples[:, 0:-3]) - ) - else: - samples = np.column_stack((samples[:, -1], samples[:, -2], samples[:, 0:-4])) - np.savetxt(path_gd_samples, samples) - - chain = g.samples_for_root( - path_gd, - cache=False, - settings={"ignore_rows": 0.0, "smooth_scale_2D": 0.5, "smooth_scale_1D": 0.5}, - ) - - return chain - - -def extract_param_chain(chain, param_names): - margestats = chain.getMargeStats() - likestats = chain.getLikeStats() - - param_values = {} - for param_name in param_names: - if param_name not in chain.getParamNames().list(): - raise ValueError(f"Parameter {param_name} not found in chain.") - - param_stats = margestats.parWithName(param_name) - param_values[param_name] = { - "mean": param_stats.mean, - "1sigma_minus": param_stats.mean - param_stats.limits[0].lower, - "1sigma_plus": param_stats.limits[0].upper - param_stats.mean, - "2sigma_minus": param_stats.mean - param_stats.limits[1].lower, - "2sigma_plus": param_stats.limits[1].upper - param_stats.mean, - } - - param_stats = likestats.parWithName(param_name) - param_names_getdist = chain.getParamNames() - par = param_names_getdist.parWithName(param_name) - kde = chain.get1DDensity(par, num_bins=1000) - kde_map = kde.x[np.argmax(kde.P)] - param_values[param_name].update( - { - "MAP": kde_map, - } - ) - - par = chain.getParamNames().parWithName("S_8") - par_om = chain.getParamNames().parWithName("OMEGA_M") - kde = chain.get2DDensity(par, par_om, fine_bins_2D=1000) - s8_kde_map = kde.x[np.unravel_index(np.argmax(kde.P), kde.P.shape)[1]] - om_kde_map = kde.y[np.unravel_index(np.argmax(kde.P), kde.P.shape)[0]] - param_values["S_8"].update( - { - "MAP_2D": s8_kde_map, - } - ) - param_values["OMEGA_M"].update( - { - "MAP_2D": om_kde_map, - } - ) - - return param_values - - -def concatenate_param_stats(name, param_values, verbose=False): - output = [name] - for key in param_values.keys(): - param_stat = param_values[key] - if verbose: - print( - f"{name} - {key}: {param_stat['mean']:.4f} +{param_stat['1sigma_plus']:.4f}/-{param_stat['1sigma_minus']:.4f} (1σ), +{param_stat['2sigma_plus']:.4f}/-{param_stat['2sigma_minus']:.4f} (2σ)" - ) - - param_list = [ - param_stat["mean"], - param_stat["1sigma_minus"], - param_stat["1sigma_plus"], - param_stat["2sigma_minus"], - param_stat["2sigma_plus"], - param_stat["MAP"], - ] - - if key == "S_8": - param_list.append(param_stat["MAP_2D"]) - - if key == "OMEGA_M": - param_list.append(param_stat["MAP_2D"]) - - output += param_list - - return output - - -def merge_param_stats(params_configuration, params_harmonic): - merged_params = {} - for key in params_configuration.keys(): - if key in params_harmonic: - merged_params[key] = { - "configuration": params_configuration[key], - "harmonic": params_harmonic[key], - } - return merged_params - - -def concatenate_merge_params(name, merged_params, verbose=False): - output = [name] - for key in merged_params.keys(): - param_config = merged_params[key]["configuration"] - param_harm = merged_params[key]["harmonic"] - - if verbose: - print( - f"{name} - {key} (Configuration): {param_config['mean']:.4f} +{param_config['1sigma_plus']:.4f}/-{param_config['1sigma_minus']:.4f} (1σ), +{param_config['2sigma_plus']:.4f}/-{param_config['2sigma_minus']:.4f} (2σ)" - ) - print( - f"{name} - {key} (Harmonic): {param_harm['mean']:.4f} +{param_harm['1sigma_plus']:.4f}/-{param_harm['1sigma_minus']:.4f} (1σ), +{param_harm['2sigma_plus']:.4f}/-{param_harm['2sigma_minus']:.4f} (2σ)" - ) - - param_list = [ - param_config["mean"], - param_config["1sigma_minus"], - param_config["1sigma_plus"], - param_config["2sigma_minus"], - param_config["2sigma_plus"], - param_config["MAP"], - param_harm["mean"], - param_harm["1sigma_minus"], - param_harm["1sigma_plus"], - param_harm["2sigma_minus"], - param_harm["2sigma_plus"], - param_harm["MAP"], - ] - - output += param_list - - return output - - -chain_harmonic = [] -chain_config = [] - -for i, root in enumerate(tqdm(roots)): - if os.path.isfile(f"{root_dir}/{root}/{root}/getdist_{root}.txt"): - # Load samples and write paramnames for harmonic space - load_samples_and_write_paramames(root_dir, root, chain_type="harmonic") - write_samples_getdist_format(root_dir, root, chain_type="harmonic") - chain_harm = g.samples_for_root( - root_dir + f"/{root}/{root}/getdist_{root}_cell", - cache=False, - settings={ - "ignore_rows": 0.0, - "smooth_scale_2D": 0.5, - "smooth_scale_1D": 0.5, - }, - ) - chain_harmonic.append(chain_harm) - - # Load samples and write paramnames for harmonic space - load_samples_and_write_paramames(root_dir, root, chain_type="configuration") - write_samples_getdist_format(root_dir, root, chain_type="configuration") - chain_conf = g.samples_for_root( - root_dir + f"/{root}/{root}/getdist_{root}", - cache=False, - settings={ - "ignore_rows": 0.0, - "smooth_scale_2D": 0.5, - "smooth_scale_1D": 0.5, - }, - ) - chain_config.append(chain_conf) -# -param_names = ["S_8", "OMEGA_M", "SIGMA_8", "a"] - -output_mocks_harm = np.array( - [ - "Name", - "S8_mean", - "S8_1sigma_minus", - "S8_1sigma_plus", - "S8_2sigma_minus", - "S8_2sigma_plus", - "S8_MAP", - "S8_MAP_2D", - "OMEGA_M_mean", - "OMEGA_M_1sigma_minus", - "OMEGA_M_1sigma_plus", - "OMEGA_M_2sigma_minus", - "OMEGA_M_2sigma_plus", - "OMEGA_M_MAP", - "OMEGA_M_MAP_2D", - "SIGMA_8_mean", - "SIGMA_8_1sigma_minus", - "SIGMA_8_1sigma_plus", - "SIGMA_8_2sigma_minus", - "SIGMA_8_2sigma_plus", - "SIGMA_8_MAP", - "a_mean", - "a_1sigma_minus", - "a_1sigma_plus", - "a_2sigma_minus", - "a_2sigma_plus", - "a_MAP", - ] -) - -output_mocks_config = np.array( - [ - "Name", - "S8_mean", - "S8_1sigma_minus", - "S8_1sigma_plus", - "S8_2sigma_minus", - "S8_2sigma_plus", - "S8_MAP", - "S8_MAP_2D", - "OMEGA_M_mean", - "OMEGA_M_1sigma_minus", - "OMEGA_M_1sigma_plus", - "OMEGA_M_2sigma_minus", - "OMEGA_M_2sigma_plus", - "OMEGA_M_MAP", - "OMEGA_M_MAP_2D", - "SIGMA_8_mean", - "SIGMA_8_1sigma_minus", - "SIGMA_8_1sigma_plus", - "SIGMA_8_2sigma_minus", - "SIGMA_8_2sigma_plus", - "SIGMA_8_MAP", - "a_mean", - "a_1sigma_minus", - "a_1sigma_plus", - "a_2sigma_minus", - "a_2sigma_plus", - "a_MAP", - ] -) - -for i, root in enumerate(tqdm(roots[:-1])): - param_values_harm = extract_param_chain(chain_harmonic[i], param_names) - - param_harm = concatenate_param_stats(root, param_values_harm, verbose=False) - - output_mocks_harm = np.vstack((output_mocks_harm, param_harm)) - - param_values_config = extract_param_chain(chain_config[i], param_names) - - param_config = concatenate_param_stats(root, param_values_config, verbose=False) - - output_mocks_config = np.vstack((output_mocks_config, param_config)) - -np.savetxt( - f"summary_parameter_constraints_harmonic_space_{chain_version}.txt", - output_mocks_harm, - fmt="%s", - delimiter=";", -) -np.savetxt( - f"summary_parameter_constraints_configuration_space_{chain_version}.txt", - output_mocks_config, - fmt="%s", - delimiter=";", -) -print( - f"Saved summary of parameter constraints for harmonic space in summary_parameter_constraints_harmonic_space_{chain_version}.txt" -) -print( - f"Saved summary of parameter constraints for configuration space in summary_parameter_constraints_configuration_space_{chain_version}.txt" -) - - -import pandas as pd - -output_df_harm = pd.read_csv( - f"summary_parameter_constraints_harmonic_space_{chain_version}.txt", - delimiter=";", - skiprows=1, - names=output_mocks_harm[0], -) - -output_df_config = pd.read_csv( - f"summary_parameter_constraints_configuration_space_{chain_version}.txt", - delimiter=";", - skiprows=1, - names=output_mocks_config[0], -) - - -# Define the true value of the parameters -from astropy.cosmology import Planck18 as planck - -Omega_m_fid = planck.Om0 -sigma_8_fid = 0.8102 -s8_fid = sigma_8_fid * (Omega_m_fid / 0.3) ** 0.5 -h = planck.h -Omega_b_fig = planck.Ob0 -n_s_fid = 0.9665 -print( - f"Fiducial values: Omega_m = {Omega_m_fid}, sigma_8 = {sigma_8_fid}, S_8 = {s8_fid}" -) - - -sns.histplot( - output_df_harm["S8_mean"] - output_df_config["S8_mean"], - kde=True, - bins=30, - label="Mean", -) -# sns.histplot( -# output_df_harm["S8_MAP"]-output_df_config["S8_MAP"], -# kde=True, -# bins=20, -# label="MAP", -# ) -sns.histplot( - output_df_harm["S8_MAP_2D"] - output_df_config["S8_MAP_2D"], - kde=True, - bins=30, - label="2D Mode", - alpha=0.5, -) -plt.axvline(0, color="black", linestyle="--") -plt.legend(fontsize=12) - -plt.xlabel(r"$\Delta S_8$") -plt.savefig( - "./../../results/S8_comparison_harmonic_vs_configuration.pdf", - bbox_inches="tight", -) - - -output_df_config["S8_MAP_2D"].shape -output_df_harm["S8_MAP_2D"].shape - - -# Create JointGrid -g = sns.JointGrid( - x=output_df_config["OMEGA_M_MAP_2D"], - y=output_df_config["S8_MAP_2D"], - height=7, - ratio=5, - space=0, -) - -# Main 2D histogram -sns.histplot( - x=output_df_config["OMEGA_M_MAP_2D"], - y=output_df_config["S8_MAP_2D"], - bins=25, - cmap="Greens", - cbar=False, - ax=g.ax_joint, -) - -# Marginal histograms -sns.histplot( - x=output_df_config["OMEGA_M_MAP_2D"], bins=25, color="#2ca25f", ax=g.ax_marg_x -) -sns.histplot(y=output_df_config["S8_MAP_2D"], bins=25, color="#2ca25f", ax=g.ax_marg_y) - -# Add dashed reference lines -g.ax_joint.axvline(Omega_m_fid, color="k", linestyle="--") -g.ax_joint.axhline(s8_fid, color="k", linestyle="--") - -# Labels -g.set_axis_labels( - r"$\Omega_m$ estimated from mocks (Configuration space)", - r"$S_8$ estimated from mocks (Configuration space)", -) - -# Optional styling tweaks -g.ax_joint.tick_params(labelsize=12) -plt.savefig( - "./../../results/S8_vs_OmegaM_configuration_space_mocks.pdf", - bbox_inches="tight", -) diff --git a/papers/realspace/nonlin_k_analysis.py b/papers/realspace/nonlin_k_analysis.py deleted file mode 100644 index a44002a3..00000000 --- a/papers/realspace/nonlin_k_analysis.py +++ /dev/null @@ -1,104 +0,0 @@ -# # Nonlinear $k$ contributions -# -# This notebook plots the 2D heatmap of ratio of scale contributions to the $\xi_\pm$ 2PCF given angular scale $\theta$ and wavenumber $k$. - - -import matplotlib.pylab as plt -import numpy as np -import seaborn as sns - -plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") - -plt.rcParams["text.usetex"] = True - -plt.rcParams.update( - { - "font.size": 20, - "axes.titlesize": 21, - "axes.labelsize": 20, - "xtick.labelsize": 20, - "ytick.labelsize": 20, - "legend.fontsize": 20, - "figure.titlesize": 21, - } -) -sns.set_palette("husl") - -blind = "B" -ver = "v1.4.6.3" - - -data_dir = "/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/data/" - -# Read the 2D array from the text file - -file_headers = ["xip_%s_%s" % (ver, blind), "xim_%s_%s" % (ver, blind)] - -for f in file_headers: - xis = np.loadtxt(data_dir + f"theta_k_{f}.txt") - xis_reshaped = xis.reshape(-1, 201) - sorted_xis = xis_reshaped[np.argsort(xis_reshaped[:, 0])] - - np.savetxt(data_dir + f"theta_k_{f}_sorted.txt", sorted_xis) - - -fig, axs = plt.subplots(2, 1, figsize=(8, 10)) - -# --- k grid --- -h = 0.6766 -k_plot = np.logspace(-4, 2, 200) - -file_header = "%s_%s" % (ver, blind) - -xi_thetas = np.loadtxt(data_dir + f"theta_k_xip_{file_header}_sorted.txt") -thetas = xi_thetas[:, 0] -xis = xi_thetas[:, 1:] - -# normalise -xi_plot = xis / np.max(xis, axis=1, keepdims=True) - -T, K = np.meshgrid(thetas, k_plot) - -axs[0].contour(T, K, xi_plot.T, levels=[0.9], colors="red", linewidths=1.7) -pcm = axs[0].pcolormesh(T, K, xi_plot.T, shading="auto", cmap="viridis") -pcm.set_rasterized(True) - -axs[0].axvline(5, color="k", ls="dashed", lw=1.2) -axs[0].axvline(12, color="white", ls="dashed", lw=1.6) -axs[0].axhline(1, color="k", ls="dashed", lw=1.2) # converted to h/Mpc space if needed -axs[0].axhline(0.425, color="white", ls="dashed", lw=1.6) - -axs[0].set_yscale("log") -axs[0].set_xlabel(r"$\theta\ \mathrm{(arcmin)}$") -axs[0].set_ylabel(r"$k\ (h$ Mpc$^{-1})$") - -axs[0].set_title(r"$\xi_+$") - -xi_thetas = np.loadtxt(data_dir + f"theta_k_xim_{file_header}_sorted.txt") -thetas = xi_thetas[:, 0] -xis = xi_thetas[:, 1:] - -xi_plot = xis / np.max(xis, axis=1, keepdims=True) - -T, K = np.meshgrid(thetas, k_plot) - -axs[1].contour(T, K, xi_plot.T, levels=[0.9], colors="red", linewidths=1.7) -pcm = axs[1].pcolormesh(T, K, xi_plot.T, shading="nearest", cmap="viridis") -pcm.set_rasterized(True) - -axs[1].axvline(12, color="white", ls="dashed", lw=1.6) -axs[1].axhline(2.85, color="white", ls="dashed", lw=1.6) - - -axs[1].set_yscale("log") -axs[1].set_xlabel(r"$\theta\ \mathrm{(arcmin)}$") -axs[1].set_ylabel(r"$k\ (h$ Mpc$^{-1})$") -axs[1].set_title(r"$\xi_-$") - - -fig.tight_layout() - -cbar_ax = fig.add_axes([0.99, 0.15, 0.02, 0.7]) -cbar = fig.colorbar(pcm, cax=cbar_ax) - -fig.savefig("./../../results/theta_k_xip_xim_{ver}_{blind}.pdf", bbox_inches="tight") diff --git a/pyproject.toml b/pyproject.toml index c6a68dd0..ecdec6f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,9 +9,9 @@ authors = [ ] license = {text = "MIT"} readme = "README.md" -# 3.12 floor: cosmo-numba (a hard dependency below) requires >=3.12, and the -# production container is Python 3.12 (shapepipe base image). Keeping the floor -# in sync with the container is what makes `uv lock` resolvable. +# 3.12 floor set by Smokescreen 1.5.6 (and firecrown v1.15); the container base +# (shapepipe:develop) is already python:3.12-slim-bookworm, so this aligns +# pyproject with the actual runtime. requires-python = ">=3.12" classifiers = [ "License :: OSI Approved :: MIT License", @@ -28,38 +28,13 @@ dependencies = [ "camb>=1.6", "clmm", "colorama", - # Blinding closure (core, no extra): cryptography + sacc here, plus the - # Smokescreen fork pin below; pyccl (the theory backend) is already core. - # cryptography and sacc are declared explicitly so the core runtime - # closure is self-documenting and independent of fork-metadata drift. - "cryptography", # Track cs_util's develop branch directly (git dependency) rather than a # PyPI pin: the two repos are iterating together heavily and cs_util # releases are infrequent. This PR's cosmology repoint needs get_cosmo / # get_theo_c_ell / get_theo_xi / PLANCK18 in cs_util.cosmo, which land via # CosmoStat/cs_util#76 — so this goes green once #76 merges into develop. "cs_util @ git+https://github.com/CosmoStat/cs_util.git@develop", - # Fast numba B-mode kernels (Schneider et al. 2022): the Schneider E/B split - # and COSEBIS live here, imported in b_modes.py. Tracks aguinot/cosmo-numba - # main (not published on PyPI). main carries the numpy-2 FFT fix via its - # rocket-fft dependency (which teaches numba's nopython mode to handle - # np.fft), and declares numba/numpy/rocket-fft from its requirements.txt so - # those constraints reach the resolver. - "cosmo-numba @ git+https://github.com/aguinot/cosmo-numba.git@main", "emcee", - # numba is the load-bearing pin of this whole environment: its numpy ceiling - # (numba 0.66 -> numpy<2.5) is what keeps the resolver from drifting numpy - # forward and breaking numba/ngmix — the failure this lockfile exists to - # prevent. cosmo-numba above also carries this constraint, but we pin numba - # ourselves too: it makes the one critical constraint visible and resilient - # to cosmo-numba's dependency metadata (which has silently emptied out - # between refs before). We pin numba, never numpy directly. - "numba", - # Imported directly across src/ (calibration, plots) alongside seaborn — - # declared explicitly rather than leaned on as a seaborn transitive. - "matplotlib", - "pandas", - "pyyaml", # SHA-pinned snapshot of getdist branch `upper_triangle_whisker`. "getdist @ git+https://github.com/benabed/getdist.git@113cd22a9a0d013b6f72fe734be81f260f3d3be5", "h5py", @@ -72,6 +47,7 @@ dependencies = [ "jupytext>=1.15", "lenspack", "lmfit", + "numexpr", "numpy>=2.0", "opencv-python-headless", "pyccl", @@ -79,7 +55,10 @@ dependencies = [ "pymaster", "regions", "reproject", - "sacc>=0.12", + # SACC (LSST DESC's data-vector container) is the standard format for all + # data products from the tomographic round on (PRD #241); sp_validation.sacc_io + # is core library code, so sacc is a core dependency. + "sacc>=2.4,<3", # scipy 1.18 ported FITPACK from Fortran to C, changing the return shape of # RectBivariateSpline(scalar, scalar, grid=False) from 0-d `array(x)` to # shape-(1,) `array([x])`. camb's BBN Y_He predictor (bbn.py) wraps the @@ -97,12 +76,6 @@ dependencies = [ # getdist feature-branch below, which is an external fork we pin for repro.) "shear_psf_leakage @ git+https://github.com/CosmoStat/shear_psf_leakage.git@develop", "skyproj", - # UNIONS-WL fork of DESC Smokescreen, pinned by SHA on the fork's - # packaging branch: it declares pyccl and imports its theory backends - # lazily, so the install closure is CCL-only. Provisional pin — swapped to the - # fork's release tag once the fork packaging PRs merge. Git pin only; - # nothing is published to PyPI. - "smokescreen @ git+https://github.com/UNIONS-WL/Smokescreen@588a6b9b26560bd5ba3dd5ba342f3c40152644f9", "statsmodels", "treecorr>=5.0", "tqdm", @@ -112,13 +85,6 @@ dependencies = [ [project.urls] Homepage = "https://github.com/CosmoStat/sp_validation" -[tool.uv] -# The reproducibility target is the Linux container; scope the lock to Linux -# (mirrors shapepipe) so `uv lock` resolves the linux-centric stack (pymaster, -# mpi4py) without hunting for macOS/Windows wheels. macOS dev installs still -# work via `uv pip install -e .` (unlocked), just not `uv sync` from the lock. -environments = ["sys_platform == 'linux'"] - [project.optional-dependencies] test = [ "pytest", @@ -146,20 +112,31 @@ glass = [ "glass==2025.1", "glass.ext.camb==2023.6", "cosmology==2022.10.9", - # fitsio: make_unions_glass_sim.py writes the mock catalogue as FITS. - "fitsio", ] -# Cosmo-inference workflow runners (workflow/scripts/*). Kept optional: the core -# library resolves without them, but the container installs this extra so the -# Snakemake workflow and cross-validation runners are available. -workflow = [ - "snakemake", - # run_2pcf_highres.py drives the MPI convergence run; the container ships - # OpenMPI (/opt/ompi) so mpi4py builds against it. - "mpi4py", - # NOTE: workflow/scripts/cv_*.py also import `cv_runner`, which is not - # published or resolvable (no public repo found) — left undeclared pending - # its source. Same for `unions_wl` (scripts/check_footprint.py). +# Data-vector blinding (PRD #241 §3-§5): Smokescreen applies the Muir et al. +# shift d → d + t(hidden) − t(fid), with firecrown + CCL as the theory engine +# (only compute_theory_vector is used; sampling stays with CosmoSIS). The blind +# must be exactly recomputable from the seed at unblinding time, so the whole +# theory stack is pinned exactly, as a set. Smokescreen 1.5.6 + firecrown v1.15 +# both set the python floor (>=3.12). +# +# firecrown is not on PyPI and declares conda-forge-only / unused sampler +# connectors as hard deps, so installing this extra requires the dependency +# override file: `uv pip install --overrides uv-overrides.txt -e '.[blinding]'` +# (see uv-overrides.txt; the Dockerfile does this for the container). +# After installing this extra, run `python scripts/patch_firecrown.py` — it +# makes pip-installed firecrown importable without NumCosmo (conda-forge-only); +# see that script's docstring for the full story. +blinding = [ + "firecrown @ git+https://github.com/LSSTDESC/firecrown.git@v1.15.1", + "smokescreen==1.5.6", + "pyccl==3.3.4", + # firecrown 1.15.1 subclasses npt.NDArray (DataVector); numpy 2.5 turned + # npt.NDArray into a non-subclassable typing alias, breaking firecrown at + # import. firecrown's own env caps numpy<2.4; 2.4.3 is verified against + # the full compiled stack (pyccl/camb/treecorr/healpy/pymaster) + the + # sp_validation fast suite. + "numpy>=2.2,<2.5", ] develop = ["sp_validation[test,docs]"] @@ -170,14 +147,7 @@ addopts = [ "--cov=sp_validation", "--cov-report=term", "--cov-report=xml", - "--junitxml=pytest.xml", - # The base ShapePipe image's test extra ships old pytest-pydocstyle / - # pytest-pycodestyle, whose pytest_collect_file hooks use the `path` arg the - # newer pytest our lock installs has removed — they crash collection. - # sp_validation lints with ruff, not these, so don't load them. Harmless if - # absent (`-p no:` just skips an unregistered plugin). - "-p", "no:pydocstyle", - "-p", "no:pycodestyle" + "--junitxml=pytest.xml" ] markers = [ "fast: marks tests as fast (deselect with '-m \"not fast\"')", diff --git a/scripts/calibration/calibrate_comprehensive_cat.py b/scripts/calibration/calibrate_comprehensive_cat.py index 4928df9b..4b15ed89 100644 --- a/scripts/calibration/calibrate_comprehensive_cat.py +++ b/scripts/calibration/calibrate_comprehensive_cat.py @@ -96,13 +96,8 @@ ) # %% -additive_correction = cm.get("additive_correction", True) -if not additive_correction: - print("Additive bias correction disabled (additive_correction: False)") - g_corr_mc, g_uncorr, w, mask_metacal, c, c_err = calibration.get_calibrated_m_c( - gal_metacal, - additive_correction=additive_correction, + gal_metacal ) num_ok = len(g_corr_mc[0]) diff --git a/scripts/calibration/extract_info.py b/scripts/calibration/extract_info.py index 123c7437..1cbf836f 100644 --- a/scripts/calibration/extract_info.py +++ b/scripts/calibration/extract_info.py @@ -32,7 +32,6 @@ import os import sys -import h5py import numpy as np from astropy.io import fits @@ -52,11 +51,8 @@ # ### Create and open output files and directories -os.makedirs(output_dir, exist_ok=True) -stats_file = open_stats_file(output_dir, stats_file_name) - -output_shape_cat_stem = output_shape_cat_base -output_ext = output_format +make_out_dirs(output_dir, plot_dir, [], verbose=verbose) +stats_file = open_stats_file(plot_dir, stats_file_name) # ## 2. Load data # @@ -111,8 +107,6 @@ tile_IDs, path_tile_ID, path_found_ID, path_missing_ID, verbose=verbose ) -print_stats(f"Tiles in input catalogue: {n_found}", stats_file, verbose=verbose) - # ### Load star catalogue if star_cat_path: @@ -152,40 +146,42 @@ verbose=verbose, ) - # Flags to indicate valid star sample. - # Star matching, PSF-catalogue output and star metacalibration are all - # diagnostics that require an input star catalogue; the image-simulation - # pipeline has none (star_cat_path is None), so every star-dependent block - # below is guarded and simply skipped for the sims. - - m_star = ( - (dd["FLAGS"][ind_star] == 0) - & (dd["IMAFLAGS_ISO"][ind_star] == 0) - & (dd["NGMIX_MCAL_FLAGS"][ind_star] == 0) - & (dd["NGMIX_G1_PSF_ORIG_NOSHEAR"][ind_star] != -10) - ) +# #### Refine: Match to valid, unflagged galaxy sample - ra_star, dec_star, g_star_psf = spv_cat.match_subsample( - dd, - ind_star, - m_star, - [col_name_ra, col_name_dec], - key_PSF_g1, - key_PSF_g2, - n_star_tot, - stats_file, - verbose=verbose, - ) +# + +# Flags to indicate valid star sample - # ### Write PSF catalogue with multi-epoch shapes from shape measurement methods +m_star = ( + (dd["FLAGS"][ind_star] == 0) + & (dd["IMAFLAGS_ISO"][ind_star] == 0) + & (dd["NGMIX_MCAL_FLAGS"][ind_star] == 0) + & (dd["NGMIX_G1_PSF_ORIG_NOSHEAR"][ind_star] != -10) +) - spv_cat.write_PSF_cat( - f"{output_PSF_cat_base}_{shape}.fits", - ra_star, - dec_star, - g_star_psf[0], - g_star_psf[1], - ) +ra_star, dec_star, g_star_psf = spv_cat.match_subsample( + dd, + ind_star, + m_star, + [col_name_ra, col_name_dec], + key_PSF_g1, + key_PSF_g2, + n_star_tot, + stats_file, + verbose=verbose, +) +# - + +# MKDEBUG: Moved from end of this script + +# ### Write PSF catalogue with multi-epoch shapes from shape measurement methods + +spv_cat.write_PSF_cat( + f"{output_PSF_cat_base}_{shape}.fits", + ra_star, + dec_star, + g_star_psf[0], + g_star_psf[1], +) # ## Check for objects with invalid PSF @@ -257,9 +253,8 @@ if verbose: print("Writing comprehensive catalogue...") -comprehensive_cat_path = f"{output_shape_cat_stem}_comprehensive_{shape}{output_ext}" spv_cat.write_shape_catalog( - comprehensive_cat_path, + f"{output_shape_cat_base}_comprehensive_{shape}.fits", ra_all, dec_all, iv_w, @@ -270,17 +265,6 @@ add_cols=ext_cols_pre_cal, add_cols_format=add_cols_pre_cal_format, ) - -# Write tile count to HDF5 attributes -if output_ext == ".hdf5": - try: - with h5py.File(comprehensive_cat_path, "a") as hf: - hf.attrs["n_tiles"] = n_found - if verbose: - print(f" Added n_tiles={n_found} to HDF5 attributes") - except Exception as e: - if verbose: - print(f" Warning: could not add n_tiles attribute: {e}") # - do_selection_calibration = False @@ -291,7 +275,6 @@ else: if verbose: print("Continuing with selection and calibration") - os.makedirs(os.path.join(output_dir, plot_dir), exist_ok=True) # ## 4. Select galaxies @@ -645,24 +628,23 @@ # ## Metacalibration for stars -if star_cat_path: - star_metacal = metacal(dd[ind_star], m_star, masking_type="star", verbose=verbose) +star_metacal = metacal(dd[ind_star], m_star, masking_type="star", verbose=verbose) - # #### Number density +# #### Number density - # + - # mask for 'no shear' images +# + +# mask for 'no shear' images - mask_ns_stars = star_metacal.mask_dict["ns"] - n_star = len(star_metacal.ns["g1"][mask_ns_stars]) +mask_ns_stars = star_metacal.mask_dict["ns"] +n_star = len(star_metacal.ns["g1"][mask_ns_stars]) - print_stats(f"Number of stars = {n_star}", stats_file, verbose=verbose) - print_stats( - "Star density = {:.2f} stars/deg2".format(n_star / area_deg2), - stats_file, - verbose=verbose, - ) - # - +print_stats(f"Number of stars = {n_star}", stats_file, verbose=verbose) +print_stats( + "Star density = {:.2f} stars/deg2".format(n_star / area_deg2), + stats_file, + verbose=verbose, +) +# - # ## Additive bias # Use raw, uncorrected ellipticities. @@ -748,21 +730,20 @@ print_stats(rs, stats_file, verbose=verbose) # + -if star_cat_path: - print_stats("stars:", stats_file, verbose=verbose) +print_stats("stars:", stats_file, verbose=verbose) - print_stats("total response matrix:", stats_file, verbose=verbose) - rs = np.array2string(star_metacal.R) - print_stats(rs, stats_file, verbose=verbose) +print_stats("total response matrix:", stats_file, verbose=verbose) +rs = np.array2string(star_metacal.R) +print_stats(rs, stats_file, verbose=verbose) - print_stats("shear response matrix:", stats_file, verbose=verbose) - R_shear_stars = np.mean(star_metacal.R_shear, 2) - rs = np.array2string(R_shear_stars) - print_stats(rs, stats_file, verbose=verbose) +print_stats("shear response matrix:", stats_file, verbose=verbose) +R_shear_stars = np.mean(star_metacal.R_shear, 2) +rs = np.array2string(R_shear_stars) +print_stats(rs, stats_file, verbose=verbose) - print_stats("selection response matrix:", stats_file, verbose=verbose) - rs = np.array2string(star_metacal.R_selection) - print_stats(rs, stats_file, verbose=verbose) +print_stats("selection response matrix:", stats_file, verbose=verbose) +rs = np.array2string(star_metacal.R_selection) +print_stats(rs, stats_file, verbose=verbose) # - # ### Plot distribution of response matrix elements @@ -776,11 +757,14 @@ linestyles = ["-", "-", ":", ":"] # + -labels = ["$R_{11}$ galaxies", "$R_{22}$ galaxies"] -xs = [gal_metacal.R_shear[0, 0], gal_metacal.R_shear[1, 1]] -if star_cat_path: - labels += ["$R_{11}$ stars", "$R_{22}$ stars"] - xs += [star_metacal.R_shear[0, 0], star_metacal.R_shear[1, 1]] +labels = ["$R_{11}$ galaxies", "$R_{22}$ galaxies", "$R_{11}$ stars", "$R_{22}$ stars"] + +xs = [ + gal_metacal.R_shear[0, 0], + gal_metacal.R_shear[1, 1], + star_metacal.R_shear[0, 0], + star_metacal.R_shear[1, 1], +] title = shape out_name = f"R_{shape}_diag.pdf" @@ -795,16 +779,19 @@ x_range, n_bin, out_path, - colors=colors[: len(xs)], - linestyles=linestyles[: len(xs)], + colors=colors, + linestyles=linestyles, ) # + -labels = ["$R_{12}$ galaxies", "$R_{21}$ galaxies"] -xs = [gal_metacal.R_shear[0, 1], gal_metacal.R_shear[1, 0]] -if star_cat_path: - labels += ["$R_{12}$ stars", "$R_{21}$ stars"] - xs += [star_metacal.R_shear[0, 1], star_metacal.R_shear[1, 0]] +labels = ["$R_{12}$ galaxies", "$R_{21}$ galaxies", "$R_{12}$ stars", "$R_{21}$ stars"] + +xs = [ + gal_metacal.R_shear[0, 1], + gal_metacal.R_shear[1, 0], + star_metacal.R_shear[0, 1], + star_metacal.R_shear[1, 0], +] title = shape out_name = f"R_{shape}_offdiag.pdf" out_path = os.path.join(plot_dir, out_name) @@ -818,8 +805,8 @@ x_range, n_bin, out_path, - colors=colors[: len(xs)], - linestyles=linestyles[: len(xs)], + colors=colors, + linestyles=linestyles, ) # - @@ -858,50 +845,49 @@ ) # + -if star_cat_path: - xs = [star_metacal.ns["g1"][mask_ns_stars], star_metacal.ns["g2"][mask_ns_stars]] - weights = [star_metacal.ns["w"][mask_ns_stars]] * 2 - - title = "stars" - out_name = f"ell_stars_{shape}.pdf" - out_path = os.path.join(plot_dir, out_name) - - plot_histograms( - xs, - labels, - title, - x_label, - y_label, - x_range, - n_bin, - out_path, - weights=weights, - colors=colors, - linestyles=linestyles, - ) - # - - - x_range = (-0.15, 0.15) - n_bin = 250 - - # + - xs = [dd[key_PSF_g1][mask_ns_stars], dd[key_PSF_g2][mask_ns_stars]] - title = "PSF" - out_name = f"ell_PSF_{shape}.pdf" - out_path = os.path.join(plot_dir, out_name) - - plot_histograms( - xs, - labels, - title, - x_label, - y_label, - x_range, - n_bin, - out_path, - colors=colors, - linestyles=linestyles, - ) +xs = [star_metacal.ns["g1"][mask_ns_stars], star_metacal.ns["g2"][mask_ns_stars]] +weights = [star_metacal.ns["w"][mask_ns_stars]] * 2 + +title = "stars" +out_name = f"ell_stars_{shape}.pdf" +out_path = os.path.join(plot_dir, out_name) + +plot_histograms( + xs, + labels, + title, + x_label, + y_label, + x_range, + n_bin, + out_path, + weights=weights, + colors=colors, + linestyles=linestyles, +) +# - + +x_range = (-0.15, 0.15) +n_bin = 250 + +# + +xs = [dd[key_PSF_g1][mask_ns_stars], dd[key_PSF_g2][mask_ns_stars]] +title = "PSF" +out_name = f"ell_PSF_{shape}.pdf" +out_path = os.path.join(plot_dir, out_name) + +plot_histograms( + xs, + labels, + title, + x_label, + y_label, + x_range, + n_bin, + out_path, + colors=colors, + linestyles=linestyles, +) # - # ## Magnitudes @@ -965,7 +951,7 @@ # ### Write basic shape catalogue spv_cat.write_shape_catalog( - f"{output_shape_cat_stem}_{shape}{output_ext}", + f"{output_shape_cat_base}_{shape}.fits", ra, dec, w, @@ -1004,7 +990,7 @@ # Extended catalogue with SNR, individual R matrices, ext_cols spv_cat.write_shape_catalog( - f"{output_shape_cat_stem}_extended_{shape}{output_ext}", + f"{output_shape_cat_base}_extended_{shape}.fits", ra, dec, w, @@ -1034,4 +1020,4 @@ ra = dd["RA"][cut_overlap] dec = dd["DEC"][cut_overlap] tile_id = dd["TILE_ID"][cut_overlap] - write_galaxy_cat(f"{output_shape_cat_stem}{output_ext}", ra, dec, tile_id) + write_galaxy_cat(f"{output_shape_cat_base}.fits", ra, dec, tile_id) diff --git a/scripts/calibration/params.py b/scripts/calibration/params.py index 5b2ef0c9..2d0bd45f 100644 --- a/scripts/calibration/params.py +++ b/scripts/calibration/params.py @@ -105,9 +105,6 @@ ## Output -### Output file format extension: '.fits' or '.hdf5' -output_format = ".hdf5" - ### Additional output columns add_cols = [ "FLUX_RADIUS", diff --git a/scripts/compute_m_bias_image_sims.py b/scripts/compute_m_bias_image_sims.py deleted file mode 100644 index 04ca49b7..00000000 --- a/scripts/compute_m_bias_image_sims.py +++ /dev/null @@ -1,361 +0,0 @@ -#!/usr/bin/env python -"""Compute multiplicative and additive shear bias from image simulations. - -Usage: - compute_m_bias_image_sims.py -c config.yaml [-v] [--cumulative] [--n_tiles N] -""" - -import argparse -import os -import sys - -# Configure matplotlib for non-interactive backend -import matplotlib -import numpy as np -import yaml - -matplotlib.use("Agg") - -import matplotlib.pyplot as plt - -from sp_validation.image_sims import ImageSimMBias - - -def to_python(obj): - """Recursively cast numpy scalars/arrays to plain Python types. - - ``ImageSimMBias.run`` returns a nested dict of numpy floats; dumping those - to YAML with ``yaml.dump`` writes opaque ``!!python/object`` binary tags. A - recursive pass down the results tree (dicts, lists, arrays, scalars) leaves - a clean, human-readable, ``safe_load``-able document. - """ - if isinstance(obj, dict): - return {key: to_python(val) for key, val in obj.items()} - if isinstance(obj, (list, tuple)): - return [to_python(val) for val in obj] - if isinstance(obj, np.ndarray): - return float(obj.item()) if obj.size == 1 else obj.tolist() - if isinstance(obj, (np.integer, np.floating)): - return float(obj) - return obj - - -def parse_args(): - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("-c", "--config", required=True, help="config YAML file") - p.add_argument("-v", "--verbose", action="store_true", help="verbose output") - p.add_argument( - "--cumulative", - action="store_true", - default=True, - help="track convergence as tiles accumulate (default: True)", - ) - p.add_argument( - "--n_tiles", type=int, help="number of tiles (auto-detected if not given)" - ) - return p.parse_args() - - -def get_n_tiles(grids_dir, num): - """Detect number of tiles from final_cat HDF5 files.""" - try: - import h5py - - # Count tiles in first sim's final_cat - for sim in ["1z2z_grid", "1m2z_grid", "1p2z_grid", "1z2m_grid", "1z2p_grid"]: - sim_name = f"{sim}_{num}" - final_cat = os.path.join(grids_dir, sim_name, f"final_cat_{sim_name}.hdf5") - if os.path.isfile(final_cat): - with h5py.File(final_cat, "r") as hf: - if "patches" in hf: - n_tiles = sum( - 1 for patch in hf["patches"] for _ in hf[f"patches/{patch}"] - ) - return n_tiles - except Exception: - pass - return None - - -def update_cumulative_file(cumulative_path, n_tiles, results): - """Update the cumulative m/c bias tracking file. - - Writes ``results`` under the ``n_tiles`` key, *overwriting* an existing - entry for that count. The earlier behaviour silently skipped when the key - was already present, which meant a re-run against fixed catalogues left the - old (possibly wrong) number in place -- a stale value masquerading as - current. A fresh run is the authority for its tile count, so it overwrites. - - Returns ``True`` when a new key was added, ``False`` when an existing entry - was overwritten (the file is written either way). - """ - if os.path.isfile(cumulative_path): - with open(cumulative_path) as f: - try: - cumulative = yaml.safe_load(f) or {} - except yaml.YAMLError: - # Legacy file written before the to_python cleanup: it carries - # numpy python-object tags that safe_load rejects. Load it - # unsafely, then the to_python pass on write heals it in place. - f.seek(0) - cumulative = yaml.unsafe_load(f) or {} - else: - cumulative = {} - - is_new = str(n_tiles) not in cumulative - cumulative[str(n_tiles)] = results - with open(cumulative_path, "w") as f: - yaml.dump(to_python(cumulative), f, default_flow_style=False) - return is_new - - -def plot_convergence(cumulative_path, diagnostics_dir): - """Create convergence plots: m/c vs n_tiles and errors vs n_tiles.""" - os.makedirs(diagnostics_dir, exist_ok=True) - - try: - with open(cumulative_path) as f: - cumulative = yaml.safe_load(f) - except Exception as e: - print(f"Warning: could not read cumulative file {cumulative_path}: {e}") - return - - if not cumulative: - print("No cumulative data yet, skipping plots") - return - - # Sort by n_tiles - n_tiles_list = sorted([int(k) for k in cumulative.keys()]) - m1_vals = [] - m1_err_vals = [] - c1_vals = [] - c1_err_vals = [] - m2_vals = [] - m2_err_vals = [] - c2_vals = [] - c2_err_vals = [] - - for n in n_tiles_list: - res = cumulative[str(n)] - m1_vals.append(res["m1"]) - m1_err_vals.append(res["m1_err"]) - c1_vals.append(res["c1"]) - c1_err_vals.append(res["c1_err"]) - m2_vals.append(res["m2"]) - m2_err_vals.append(res["m2_err"]) - c2_vals.append(res["c2"]) - c2_err_vals.append(res["c2_err"]) - - n_tiles_str = ( - f"n_tiles = {n_tiles_list}" - if len(n_tiles_list) > 1 - else f"n_tiles = {n_tiles_list[0]}" - ) - - # Plot 1: m and c with error bars - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5)) - fig.suptitle(f"m and c convergence ({n_tiles_str})", fontsize=12) - - ax1.errorbar( - n_tiles_list, m1_vals, yerr=m1_err_vals, fmt="o-", label="m1", capsize=5 - ) - ax1.errorbar( - n_tiles_list, m2_vals, yerr=m2_err_vals, fmt="s-", label="m2", capsize=5 - ) - ax1.axhline(0, color="k", linestyle="--", alpha=0.3) - ax1.set_xlabel("Number of tiles") - ax1.set_ylabel("Multiplicative bias m") - ax1.legend() - ax1.grid(True, alpha=0.3) - - ax2.errorbar( - n_tiles_list, c1_vals, yerr=c1_err_vals, fmt="o-", label="c1", capsize=5 - ) - ax2.errorbar( - n_tiles_list, c2_vals, yerr=c2_err_vals, fmt="s-", label="c2", capsize=5 - ) - ax2.axhline(0, color="k", linestyle="--", alpha=0.3) - ax2.set_xlabel("Number of tiles") - ax2.set_ylabel("Additive bias c") - ax2.legend() - ax2.grid(True, alpha=0.3) - - plt.tight_layout() - plot1_path = os.path.join(diagnostics_dir, "mbias_convergence.png") - plt.savefig(plot1_path, dpi=150) - plt.close() - print(f"Saved convergence plot to {plot1_path}") - - # Plot 2: error bars only - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5)) - fig.suptitle(f"Error convergence ({n_tiles_str})", fontsize=12) - - ax1.errorbar( - n_tiles_list, - [0] * len(n_tiles_list), - yerr=m1_err_vals, - fmt="o-", - label="m1 error", - capsize=5, - alpha=0.7, - ) - ax1.errorbar( - n_tiles_list, - [0] * len(n_tiles_list), - yerr=m2_err_vals, - fmt="s-", - label="m2 error", - capsize=5, - alpha=0.7, - ) - ax1.set_xlabel("Number of tiles") - ax1.set_ylabel("Multiplicative bias error") - ax1.legend() - ax1.grid(True, alpha=0.3) - ax1.set_ylim(bottom=0) - - ax2.errorbar( - n_tiles_list, - [0] * len(n_tiles_list), - yerr=c1_err_vals, - fmt="o-", - label="c1 error", - capsize=5, - alpha=0.7, - ) - ax2.errorbar( - n_tiles_list, - [0] * len(n_tiles_list), - yerr=c2_err_vals, - fmt="s-", - label="c2 error", - capsize=5, - alpha=0.7, - ) - ax2.set_xlabel("Number of tiles") - ax2.set_ylabel("Additive bias error") - ax2.legend() - ax2.grid(True, alpha=0.3) - ax2.set_ylim(bottom=0) - - plt.tight_layout() - plot2_path = os.path.join(diagnostics_dir, "mbias_errors.png") - plt.savefig(plot2_path, dpi=150) - plt.close() - print(f"Saved errors plot to {plot2_path}") - - -def main(): - args = parse_args() - - with open(args.config) as f: - config = yaml.safe_load(f) - - print(f"Config: {args.config}") - print(f"Grids : {config['grids_dir']}") - print(f"Run : grid_{config['num']}") - print(f"g_in : ±{config['shear_amplitude']}") - print() - - # Auto-detect n_tiles if --cumulative - if args.cumulative and not args.n_tiles: - n_tiles = get_n_tiles(config["grids_dir"], config["num"]) - if n_tiles: - args.n_tiles = n_tiles - print(f"Auto-detected {n_tiles} tiles") - - mb = ImageSimMBias(config) - - print("Loading catalogues...") - mb.load_catalogs(verbose=args.verbose) - - # ``run`` returns a document with the primary scheme's m/c mirrored at the - # top level plus a per-scheme ``weights`` block. Cast the whole tree to - # plain Python floats so the YAML/text output is human-readable (raw numpy - # scalars serialise as !!python/object binary). - results = to_python(mb.run(verbose=True)) - - print() - print("=" * 40) - print(" Results") - print("=" * 40) - for scheme, res in results["weights"].items(): - print(f" weights: {scheme}") - print(f" m1 = {res['m1']:+.4f} +-{res['m1_err']:.4f}") - print(f" c1 = {res['c1']:+.4f} +-{res['c1_err']:.4f}") - print(f" m2 = {res['m2']:+.4f} +-{res['m2_err']:.4f}") - print(f" c2 = {res['c2']:+.4f} +-{res['c2_err']:.4f}") - print("=" * 40) - - # Cumulative tracking - if args.cumulative: - results_dir = config.get( - "diagnostics_dir", config.get("results_dir", "results") - ) - os.makedirs(results_dir, exist_ok=True) - else: - results_dir = None - - # Output path: in results dir if cumulative, else from config or current dir - if results_dir: - out_path = os.path.join(results_dir, "m_bias_results.yaml") - else: - out_path = config.get("output_path", "m_bias_results.yaml") - - # A result file describes itself: the provenance block the rule assembled - # (manifest hash, both repos' branch+commit, container sif + GHCR revision) - # rides verbatim from the config into the output yaml. It is appended as a - # separate top-level key, so the numeric m/c fields serialise byte-for-byte - # as before -- the reproduction gate sees only the added `provenance:` block. - output = dict(results) - if "provenance" in config: - output["provenance"] = config["provenance"] - - with open(out_path, "w") as f: - yaml.dump(output, f, default_flow_style=False) - print(f"Results written to {out_path}") - - # Also write to text file for readability - if results_dir: - txt_path = os.path.join(results_dir, "m_bias_results.txt") - else: - txt_path = config.get("output_path", "m_bias_results.yaml").replace( - ".yaml", ".txt" - ) - - with open(txt_path, "w") as f: - f.write("Multiplicative and additive shear bias from image simulations\n") - f.write("=" * 60 + "\n") - for scheme, res in results["weights"].items(): - f.write(f"\nweights: {scheme}\n") - f.write(f" m1 = {res['m1']:+.6f} ± {res['m1_err']:.6f}\n") - f.write(f" c1 = {res['c1']:+.6f} ± {res['c1_err']:.6f}\n") - f.write(f" m2 = {res['m2']:+.6f} ± {res['m2_err']:.6f}\n") - f.write(f" c2 = {res['c2']:+.6f} ± {res['c2_err']:.6f}\n") - f.write( - "\nErrors computed via bootstrap resampling " - f"(n={config['n_bootstrap']} resamples)\n" - ) - print(f"Results written to {txt_path}") - - if args.cumulative: - cumulative_path = os.path.join(results_dir, "mbias_cumulative.yaml") - if args.n_tiles: - added = update_cumulative_file(cumulative_path, args.n_tiles, results) - verb = "Added" if added else "Overwrote" - print(f"\n{verb} n_tiles={args.n_tiles} in {cumulative_path}") - # Regenerate plots after every update (an overwrite can shift the - # curve, so the plots must track it -- not just fresh additions). - try: - plot_convergence(cumulative_path, results_dir) - except Exception as e: - print( - f"Warning: could not generate convergence plots: {e}", - file=sys.stderr, - ) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/diagnostics_image_sims.py b/scripts/diagnostics_image_sims.py deleted file mode 100644 index 699d5c49..00000000 --- a/scripts/diagnostics_image_sims.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/env python -"""Per-sim diagnostics for image simulation catalogues. - -For each requested grid catalogue, produces: - - footprint (RA/Dec scatter) - - ellipticity histograms (e1, e2) - - weight histogram - - response matrix element histograms (R_g11, R_g22, R_g12, R_g21) - - PSF leakage scatter (e1 vs e1_PSF, e2 vs e2_PSF) - - additive bias (weighted mean e1, e2) - -Shares the estimator's config schema (``sp_validation.image_sims``): the same -``grids_dir`` / ``num`` / ``catalog_name`` keys, the ``branches`` list (the sim -map, not a hard-coded five), and the same ``w_col`` weight semantics -- a column -name, or ``null`` for unit weights. Reading ``w_col`` (rather than hard-coding -``w_des``) means the diagnostics never KeyError on a catalogue that lacks the -weight column, and they weight exactly as the m-bias run they accompany. - -Usage: - diagnostics_image_sims.py -c config.yaml [-v] -""" - -import argparse -import os -import sys - -import matplotlib -import numpy as np -import yaml - -matplotlib.use("Agg") -import matplotlib.pyplot as plt -from astropy.io import fits - -# Conventional campaign layout, used only when the config carries no branch map -# -- the same fallback the estimator uses. -_DEFAULT_BRANCHES = ["1z2z", "1m2z", "1p2z", "1z2m", "1z2p"] - - -def load(path): - with fits.open(path) as hdul: - return {col.name: hdul[1].data[col.name].copy() for col in hdul[1].columns} - - -def weights(cat, w_col): - """Per-object weights: the ``w_col`` column, or unit weights when null. - - Mirrors the estimator's ``w_col`` contract (image_sims._load_cat): ``None`` - -> every object unit weight (the no-weighting mode, #227). Reading it here - means the diagnostics never KeyError when the weight column is absent. - """ - return cat[w_col].copy() if w_col else np.ones(len(cat["RA"])) - - -def parse_args(): - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("-c", "--config", required=True) - p.add_argument("-v", "--verbose", action="store_true") - return p.parse_args() - - -def savefig(fig, out_dir, name): - path = f"{out_dir}/{name}.png" - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - return path - - -def plot_footprints(cats, colors, out_dir): - fig, ax = plt.subplots(figsize=(8, 6)) - for name, d in cats.items(): - ax.scatter(d["RA"], d["Dec"], s=1, alpha=0.4, label=name, color=colors[name]) - ax.set_xlabel("RA [deg]") - ax.set_ylabel("Dec [deg]") - ax.legend(markerscale=5) - ax.set_title("Footprint") - return savefig(fig, out_dir, "footprint") - - -def plot_ellipticity(cats, colors, w_col, out_dir, nbins=100): - fig, axs = plt.subplots(1, 2, figsize=(14, 5)) - bins = np.linspace(-1.0, 1.0, nbins + 1) - for name, d in cats.items(): - w = weights(d, w_col) - for ax, col, label in zip(axs, ["e1", "e2"], [r"$e_1$", r"$e_2$"]): - ax.hist( - d[col], - bins=bins, - density=True, - weights=w, - histtype="step", - label=name, - color=colors[name], - ) - for ax, label in zip(axs, [r"$e_1$", r"$e_2$"]): - ax.set_xlabel(label) - ax.set_ylabel("normalised count") - ax.legend(fontsize=7) - wlabel = w_col if w_col else "unit" - fig.suptitle(f"Ellipticity histograms ({wlabel} weighted)") - return savefig(fig, out_dir, "ellipticity_hist") - - -def plot_weights(cats, colors, w_col, out_dir, nbins=50): - fig, ax = plt.subplots(figsize=(8, 5)) - for name, d in cats.items(): - ax.hist( - weights(d, w_col), - bins=nbins, - density=True, - histtype="step", - label=name, - color=colors[name], - ) - wlabel = w_col if w_col else "unit" - ax.set_xlabel(wlabel) - ax.set_ylabel("normalised count") - ax.legend() - ax.set_title("Weight distribution") - return savefig(fig, out_dir, "weight_hist") - - -def plot_response(cats, colors, out_dir, nbins=50): - cols = ["R_g11", "R_g22", "R_g12", "R_g21"] - fig, axs = plt.subplots(2, 2, figsize=(12, 10)) - for ax, col in zip(axs.flat, cols): - for name, d in cats.items(): - ax.hist( - d[col], - bins=nbins, - range=(-1, 2), - density=True, - histtype="step", - label=name, - color=colors[name], - ) - ax.set_xlim(-1, 2) - ax.set_xlabel(col) - ax.set_ylabel("normalised count") - ax.legend(fontsize=7) - fig.suptitle("Response matrix elements") - fig.tight_layout() - return savefig(fig, out_dir, "response_hist") - - -def plot_psf_leakage(cats, colors, out_dir): - fig, axs = plt.subplots(1, 2, figsize=(14, 5)) - for name, d in cats.items(): - for ax, eg, ep, label in zip( - axs, - ["e1", "e2"], - ["e1_PSF", "e2_PSF"], - [r"$e_1$", r"$e_2$"], - ): - ax.scatter(d[ep], d[eg], s=1, alpha=0.3, label=name, color=colors[name]) - for ax, xlab, ylab in zip( - axs, [r"$e_1^{\rm PSF}$", r"$e_2^{\rm PSF}$"], [r"$e_1$", r"$e_2$"] - ): - ax.set_xlabel(xlab) - ax.set_ylabel(ylab) - ax.legend(markerscale=5, fontsize=7) - fig.suptitle("Object-wise PSF leakage") - return savefig(fig, out_dir, "psf_leakage") - - -def calculate_additive_bias(cats, w_col, verbose=True): - print("\n--- Additive bias (weighted mean ellipticity) ---") - results = {} - for name, d in cats.items(): - w = weights(d, w_col) - c1 = np.average(d["e1"], weights=w) - c2 = np.average(d["e2"], weights=w) - results[name] = (c1, c2) - if verbose: - print(f" {name}: c1 = {c1:+.5f} c2 = {c2:+.5f}") - return results - - -def main(): - args = parse_args() - with open(args.config) as f: - config = yaml.safe_load(f) - - # Same config schema as the estimator: grids_dir (not base), num, - # catalog_name, the branch map, and the w_col weight contract. - grids_dir = config["grids_dir"] - num = config["num"] - cat_name = config.get("catalog_name", "shape_catalog_cut_ngmix.fits") - branches = list(config.get("branches", _DEFAULT_BRANCHES)) - w_col = config["w_col"] # required, like the estimator; null -> unit weights - out_dir = config.get("diagnostics_dir", f"{grids_dir}/diagnostics") - - # Colour per branch from a palette, so any branch list plots (no hard-coded - # five-branch colour map). - palette = plt.get_cmap("tab10") - colors = {name: palette(i % 10) for i, name in enumerate(branches)} - - os.makedirs(out_dir, exist_ok=True) - - print(f"Loading catalogues from {grids_dir}...") - cats = {} - for name in branches: - path = f"{grids_dir}/{name}_grid_{num}/{cat_name}" - if not os.path.exists(path): - print(f" WARNING: {path} not found, skipping") - continue - cats[name] = load(path) - if args.verbose: - print(f" {name}: {len(cats[name]['RA'])} objects") - - if not cats: - print("No catalogues found, exiting.") - return 1 - - print(f"\nSaving plots to {out_dir}/") - print(f" footprint -> {plot_footprints(cats, colors, out_dir)}") - print(f" ellipticity -> {plot_ellipticity(cats, colors, w_col, out_dir)}") - print(f" weights -> {plot_weights(cats, colors, w_col, out_dir)}") - print(f" response -> {plot_response(cats, colors, out_dir)}") - print(f" PSF leakage -> {plot_psf_leakage(cats, colors, out_dir)}") - calculate_additive_bias(cats, w_col, verbose=True) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/patch_firecrown.py b/scripts/patch_firecrown.py new file mode 100644 index 00000000..7b528da1 --- /dev/null +++ b/scripts/patch_firecrown.py @@ -0,0 +1,225 @@ +"""Make pip-installed firecrown importable without NumCosmo. + +Run *inside* the target environment, after installing the ``[blinding]`` extra: + + python scripts/patch_firecrown.py + +Why this exists (PRD #241, PR 1): firecrown is the theory engine for +Smokescreen blinding — only ``compute_theory_vector`` on the SACC-read +cosmic-shear path is used. Upstream distributes firecrown via conda-forge, +where NumCosmo (a GObject-introspection C library, absent from PyPI) is always +present; in a pip/uv environment, firecrown 1.15.1 hits NumCosmo at *import +time* through two paths that have nothing to do with cosmic shear: + +1. ``firecrown/generators/__init__.py`` eagerly re-exports the LSST Y1/Y10 + predefined n(z) bin constants, defeating the lazy ``__getattr__`` that + ``_inferred_galaxy_zdist`` already provides — and computing those constants + imports NumCosmo. +2. ``firecrown/likelihood/__init__.py`` eagerly imports the cluster + likelihoods, which import ``crow`` (lsstdesc-crow), which subclasses a + NumCosmo C class at module load (``class CountsIntegralND(Ncm.IntegralND)``). + +This script (a) restores laziness in ``generators``, (b) makes the cluster +imports optional, and (c) installs a *loud* ``numcosmo_py`` shim so that any +genuine NumCosmo use raises immediately instead of being silently faked. +Everything is exact-string surgery against the pinned firecrown v1.15.1: if a +target string is missing (e.g. after a version bump), the script fails loudly +so the pin and the patch get reviewed together. Idempotent — safe to re-run. + +The right long-term fix is upstream (guarded/lazy imports in firecrown); until +then this file is the entire cost of staying pip-installable. +""" + +import importlib.metadata +import importlib.util +import subprocess +import sys +from pathlib import Path + +EXPECTED_FIRECROWN = "1.15.1" + +GENERATORS_OLD = """\ + # Lazy-loaded bins (via __getattr__) + Y1_LENS_BINS, + Y1_SOURCE_BINS, + Y10_LENS_BINS, + Y10_SOURCE_BINS, + LSST_Y1_LENS_HARMONIC_BIN_COLLECTION, + LSST_Y1_SOURCE_HARMONIC_BIN_COLLECTION, + LSST_Y10_LENS_HARMONIC_BIN_COLLECTION, + LSST_Y10_SOURCE_HARMONIC_BIN_COLLECTION, +) +""" + +GENERATORS_NEW = """\ +) + +# NOTE (sp_validation patch, scripts/patch_firecrown.py): the LSST Y1/Y10 +# predefined bin constants are computed lazily in _inferred_galaxy_zdist via a +# module-level __getattr__ that imports NumCosmo. Importing them EAGERLY here +# forced NumCosmo at `import firecrown.generators` (hence at +# `import firecrown.likelihood`), which pip cannot satisfy. Re-expose them +# lazily instead; the SACC-read cosmic-shear path never touches them. +_LAZY_BIN_NAMES = frozenset( + { + "Y1_LENS_BINS", + "Y1_SOURCE_BINS", + "Y10_LENS_BINS", + "Y10_SOURCE_BINS", + "LSST_Y1_LENS_HARMONIC_BIN_COLLECTION", + "LSST_Y1_SOURCE_HARMONIC_BIN_COLLECTION", + "LSST_Y10_LENS_HARMONIC_BIN_COLLECTION", + "LSST_Y10_SOURCE_HARMONIC_BIN_COLLECTION", + } +) + + +def __getattr__(name): + if name in _LAZY_BIN_NAMES: + from . import _inferred_galaxy_zdist as _z + + return getattr(_z, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + +""" + +LIKELIHOOD_OLD = """\ +# Cluster statistics +from firecrown.likelihood._binned_cluster import BinnedCluster +from firecrown.likelihood._binned_cluster_number_counts import ( + BinnedClusterNumberCounts, +) +from firecrown.likelihood._binned_cluster_number_counts_shear import ( + BinnedClusterShearProfile, +) +""" + +LIKELIHOOD_NEW = """\ +# Cluster statistics. +# NOTE (sp_validation patch, scripts/patch_firecrown.py): the cluster +# likelihoods import `crow` (lsstdesc-crow), which subclasses NumCosmo C +# classes at module load. NumCosmo is conda-forge-only, so in a pip/uv env +# these imports fail. They are NOT on the cosmic-shear (TwoPoint/WeakLensing) +# path, so they become optional: without NumCosmo the cluster classes are +# unavailable but everything else loads. +try: + from firecrown.likelihood._binned_cluster import BinnedCluster + from firecrown.likelihood._binned_cluster_number_counts import ( + BinnedClusterNumberCounts, + ) + from firecrown.likelihood._binned_cluster_number_counts_shear import ( + BinnedClusterShearProfile, + ) +except (ImportError, RuntimeError, TypeError): # pragma: no cover + BinnedCluster = None # type: ignore[assignment,misc] + BinnedClusterNumberCounts = None # type: ignore[assignment,misc] + BinnedClusterShearProfile = None # type: ignore[assignment,misc] +""" + +SHIM = '''\ +"""Minimal loud shim for numcosmo_py (installed by sp_validation). + +NumCosmo is a GObject-introspection C library available only via conda-forge. +With the companion patches to firecrown (scripts/patch_firecrown.py), the +SACC-read cosmic-shear likelihood path never imports it; this shim provides +the import-time names so the patched package loads, and any genuine numerical +use of NumCosmo raises loudly rather than being silently faked. +""" + + +class _Missing: + def __init__(self, path="numcosmo_py"): + self._p = path + + def __getattr__(self, name): + return _Missing(f"{self._p}.{name}") + + def __call__(self, *a, **k): + raise RuntimeError( + f"{self._p} was called, but NumCosmo is not installed (conda-forge " + "only, not on PyPI). It is not needed for the SACC-read " + "cosmic-shear likelihood path." + ) + + def __getitem__(self, item): + return _Missing(f"{self._p}[...]") + + +Ncm = _Missing("numcosmo_py.Ncm") +Nc = _Missing("numcosmo_py.Nc") +GObject = _Missing("numcosmo_py.GObject") + + +def dict_to_var_dict(*a, **k): + raise RuntimeError("numcosmo_py.dict_to_var_dict unavailable (no NumCosmo)") + + +def var_dict_to_dict(*a, **k): + raise RuntimeError("numcosmo_py.var_dict_to_dict unavailable (no NumCosmo)") +''' + + +def patch_file(path: Path, old: str, new: str) -> str: + text = path.read_text() + if new in text: + return "already patched" + if old not in text: + sys.exit( + f"FATAL: expected text not found in {path}.\n" + "firecrown has probably been bumped past the pinned version this " + "patch targets — review scripts/patch_firecrown.py together with " + "the [blinding] pin in pyproject.toml." + ) + path.write_text(text.replace(old, new, 1)) + return "patched" + + +def main() -> None: + spec = importlib.util.find_spec("firecrown") + if spec is None or spec.origin is None: + sys.exit("FATAL: firecrown is not installed in this environment.") + pkg = Path(spec.origin).parent + + # Metadata, not `import firecrown` — pre-patch, importing is what's broken. + version = importlib.metadata.version("firecrown") + if version != EXPECTED_FIRECROWN: + sys.exit( + f"FATAL: firecrown {version} != expected {EXPECTED_FIRECROWN}; " + "review this patch against the new version before bumping " + "EXPECTED_FIRECROWN." + ) + + print( + "generators/__init__.py:", + patch_file(pkg / "generators" / "__init__.py", GENERATORS_OLD, GENERATORS_NEW), + ) + print( + "likelihood/__init__.py:", + patch_file(pkg / "likelihood" / "__init__.py", LIKELIHOOD_OLD, LIKELIHOOD_NEW), + ) + + # Loud numcosmo_py shim — only when no real NumCosmo is present. + if importlib.util.find_spec("numcosmo_py") is None: + shim_dir = pkg.parent / "numcosmo_py" + shim_dir.mkdir(exist_ok=True) + (shim_dir / "__init__.py").write_text(SHIM) + print("numcosmo_py shim: installed") + else: + print("numcosmo_py shim: skipped (numcosmo_py importable)") + + check = subprocess.run( + [ + sys.executable, + "-c", + "import firecrown.likelihood; import smokescreen", + ], + capture_output=True, + text=True, + ) + if check.returncode != 0: + sys.exit(f"FATAL: post-patch import check failed:\n{check.stderr}") + print("post-patch import check: firecrown.likelihood + smokescreen OK") + + +if __name__ == "__main__": + main() diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index e37fc4d0..9032bc63 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -56,7 +56,7 @@ def get_calibrated_quantities(gal_metacal): return g_corr, g_uncorr, w, mask -def get_calibrated_m_c(gal_metacal, additive_correction=True): +def get_calibrated_m_c(gal_metacal): """Get Calibrated C. Return catalogue quantities for objects calibrated for multiplicative and @@ -66,11 +66,6 @@ def get_calibrated_m_c(gal_metacal, additive_correction=True): ---------- gal_metacal : dict galaxy metacalibration catalogue - additive_correction : bool, optional, default=True - if False, do not subtract the additive bias c from the shear - estimates; use for constant-shear image sims, where the mean - shear is the signal (see issue #226). c and c_err are still - computed and returned Returns ------- @@ -104,11 +99,10 @@ def get_calibrated_m_c(gal_metacal, additive_correction=True): c_err[comp] = np.std(g_uncorr[comp]) # Shear estimate corrected for additive bias - g_corr_mc = np.copy(g_corr) - if additive_correction: - c_corr = np.linalg.inv(gal_metacal.R).dot(c) - for comp in (0, 1): - g_corr_mc[comp] = g_corr[comp] - c_corr[comp] + g_corr_mc = np.zeros_like(g_corr) + c_corr = np.linalg.inv(gal_metacal.R).dot(c) + for comp in (0, 1): + g_corr_mc[comp] = g_corr[comp] - c_corr[comp] return g_corr_mc, g_uncorr, w, mask_metacal, c, c_err diff --git a/src/sp_validation/catalog.py b/src/sp_validation/catalog.py index e39d2076..b1b175d9 100644 --- a/src/sp_validation/catalog.py +++ b/src/sp_validation/catalog.py @@ -12,7 +12,6 @@ """ import getpass -import os import h5py import numpy as np @@ -309,35 +308,6 @@ def match_subsample( return ra, dec, g -def match_catalogs_radec(ra1, dec1, ra2, dec2, thresh_deg=0.0002): - """Match two catalogues by RA/Dec. - - Match each object in catalogue 2 to the nearest in catalogue 1 - within a threshold. - - Parameters - ---------- - ra1, dec1 : array_like - coordinates of reference catalogue [deg] - ra2, dec2 : array_like - coordinates of catalogue to match [deg] - thresh_deg : float, optional - maximum separation [deg], default 0.0002 - - Returns - ------- - idx1 : ndarray of int - indices into catalogue 1 of matched objects - idx2 : ndarray of int - indices into catalogue 2 of matched objects - """ - coord1 = coords.SkyCoord(ra=ra1 * u.degree, dec=dec1 * u.degree) - coord2 = coords.SkyCoord(ra=ra2 * u.degree, dec=dec2 * u.degree) - idx1, sep, _ = coord2.match_to_catalog_sky(coord1) - mask = sep.deg < thresh_deg - return idx1[mask], np.where(mask)[0] - - def match_stars2(ra_gal, dec_gal, ra_star, dec_star, thresh=0.0002): """Add docstring. @@ -572,91 +542,55 @@ def write_shape_catalog( ) ) - ext = os.path.splitext(output_path)[1].lower() - - if ext in (".hdf5", ".hdf", ".h5"): - # Build flat list of (name, 1d-array) pairs, splitting 2D columns - fields = [] - for col, _ in col_info_arr: - arr = np.asarray(col.array) - if arr.ndim == 2: - for idx in range(arr.shape[1]): - fields.append((f"{col.name}_{idx}", arr[:, idx])) - else: - fields.append((col.name, arr)) - - # Build structured numpy array and write as single "data" dataset - dtype = np.dtype([(name, arr.dtype) for name, arr in fields]) - structured = np.empty(len(fields[0][1]), dtype=dtype) - for name, arr in fields: - structured[name] = arr - - with h5py.File(output_path, "w") as f: - f.create_dataset("data", data=structured) - if add_header: - for key, val in add_header.items(): - f.attrs[key] = str(val) - if all(v is not None for v in (R, R_shear, R_select, c)): - f.attrs["R"] = R - f.attrs["R_shear"] = R_shear - f.attrs["R_select"] = R_select - f.attrs["c"] = c - if c_err is not None: - f.attrs["c1_err"] = c_err[0] - f.attrs["c2_err"] = c_err[1] - if sigma_epsilon is not None: - f.attrs["sig_eps"] = sigma_epsilon - if alpha_leakage is not None: - f.attrs["alpha"] = alpha_leakage - - else: - # Write columns to FITS file - cols = [col for col, _ in col_info_arr] - table_hdu = fits.BinTableHDU.from_columns(cols) - - # Add human-readable descriptions - for idx, col_info in enumerate(col_info_arr): - table_hdu.header[f"TTYPE{idx + 1}"] = ( - col_info[0].name, - col_info[1], - ) + # Write columns to FITS file + cols = [] + for col, _ in col_info_arr: + cols.append(col) + table_hdu = fits.BinTableHDU.from_columns(cols) + + # Add human-readable descriptions + for idx, col_info in enumerate(col_info_arr): + table_hdu.header[f"TTYPE{idx + 1}"] = ( + col_info[0].name, + col_info[1], + ) - # Primary HDU with information in header - primary_header = fits.Header() + # Primary HDU with information in header + primary_header = fits.Header() - if add_header: - primary_header.update(add_header) + if add_header: + primary_header.update(add_header) - primary_header = cat.write_header_info_sp( - primary_header, - software_name="sp_validation", - software_version=__version__, - author=getpass.getuser(), - ) + primary_header = cat.write_header_info_sp( + primary_header, + software_name="sp_validation", + software_version=__version__, + author=getpass.getuser(), + ) - if all(v is not None for v in (R, R_shear, R_select, c)): - cat.add_shear_bias_to_header(primary_header, R, R_shear, R_select, c) - if c_err is not None: - primary_header["c1_err"] = (c_err[0], "Standard deviation of c_1") - primary_header["c2_err"] = (c_err[1], "Standard deviation of c_2") + if all(v is not None for v in (R, R_shear, R_select, c)): + cat.add_shear_bias_to_header(primary_header, R, R_shear, R_select, c) + if c_err is not None: + primary_header["c1_err"] = (c_err[0], "Standard deviation of c_1") + primary_header["c2_err"] = (c_err[1], "Standard deviation of c_2") - primary_header["w"] = "DES weight" + primary_header["w"] = "DES weight" - if sigma_epsilon is not None: - primary_header["sig_eps"] = (sigma_epsilon, "Shape noise RMS") + if sigma_epsilon is not None: + primary_header["sig_eps"] = (sigma_epsilon, "Shape noise RMS") - if alpha_leakage: - primary_header["alpha"] = ( - alpha_leakage, - "Mean scale-dependent PSF leakage", - ) + if alpha_leakage: + primary_header["alpha"] = ( + alpha_leakage, + "Mean scale-dependent PSF leakage", + ) - primary_hdu = fits.PrimaryHDU(header=primary_header) + primary_hdu = fits.PrimaryHDU(header=primary_header) - # Final file - hdu_list = fits.HDUList([primary_hdu, table_hdu]) + # Final file + hdu_list = fits.HDUList([primary_hdu, table_hdu]) - hdu_list.writeto(output_path, overwrite=True) + hdu_list.writeto(output_path, overwrite=True) def write_galaxy_cat(output_path, ra, dec, tile_id): diff --git a/src/sp_validation/catalog_builders.py b/src/sp_validation/catalog_builders.py index 78dc7838..1375ef8d 100644 --- a/src/sp_validation/catalog_builders.py +++ b/src/sp_validation/catalog_builders.py @@ -1124,22 +1124,6 @@ def read_cat(self, load_into_memory=False): fpath = self._params["input_path"] verbose = self._params["verbose"] - # Image-simulation path: a single per-run comprehensive catalogue in - # FITS, not the joined multi-patch HDF5 the data path builds. Read the - # FITS table directly into memory; there is no separate data_ext group. - extension = os.path.splitext(fpath)[1] - if extension == ".fits": - if verbose: - print(f"Reading FITS file {fpath}, HDU 1...") - dat = fits.getdata(fpath, 1) - dat_ext = None - if verbose: - print( - f"Found {len(dat)} (~{format.millify(len(dat))}) objects" - + " in catalogue" - ) - return dat, dat_ext - if verbose: print(f"Reading HDF5 file {fpath}...") diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 48251bbb..0c9273d8 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -15,6 +15,7 @@ find_conservative_scale_cut_key, ) from ..statistics import chi2_and_pte +from ..version import __version__ from .catalog_characterization import CatalogCharacterizationMixin from .cosebis import CosebisMixin from .pseudo_cl import PseudoClMixin @@ -380,6 +381,23 @@ def _output_path(self, *parts): """ return os.path.abspath(os.path.join(self.cc["paths"]["output"], *parts)) + def sacc_nz(self, version): + """Single-bin ``nz`` mapping ``{0: (z, nz)}`` for the SACC writers. + + The tomography-native writer interface (``sacc_writers``) takes an nz + dict keyed by 0-based source bin; the round is single-bin, so the whole + survey n(z) is bin 0. ``get_redshift`` returns ``(z, nz)``. + """ + return {0: tuple(self.get_redshift(version))} + + def sacc_metadata(self, version): + """Provenance metadata stored on every SACC part for ``version``.""" + return { + "catalogue_version": version, + "sp_validation_version": __version__, + "npatch": self.npatch, + } + def get_redshift(self, version): """Load redshift distribution for a catalog version. diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index aa4657ba..6b1d6146 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -7,6 +7,7 @@ import numpy as np +from .. import sacc_io from ..b_modes import ( calculate_cosebis, find_conservative_scale_cut_key, @@ -15,6 +16,7 @@ plot_cosebis_scale_cut_heatmap, save_cosebis_results, ) +from .sacc_writers import cosebis_to_sacc class CosebisMixin: @@ -140,6 +142,46 @@ def calculate_cosebis( return results + @staticmethod + def _fiducial_cosebis_result(results, fiducial_scale_cut): + """Select the fiducial scale cut's result dict + its ``(min, max)`` cut. + + ``calculate_cosebis`` returns either a single result dict (full range) or + a multi-cut mapping keyed by ``(theta_min, theta_max)`` tuples. Only the + fiducial cut is a SACC data product: pick it via + ``find_conservative_scale_cut_key`` when a fiducial cut is given, else the + widest cut — mirroring ``plot_cosebis``. + """ + multi_cut = isinstance(results, dict) and all( + isinstance(k, tuple) for k in results + ) + if not multi_cut: + return results, tuple(results["scale_cut"]) + key = ( + find_conservative_scale_cut_key(results, fiducial_scale_cut) + if fiducial_scale_cut is not None + else max(results, key=lambda x: x[1] - x[0]) + ) + return results[key], tuple(key) + + def cosebis_to_sacc_part(self, version, out_path, results, fiducial_scale_cut=None): + """Write the COSEBIs SACC part at the fiducial scale cut. + + ``results`` is the object ``calculate_cosebis`` returned (single dict or + multi-cut mapping). Only the fiducial cut's ``{En, Bn, cov}`` becomes the + part — a ``FullCovariance`` must cover every stored point and the cuts + overlap in mode space, so the non-fiducial cuts stay in the diagnostic + ``.npz`` sidecar. The nz/metadata are the version's. + """ + result, scale_cut = self._fiducial_cosebis_result(results, fiducial_scale_cut) + s = cosebis_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + result, + scale_cut, + ) + sacc_io.save(s, out_path) + def plot_cosebis( self, version=None, diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 514049a6..29ff7846 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -17,6 +17,7 @@ from astropy.io import fits from cs_util.cosmo import get_theo_c_ell +from .. import sacc_io from ..pseudo_cl import ( apply_random_rotation, get_n_gal_map, @@ -26,6 +27,8 @@ ) from ..rho_tau import get_params_rho_tau from ..statistics import chi2_and_pte, cov_from_one_covariance +from .sacc_writers import BIN as SACC_BIN +from .sacc_writers import pseudo_cl_to_sacc class PseudoClMixin: @@ -451,14 +454,37 @@ def calculate_pseudo_cl_g_ng_cov(self, gaussian_part="iNKA"): f"Done Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part" ) - def calculate_pseudo_cl(self): + def calculate_pseudo_cl(self, out_path=None): """ Compute the pseudo-Cl of given catalogs. + + Each version's spectra are born as a SACC part via + :func:`sacc_writers.pseudo_cl_to_sacc` — EE/BB/EB carrying the shared + NaMaster bandpower window, with this instance's (blinded) n(z) stamped + in. The in-memory ``self._pseudo_cls[ver]`` ``"pseudo_cl"`` entry keeps + the ``ELL``/``EE``/``EB``/``BB`` arrays the plotting and B-mode-summary + consumers read by column name. + + ``out_path`` is the exact destination the part is *born at* — the + Snakemake-declared output. It must resolve per version; single-version + rules (the tagged blinded producer) pass their tagged output directly. + When ``None`` (multi-version diagnostic / the ``pseudo_cls`` property) + each part defaults to the untagged native ``pseudo_cl_{ver}.sacc``. + Skip-if-exists keys on this final path, so no two rules ever share an + undeclared native basename (a tagged product born at its native name and + then renamed would let one rule's skip-if-exists silently adopt — and + the rename delete — another rule's declared, differently-blinded file). """ self.print_start("Computing pseudo-Cl's") nside = self.nside + if out_path is not None and len(self.versions) != 1: + raise ValueError( + "calculate_pseudo_cl(out_path=...) writes one part to one path, " + f"but {len(self.versions)} versions are configured; call per version" + ) + try: self._pseudo_cls except AttributeError: @@ -468,20 +494,30 @@ def calculate_pseudo_cl(self): self._pseudo_cls[ver] = {} - out_path = self._output_path(f"pseudo_cl_{ver}.fits") - if os.path.exists(out_path): - self.print_done(f"Skipping Pseudo-Cl's calculation, {out_path} exists") - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]["pseudo_cl"] = cl_shear + ver_out_path = out_path or self._output_path(f"pseudo_cl_{ver}.sacc") + if os.path.exists(ver_out_path): + self.print_done( + f"Skipping Pseudo-Cl's calculation, {ver_out_path} exists" + ) + self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc( + ver_out_path + ) elif self.cell_method == "map": - self.calculate_pseudo_cl_map(ver, nside, out_path) + self.calculate_pseudo_cl_map(ver, nside, ver_out_path) elif self.cell_method == "catalog": - self.calculate_pseudo_cl_catalog(ver, out_path) + self.calculate_pseudo_cl_catalog(ver, ver_out_path) else: raise ValueError(f"Unknown cell method: {self.cell_method}") self.print_done("Done pseudo-Cl's") + @staticmethod + def _load_pseudo_cl_sacc(out_path): + """Read a pseudo-Cl SACC part into the ELL/EE/EB/BB dict consumers use.""" + s = sacc_io.load(out_path) + ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(s, SACC_BIN) + return {"ELL": ell, "EE": ee, "EB": eb, "BB": bb} + def calculate_pseudo_cl_map(self, ver, nside, out_path): params = get_params_rho_tau(self.cc[ver], survey=ver) @@ -547,10 +583,9 @@ def calculate_pseudo_cl_map(self, ver, nside, out_path): cl_shear = cl_shear - cl_noise self.print_cyan("Saving pseudo-Cl's...") - self.save_pseudo_cl(ell_eff, cl_shear, out_path) + self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]["pseudo_cl"] = cl_shear + self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) def calculate_pseudo_cl_catalog(self, ver, out_path): params = get_params_rho_tau(self.cc[ver], survey=ver) @@ -563,10 +598,9 @@ def calculate_pseudo_cl_catalog(self, ver, out_path): ) self.print_cyan("Saving pseudo-Cl's...") - self.save_pseudo_cl(ell_eff, cl_shear, out_path) + self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]["pseudo_cl"] = cl_shear + self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) def get_n_gal_map(self, params, nside, cat_gal): """Weighted galaxy number-density map (thin wrapper -> primitive).""" @@ -655,26 +689,22 @@ def apply_random_rotation(self, e1, e2, rng=None): """ return apply_random_rotation(e1, e2, rng) - def save_pseudo_cl(self, ell_eff, pseudo_cl, out_path): - """ - Save pseudo-Cl's to a FITS file. + def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): + """Write the pseudo-Cl SACC part (EE/BB/EB + shared bandpower window). - Parameters - ---------- - pseudo_cl : np.array - Pseudo-Cl's to save. - out_path : str - Path to save the pseudo-Cl's to. + ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); + the writer takes the shared bandpower window from ``wsp``. No covariance + is attached here — the analysis file's pseudo-Cl block is supplied at + assembly (``assemble_sacc``) from the NaMaster / OneCovariance product. """ - # Create columns of the fits file - col1 = fits.Column(name="ELL", format="D", array=ell_eff) - col2 = fits.Column(name="EE", format="D", array=pseudo_cl[0]) - col3 = fits.Column(name="EB", format="D", array=pseudo_cl[1]) - col4 = fits.Column(name="BB", format="D", array=pseudo_cl[3]) - coldefs = fits.ColDefs([col1, col2, col3, col4]) - cell_hdu = fits.BinTableHDU.from_columns(coldefs, name="PSEUDO_CELL") - - cell_hdu.writeto(out_path, overwrite=True) + s = pseudo_cl_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + ell_eff, + cl_all, + wsp, + ) + sacc_io.save(s, out_path) def plot_pseudo_cl(self): """ diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index 9f8a247e..af819977 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -16,10 +16,12 @@ from shear_psf_leakage.rho_tau_stat import PSFErrorFit from uncertainties import ufloat +from .. import sacc_io from ..rho_tau import ( get_rho_tau_w_cov, get_samples, ) +from .sacc_writers import rho_tau_to_sacc class PSFSystematicsMixin: @@ -41,11 +43,45 @@ def calculate_rho_tau_stats(self): cov_rho=self.compute_cov_rho, npatch=self.npatch, ) + self.rho_tau_to_sacc_part( + ver, out_dir, base, rho_stat_handler, tau_stat_handler + ) self.print_done("Rho stats finished") self._rho_stat_handler = rho_stat_handler self._tau_stat_handler = tau_stat_handler + def rho_tau_to_sacc_part( + self, version, out_dir, base, rho_stat_handler, tau_stat_handler + ): + """Write the ρ/τ SACC part for one version. + + ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage from the handler tables. The + ``CovTauTh`` theory covariance ``cov_tau_{base}_th.npy`` — a + ``(3·nbin, 3·nbin)`` plus-folded k-major block over ``{τ0, τ2, τ5}`` — is + passed as ``tau_cov_th`` when it exists (the τ-plus inference block); its + absence falls back to a diagonal placeholder for the whole part (loudly: + the τ inference block is then only a variance diagonal, not the theory + covariance). ρ always carries a diagnostic ``varrho`` diagonal. + """ + tau_cov_path = os.path.join(out_dir, f"cov_tau_{base}_th.npy") + tau_cov_th = np.load(tau_cov_path) if os.path.exists(tau_cov_path) else None + if tau_cov_th is None: + self.print_magenta( + f"No τ theory covariance at {tau_cov_path}; writing ρ/τ SACC part " + "with a diagonal placeholder covariance (τ inference block is a " + "variance diagonal, not CovTauTh)." + ) + s = rho_tau_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + rho_stat_handler.rho_stats, + tau_stat_handler.tau_stats, + tau_cov_th=tau_cov_th, + ) + out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") + sacc_io.save(s, out_path) + @property def rho_stat_handler(self): if not hasattr(self, "_rho_stat_handler"): diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 7524074e..416ad8ef 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -7,6 +7,7 @@ import numpy as np +from .. import sacc_io from ..b_modes import ( calculate_eb_statistics, calculate_pure_eb_correlation, @@ -16,6 +17,7 @@ plot_pure_eb_correlations, save_pure_eb_results, ) +from .sacc_writers import pure_eb_to_sacc class PureEBMixin: @@ -132,6 +134,25 @@ def calculate_pure_eb( return results + def pure_eb_to_sacc_part(self, version, out_path, results): + """Write the pure-E/B SACC part (six ``PURE_KEYS`` blocks + covariance). + + ``results`` is the dict ``calculate_pure_eb`` returned: the six pure-mode + arrays under ``sacc_io.PURE_KEYS``, the ``"cov"`` block (in ``PURE_KEYS`` + order), and the reporting-grid TreeCorr object ``"gg"`` whose ``meanr`` + is the shared ``theta``. + """ + theta = results["gg"].meanr + eb = {key: results[key] for key in sacc_io.PURE_KEYS} + s = pure_eb_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + theta, + eb, + covariance=results["cov"], + ) + sacc_io.save(s, out_path) + def plot_pure_eb( self, versions=None, diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 76d05d0a..a0852d77 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -12,12 +12,11 @@ import matplotlib.ticker as mticker import numpy as np import treecorr -from astropy.io import fits from cs_util import plots as cs_plots class RealSpaceMixin: - def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): + def calculate_2pcf(self, ver, npatch=None, **treecorr_config): """ Calculate the two-point correlation function (2PCF) ξ± for a given catalog version with TreeCorr. @@ -34,9 +33,6 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): npatch (int, optional): The number of patches to use for the calculation. Defaults to the instance's `npatch` attribute. - save_fits (bool, optional): Whether to save the ξ± results to FITS files. - Defaults to False. - **treecorr_config: Additional TreeCorr configuration parameters that will override the instance's default `treecorr_config`. For example, `min_sep=1`. @@ -49,8 +45,11 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): calculation is skipped, and the results are loaded from the file. - If a patch file for the given configuration does not exist, it is created during the process. - - FITS files for ξ+ and ξ− are saved with additional metadata in their - headers if `save_fits` is True. + - The ``.txt`` TreeCorr dump is the only raw byproduct written here + (read back by the covariance machinery and the skip-if-exists). The + analysis ξ± data product is born as SACC in the Snakemake scripts + (``run_2pcf.py`` coarse / ``run_2pcf_highres.py`` fine), which call + ``xi_to_sacc``; there is no DES-style ξ FITS writer anymore. """ self.print_magenta(f"Computing {ver} ξ±") @@ -101,74 +100,6 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): gg.process(cat_gal) gg.write(out_fname, write_patch_results=True, write_cov=True) - # Save xi_p and xi_m results to fits file - # (moved outside so it runs even if txt exists) - if save_fits: - lst = np.arange(1, treecorr_config["nbins"] + 1) - - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) - col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=gg.xip) - col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=gg.meanr) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") - - col4 = fits.Column(name="VALUE", format="D", array=gg.xim) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") - - # append xi_plus header info - xiplus_dict = { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - } - for key in xiplus_dict: - xiplus_hdu.header[key] = xiplus_dict[key] - - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) - col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=gg.xip) - col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=gg.rnom) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") - - col4 = fits.Column(name="VALUE", format="D", array=gg.xim) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") - - # append xi_plus header info - xiplus_dict = { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - } - for key in xiplus_dict: - xiplus_hdu.header[key] = xiplus_dict[key] - # Use same naming format as txt output - fits_base = out_fname.replace(".txt", "").replace("_xi_", "_") - xiplus_hdu.writeto( - f"{fits_base.replace(ver, f'xi_plus_{ver}')}.fits", - overwrite=True, - ) - - # append xi_minus header info - ximinus_dict = {**xiplus_dict, "QUANT1": "G-R", "QUANT2": "G-R"} - for key in ximinus_dict: - ximinus_hdu.header[key] = ximinus_dict[key] - ximinus_hdu.writeto( - f"{fits_base.replace(ver, f'xi_minus_{ver}')}.fits", - overwrite=True, - ) - # Add correlation object to class if not hasattr(self, "cat_ggs"): self.cat_ggs = {} diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py new file mode 100644 index 00000000..76e9e6e0 --- /dev/null +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -0,0 +1,254 @@ +"""Born-as-SACC writers for the cosmo_val data products. + +A thin, pure layer between the ``cosmo_val`` mixins (which compute statistics as +TreeCorr / NaMaster / b_modes arrays) and :mod:`sp_validation.sacc_io` (which +knows the file layout). Each ``*_to_sacc`` function turns one already-computed +statistic into a single-statistic SACC — a *part* — carrying that statistic's +own covariance as its one covariance block. The Snakemake DAG writes one part +per rule; :func:`assemble_analysis_sacc` then loads the parts and rebuilds the +single ``{version}.sacc`` analysis file with a ``FullCovariance`` assembled +block-diagonally in canonical order (per the SACC layout contract — *not* +``sacc.concatenate_data_sets``, whose ``BlockDiagonalCovariance`` output the +contract rules out). + +The fine-grid ``{version}_xi_fine.sacc`` is a terminal product in its own right +(:func:`xi_to_sacc` with ``grid="fine"`` and a ``DiagonalCovariance`` from +TreeCorr ``varxip``/``varxim``); COSEBIs and pure-E/B consume it. + +Everything here is single-bin today (``bins=(0, 0)``); the interface is +tomography-native so a future round supplies real bin pairs unchanged. +""" + +import numpy as np + +from .. import sacc_io as sio +from ..pseudo_cl import bandpower_window_from_workspace + +# Statistics carried in the analysis file, and their custom-type k indices. +RHO_K = range(6) # ρ_0 … ρ_5 +TAU_K = (0, 2, 5) # τ_0, τ_2, τ_5 + +# NaMaster spin-2 × spin-2 decoupled-spectrum row order (EE, EB, BE, BB). +_NMT_EE, _NMT_EB, _NMT_BB = 0, 1, 3 + +BIN = (0, 0) # single-bin default until the round goes tomographic + + +def xi_to_sacc( + nz, + metadata, + theta, + xip, + xim, + *, + grid, + theta_nom=None, + npairs=None, + weight=None, + variances=None, +): + """One ξ± part (``bins=(0, 0)``) on the coarse or fine grid. + + ``variances`` (the concatenated ``[varxip; varxim]``) attaches a + ``DiagonalCovariance`` — used for the terminal fine file, where npatch=1 + leaves TreeCorr shot-noise variance as the only covariance estimate. + """ + s = sio.new_sacc(nz, metadata) + sio.add_xi( + s, + BIN, + theta, + xip, + xim, + grid=grid, + theta_nom=theta_nom, + npairs=npairs, + weight=weight, + ) + if variances is not None: + sio.add_diagonal_covariance(s, np.asarray(variances)) + return s + + +def pseudo_cl_to_sacc(nz, metadata, ell_eff, cl_all, wsp, covariance=None): + """One pseudo-Cℓ part: EE/BB/EB with the shared bandpower window. + + ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); the + window comes from :func:`bandpower_window_from_workspace`. ``covariance``, + when given, is the dense ``[EE; BB; EB]``-ordered block matching insertion. + """ + window_ells, window_weights = bandpower_window_from_workspace(wsp) + s = sio.new_sacc(nz, metadata) + sio.add_pseudo_cl( + s, + BIN, + ell_eff, + cl_all[_NMT_EE], + cl_all[_NMT_BB], + cl_all[_NMT_EB], + window_ells=window_ells, + window_weights=window_weights, + ) + if covariance is not None: + s.add_covariance(np.asarray(covariance)) + return s + + +def cosebis_to_sacc(nz, metadata, result, scale_cut): + """One COSEBIs part at the fiducial scale cut. + + ``result`` is a single scale-cut result dict from + ``b_modes.calculate_cosebis`` — ``{"En", "Bn", "cov", ...}`` — where ``cov`` + is the ``[En; Bn]``-ordered COSEBIs covariance. Non-fiducial scale cuts are + a diagnostic (the PTE scan) and stay in the sidecar ``.npz``; only the + fiducial cut is a data product, because a ``FullCovariance`` must cover + every stored point and the cuts overlap in mode space. + """ + s = sio.new_sacc(nz, metadata) + sio.add_cosebis(s, BIN, result["En"], result["Bn"], scale_cut) + s.add_covariance(np.asarray(result["cov"])) + return s + + +def pure_eb_to_sacc(nz, metadata, theta, eb, covariance=None): + """One pure-E/B part: the six ``sacc_io.PURE_KEYS`` blocks. + + ``eb`` is a mapping with the six keys (``xip_E`` … ``xim_amb``); each array + is sampled at ``theta``. ``covariance``, when given, is the dense block in + ``PURE_KEYS`` order (matching ``b_modes._EB_KEYS`` and the insertion order). + """ + s = sio.new_sacc(nz, metadata) + sio.add_pure_eb(s, BIN, theta, **{key: eb[key] for key in sio.PURE_KEYS}) + if covariance is not None: + s.add_covariance(np.asarray(covariance)) + return s + + +def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov_th=None): + """One ρ/τ part: ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage. + + ``rho_stats`` / ``tau_stats`` are the ``shear_psf_leakage`` handler tables + (columns ``theta``, ``rho_{k}_p``, ``varrho_{k}_p``, ``rho_{k}_m``, … and + the τ analogue). Both diagnostics stay out of the blind and only τ enters + inference, so the covariance is a block-diagonal placeholder except for the + τ-plus theory block: + + - ρ (all 6·nbin points): diagonal from ``varrho`` — a diagnostic placeholder, + not consumed by inference. + - τ (6·nbin points, per-k ``[τ+; τ−]``): the ``CovTauTh`` theory covariance + ``tau_cov_th`` scattered into the τ-plus rows/columns. ``CovTauTh.build_cov`` + returns a ``(3·nbin, 3·nbin)`` k-major matrix over ``{τ0, τ2, τ5}`` with the + plus/minus contributions folded into one component per k (verified against + the write-side); it therefore aligns to our τ-plus points ``{τ0+, τ2+, τ5+}`` + in k-major order, and today's CosmoSIS chain (``covdat_to_fits``) consumes + exactly this flavor for τ. The τ-minus points carry only a ``vartau`` + diagonal (no theory covariance for them exists). ``tau_cov_th=None`` falls + back to a fully diagonal τ block (a flagged placeholder, not the design). + """ + s = sio.new_sacc(nz, metadata) + theta_rho = np.asarray(rho_stats["theta"]) + for k in RHO_K: + sio.add_rho( + s, + k, + theta_rho, + np.asarray(rho_stats[f"rho_{k}_p"]), + np.asarray(rho_stats[f"rho_{k}_m"]), + ) + theta_tau = np.asarray(tau_stats["theta"]) + for k in TAU_K: + sio.add_tau( + s, + BIN, + k, + theta_tau, + np.asarray(tau_stats[f"tau_{k}_p"]), + np.asarray(tau_stats[f"tau_{k}_m"]), + ) + nbin = len(theta_tau) + rho_var = np.concatenate( + [ + np.concatenate([rho_stats[f"varrho_{k}_p"], rho_stats[f"varrho_{k}_m"]]) + for k in RHO_K + ] + ) + tau_var = np.concatenate( + [ + np.concatenate([tau_stats[f"vartau_{k}_p"], tau_stats[f"vartau_{k}_m"]]) + for k in TAU_K + ] + ) + if tau_cov_th is None: + # Fully diagonal placeholder — a DiagonalCovariance (compact, honest) for + # the standalone diagnostic file; assemble reads it back via .dense. + s.add_covariance(np.concatenate([rho_var, tau_var])) + return s + tau_cov_th = np.asarray(tau_cov_th) + n_plus = len(TAU_K) * nbin + if tau_cov_th.shape != (n_plus, n_plus): + raise ValueError( + f"tau_cov_th shape {tau_cov_th.shape} does not match the " + f"{n_plus} τ-plus points ({len(TAU_K)} indices × {nbin} bins) — " + "CovTauTh.build_cov returns one (plus-folded) component per τ index" + ) + n_rho, n_tau = len(rho_var), len(tau_var) + tau_block = np.diag(tau_var) + # τ-plus local positions in the τ block, k-major (per-k layout is [+; −]). + plus = np.concatenate( + [np.arange(2 * i * nbin, 2 * i * nbin + nbin) for i in range(len(TAU_K))] + ) + tau_block[np.ix_(plus, plus)] = tau_cov_th + full = np.zeros((n_rho + n_tau, n_rho + n_tau)) + full[:n_rho, :n_rho] = np.diag(rho_var) + full[n_rho:, n_rho:] = tau_block + s.add_covariance(full) + return s + + +# --------------------------------------------------------------------------- # +# Analysis-file assembly +# --------------------------------------------------------------------------- # +def _copy_data_points(dst, src): + """Append every data point of ``src`` into ``dst`` (tags preserved).""" + for dp in src.data: + dst.add_data_point(dp.data_type, dp.tracers, dp.value, **dp.tags) + + +def assemble_analysis_sacc(nz, metadata, parts): + """Rebuild the single ``{version}.sacc`` analysis file from parts. + + Each part is a single-statistic Sacc (from a ``*_to_sacc`` writer, loaded + from disk) carrying its own covariance = its block. This re-adds every + part's data points into one Sacc in the order the parts are given — which + must be the canonical order (ξ± coarse, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ) + — and assembles a single ``FullCovariance`` from the per-part covariance + blocks. Point insertion order and block order therefore agree by + construction, which ``sacc_io.assemble_covariance`` validates (contiguous, + tiling, square) and raises on if they don't. + + Parameters + ---------- + nz, metadata : see :func:`sp_validation.sacc_io.new_sacc`. + parts : sequence of sacc.Sacc + Single-statistic parts, each with a covariance, in canonical order. + + Returns + ------- + sacc.Sacc + The analysis Sacc with a ``FullCovariance`` covering every point. + """ + s = sio.new_sacc(nz, metadata) + blocks = [] + cursor = 0 + for part in parts: + if part.covariance is None: + raise ValueError( + "every analysis part must carry its own covariance block; " + f"a part with data types {sorted(set(dp.data_type for dp in part.data))} " + "has none" + ) + n = len(part.mean) + _copy_data_points(s, part) + blocks.append((np.arange(cursor, cursor + n), part.covariance.dense)) + cursor += n + return sio.assemble_covariance(s, blocks) diff --git a/src/sp_validation/image_sims.py b/src/sp_validation/image_sims.py deleted file mode 100644 index e259109d..00000000 --- a/src/sp_validation/image_sims.py +++ /dev/null @@ -1,324 +0,0 @@ -"""IMAGE_SIMS. - -:Description: Multiplicative and additive shear bias from image simulations. - -:Author: Martin Kilbinger - -""" - -import numpy as np -from astropy.io import fits - -from sp_validation.catalog import match_catalogs_radec - -# Conventional campaign layout, used only when the config carries no branch map -# (e.g. the synthetic-recovery tests). In a workflow run the branches and pairs -# come from manifest.yaml via the m_bias config; nothing about the injected -# shear is hard-coded on the estimator's side. -_DEFAULT_BRANCHES = ["1z2z", "1p2z", "1m2z", "1z2p", "1z2m"] -_DEFAULT_PAIRS = [ - ("1p2z", "1m2z", 0), # g1 component, index 0 → e1 - ("1z2p", "1z2m", 1), # g2 component, index 1 → e2 -] - - -# Weight-scheme name that means "no weighting": every object gets unit weight. -# ``None`` (from a YAML ``null``) is accepted as an alias, so the fiducial -# unweighted primary scheme can be written either ``none`` or ``null``. -_UNWEIGHTED = "none" - - -def _is_unweighted(scheme): - """True for the unit-weight scheme (``"none"`` or ``None``).""" - return scheme is None or scheme == _UNWEIGHTED - - -def _load_cat(path, w_cols): - """Load RA, Dec, ellipticities and per-scheme weights from a FITS catalogue. - - Reads the ``e1``/``e2`` columns, which the calibration stage writes as the - *calibrated* shear estimate ``g = R^-1 g_uncal - c`` (metacal response and - additive-bias corrected) -- not the raw ``e1_uncal``/``e2_uncal`` columns - that sit alongside them in the same catalogue. The bias this estimator - measures is therefore the *residual* m/c left after the chain's own metacal - calibration, not the raw pre-calibration bias. - - ``w_cols`` is the list of weight schemes to load. The scheme ``"none"`` - (equivalently a ``None``/``null`` entry) gives every object unit weight -- - the no-weighting mode for m-bias runs (#227: shape weights are excluded from - sim calibration); any other entry is read as a FITS column name. The weights - come back as a dict keyed by scheme so one catalogue load serves every - scheme in a multi-weight run. - """ - with fits.open(path) as hdul: - data = hdul[1].data - cat = { - "ra": data["RA"].copy(), - "dec": data["Dec"].copy(), - "e1": data["e1"].copy(), - "e2": data["e2"].copy(), - "w": {}, - } - for scheme in w_cols: - cat["w"][scheme] = ( - np.ones(len(cat["ra"])) - if _is_unweighted(scheme) - else data[scheme].copy() - ) - return cat - - -class ImageSimMBias: - """Compute multiplicative and additive shear bias from image simulations. - - The estimator consumes the *calibrated* ``e1``/``e2`` columns (the metacal - response- and additive-bias-corrected shear ``g = R^-1 g_uncal - c``), so - the headline m/c is the **residual** bias remaining after the chain's own - metacal calibration, not the raw pre-calibration bias. - - Parameters - ---------- - config : dict - Configuration dictionary with keys: - - grids_dir : str, path to the grids directory - - num : int, run number (e.g. 2 for *_grid_2) - - catalog_name : str, filename of the cut catalogue - (default 'shape_catalog_cut_ngmix.fits') - - shear_amplitude : float, input shear |g| (from manifest.yaml) - - branches : list of str, branch names in load order (incl. the - unsheared reference); defaults to the conventional 5-branch layout - - pairs : list of dicts {plus, minus, component}, the +/- sheared - branch pairing per component; defaults to the conventional pairs - - match_radius_deg : float, matching radius in degrees (required) - - pair_match : bool, match objects between the +g and -g sheared - catalogues (required); if False, use all objects of each - catalogue (the paired per-object cancellation is then unavailable) - - w_cols : list of str, weight schemes to compute in one run - (required); ``"none"`` (or a ``null`` entry) means unit weights, - any other entry is a FITS column name. The **first** entry is the - primary result surfaced at the top level of ``run()``'s output. Our - fiducial run leads with the unweighted scheme (``["none", ...]``), - per the #227 verdict that shape weights are excluded from sim - calibration; the unweighted m also avoids the ``cov(w, e)`` residual - weighted estimators carry on constant-shear sims. - - w_col : str or None, *deprecated* single weight scheme; accepted for - back-compat and used as ``[w_col]`` only when ``w_cols`` is absent. - - n_bootstrap : int, number of bootstrap resamples for errors (required) - - bootstrap_seed : int, seed for the per-pair bootstrap RNG (required); - makes the bootstrap errors bit-reproducible. The resample indices are - drawn once per pair and shared across every weight scheme, so the - schemes differ only in their weighting, never in their draws. - - The science knobs (``match_radius_deg``, ``pair_match``, ``w_cols``, - ``n_bootstrap``, ``bootstrap_seed``) are read with no in-code default: a - missing one is a config bug and raises ``KeyError`` at construction, per the - fail-fast contract (the workflow emits every one into the m_bias config). - The lone exception is the deprecated ``w_col``, which is honoured as a - fallback so pre-``w_cols`` configs still run. - """ - - def __init__(self, config): - self.cfg = config - self.g_in = config["shear_amplitude"] - self.thresh = config["match_radius_deg"] - self.pair_match = config["pair_match"] - # ``w_cols`` is the required science key. A pre-``w_cols`` config that - # still carries the deprecated scalar ``w_col`` is honoured as a - # single-scheme run; only a config with neither raises (fail-fast). - if "w_cols" in config: - w_cols = config["w_cols"] - else: - w_cols = [config["w_col"]] - # Normalise a ``None``/``null`` entry to the canonical "none" name so - # results key off a string; downstream still treats it as unit weights. - self.w_cols = [_UNWEIGHTED if _is_unweighted(w) else str(w) for w in w_cols] - self.n_boot = config["n_bootstrap"] - self.boot_seed = config["bootstrap_seed"] - # Branch list and pairing come from the manifest-derived config - # (``branches`` / ``pairs``); fall back to the conventional layout only - # when neither is given. ``branches`` fixes the catalogue load order; - # ``pairs`` fixes which sims difference into which component. - self.sim_names = list(config.get("branches", _DEFAULT_BRANCHES)) - if config.get("pairs"): - self.pairs = [ - (p["plus"], p["minus"], p["component"]) for p in config["pairs"] - ] - else: - self.pairs = list(_DEFAULT_PAIRS) - self.cats = {} - - def load_catalogs(self, verbose=True): - """Load the 5 sheared and reference catalogues.""" - grids_dir = self.cfg["grids_dir"] - num = self.cfg["num"] - cat_name = self.cfg.get("catalog_name", "shape_catalog_cut_ngmix.fits") - # ``sim_names`` (incl. the unsheared reference) comes from the config's - # branch map. The +g/-g pool estimator pairs the sheared sims directly; - # the reference is loaded for completeness and null-test diagnostics. - for name in self.sim_names: - path = f"{grids_dir}/{name}_grid_{num}/{cat_name}" - if verbose: - print(f" Loading {path}") - self.cats[name] = _load_cat(path, self.w_cols) - if verbose: - print(f" {len(self.cats[name]['ra'])} objects") - - def print_mean_ellipticities(self): - """Print the mean e1, e2 for each catalogue and weight scheme, as a check. - - The unweighted scheme (``"none"``) gives the plain unweighted means. - """ - for scheme in self.w_cols: - print(f"\nMean ellipticities (all objects, weights: {scheme}):") - for name, cat in self.cats.items(): - mean_e1 = np.average(cat["e1"], weights=cat["w"][scheme]) - mean_e2 = np.average(cat["e2"], weights=cat["w"][scheme]) - print(f" {name}: = {mean_e1:+.5f} = {mean_e2:+.5f}") - - def _m_c_pair(self, name_p, name_m, comp, verbose=True): - """Compute m and c for one shear pair and component (0=g1, 1=g2). - - Paired ("pool") estimator. The +g and -g simulations inject opposite - input shear on the *same* galaxies, so matching them directly by - RA/Dec yields a one-to-one correspondence. Differencing the two - ellipticities per object, - - m = <(e_+ - e_-) / (2 g_in) - 1> , c = <(e_+ + e_-) / 2> , - - cancels the intrinsic shape (sigma_e ~ 0.3) object-by-object in the - multiplicative term, leaving only measurement noise -- so sigma(m) - shrinks by ~sigma_e/sigma_meas relative to differencing two - independent means. (The additive term c is a *sum*, so intrinsic - shape does not cancel there and its error stays shape-noise limited.) - - With ``pair_match=False`` the +g and -g sims are *not* matched: every - object of each catalogue is used, so the per-object cancellation is - lost and m, c fall back to differencing/summing the two independent - weighted means. The paired bootstrap likewise cannot be applied (the - two arrays generally have different lengths), so each side is resampled - independently per replicate. - """ - e_key = f"e{comp + 1}" - - if self.pair_match: - # Match the +g and -g sims to each other: same galaxies, opposite - # shear. This is a nearest-neighbour match within `thresh`, not a - # strict bijection -- on grid sims galaxies are well separated so - # pairs are effectively 1:1 (verified ~99% co-located to <0.05" on - # SKiLLS grid_1); on denser fields a small fraction could share a - # +g partner and dilute the cancellation. - idx_p, idx_m = match_catalogs_radec( - self.cats[name_p]["ra"], - self.cats[name_p]["dec"], - self.cats[name_m]["ra"], - self.cats[name_m]["dec"], - thresh_deg=self.thresh, - ) - if verbose: - print(f" {name_p} <-> {name_m}: {len(idx_p)} paired objects") - else: - idx_p = slice(None) - idx_m = slice(None) - if verbose: - print( - f" no pair-matching: {name_p}: {len(self.cats[name_p][e_key])}" - f" | {name_m}: {len(self.cats[name_m][e_key])} objects" - ) - - e_p = self.cats[name_p][e_key][idx_p] - e_m = self.cats[name_m][e_key][idx_m] - - # Draw the bootstrap resample indices *once*, before the weight-scheme - # loop, and reuse them for every scheme -- the schemes then differ only - # in their weighting, never in their draws (so a scheme comparison is a - # clean weighting comparison). Pre-drawing the full ``(n_boot, n)`` block - # in one call is bit-identical to drawing ``rng.integers(0, n, n)`` once - # per replicate (numpy fills the block row-major), so the numbers match a - # single-scheme, per-iteration bootstrap to the last bit. - rng = np.random.default_rng(seed=self.boot_seed) - if self.pair_match: - n = len(e_p) - ib = rng.integers(0, n, (self.n_boot, n)) - else: - n_p, n_m = len(e_p), len(e_m) - ib_p = rng.integers(0, n_p, (self.n_boot, n_p)) - ib_m = rng.integers(0, n_m, (self.n_boot, n_m)) - - res = {} - for scheme in self.w_cols: - w_p = self.cats[name_p]["w"][scheme][idx_p] - w_m = self.cats[name_m]["w"][scheme][idx_m] - m_boot = np.empty(self.n_boot) - c_boot = np.empty(self.n_boot) - - if self.pair_match: - # Per-object shear-differenced (-> m) and summed (-> c) - # ellipticity, with a symmetric per-pair weight. - w = 0.5 * (w_p + w_m) - d = (e_p - e_m) / (2 * self.g_in) - 1 - s = (e_p + e_m) / 2 - - m = np.average(d, weights=w) - c = np.average(s, weights=w) - - # Paired bootstrap: the same object draw is applied to both - # sims, so the per-object cancellation in `d` is preserved in - # the error estimate. - for i in range(self.n_boot): - m_boot[i] = np.average(d[ib[i]], weights=w[ib[i]]) - c_boot[i] = np.average(s[ib[i]], weights=w[ib[i]]) - else: - # No matching: difference/sum the two independent weighted means. - mean_ep = np.average(e_p, weights=w_p) - mean_em = np.average(e_m, weights=w_m) - - m = (mean_ep - mean_em) / (2 * self.g_in) - 1 - c = (mean_ep + mean_em) / 2 - - # Unpaired bootstrap: the +g and -g arrays generally differ in - # length, so each side is resampled independently per replicate. - for i in range(self.n_boot): - ep_b = np.average(e_p[ib_p[i]], weights=w_p[ib_p[i]]) - em_b = np.average(e_m[ib_m[i]], weights=w_m[ib_m[i]]) - m_boot[i] = (ep_b - em_b) / (2 * self.g_in) - 1 - c_boot[i] = (ep_b + em_b) / 2 - - res[scheme] = (m, np.std(m_boot), c, np.std(c_boot)) - - return res - - def run(self, verbose=True): - """Compute m and c for both shear components and every weight scheme. - - Returns - ------- - dict - ``results["weights"][scheme]`` holds ``m1, m1_err, c1, c1_err, - m2, m2_err, c2, c2_err`` for each weight scheme. The primary - (first) scheme's keys are also mirrored at the top level, so a - reader that wants the headline m/c never has to know the scheme - name. - """ - results = {"weights": {scheme: {} for scheme in self.w_cols}} - for name_p, name_m, comp in self.pairs: - label = f"g{comp + 1}" - if verbose: - print(f"\n--- {label}: {name_p} / {name_m} ---") - res = self._m_c_pair(name_p, name_m, comp, verbose=verbose) - for scheme, (m, m_err, c, c_err) in res.items(): - w = results["weights"][scheme] - w[f"m{comp + 1}"] = m - w[f"m{comp + 1}_err"] = m_err - w[f"c{comp + 1}"] = c - w[f"c{comp + 1}_err"] = c_err - if verbose: - print( - f" [{scheme}] m{comp + 1} = {m:.4f} ± {m_err:.4f}" - f" c{comp + 1} = {c:.4f} ± {c_err:.4f}" - ) - - # Mirror the primary (first) scheme's m/c at the top level: the headline - # result reads out without knowing the scheme name, and a downstream - # gate keyed on the old flat keys still finds them. - results.update(results["weights"][self.w_cols[0]]) - return results diff --git a/src/sp_validation/masks.py b/src/sp_validation/masks.py index 9d9e666e..a7ea74d1 100644 --- a/src/sp_validation/masks.py +++ b/src/sp_validation/masks.py @@ -8,53 +8,11 @@ """ import healsparse as hsp +import numexpr as ne import numpy as np from astropy.io import fits from scipy import stats -_KIND_ALIASES = {"smaller_equal": "less_equal"} - -_KIND_OPS = { - "equal": lambda a, v: a == v, - "not_equal": lambda a, v: a != v, - "greater": lambda a, v: a > v, - "greater_equal": lambda a, v: a >= v, - "less": lambda a, v: a < v, - "less_equal": lambda a, v: a <= v, - "range": lambda a, v: (a >= v[0]) & (a <= v[1]), -} - - -def apply_condition(array, kind, value): - """Apply Condition. - - Evaluate one mask condition (``kind``/``value``, as specified in mask - config YAML files) against an array and return a boolean mask. Single - shared grammar for the object-selection ``Mask`` class and the - cosmo_inference footprint builder (see issue #181). - - Parameters - ---------- - array : numpy.ndarray - input data - kind : str - operation type, one of "equal", "not_equal", "greater", - "greater_equal", "less", "less_equal" (alias "smaller_equal"), - "range" - value : float or list - value(s) to be used in mask operation; two-element list for "range" - - Returns - ------- - numpy.ndarray - boolean mask - - """ - kind = _KIND_ALIASES.get(kind, kind) - if kind not in _KIND_OPS: - raise ValueError(f"Unknown mask condition kind: {kind!r}") - return _KIND_OPS[kind](array, value) - def correlation_matrix(masks, confidence_level=0.9): @@ -118,7 +76,8 @@ class Mask: label : str mask label kind : str - operation type; see :func:`apply_condition` for the allowed values + operation type, allowed are "equal", "not_equal, ""greater_equal", + "smaller_equal", "range" value : float or list value(s) to be used in mask operation dat : numpy.ndarray, optional @@ -164,7 +123,40 @@ def from_list(cls, masks, label="combined", verbose=False): def apply(self, dat): - self._mask = apply_condition(dat[self._col_name], self._kind, self._value) + # Get column + col_data = dat[self._col_name] + + if self._kind == "equal": + self._mask = ne.evaluate( + "col_data == value", + local_dict={"col_data": col_data, "value": self._value}, + ) + elif self._kind == "not_equal": + self._mask = ne.evaluate( + "col_data != value", + local_dict={"col_data": col_data, "value": self._value}, + ) + elif self._kind == "greater_equal": + self._mask = ne.evaluate( + "col_data >= value", + local_dict={"col_data": col_data, "value": self._value}, + ) + elif self._kind == "smaller_equal": + self._mask = ne.evaluate( + "col_data <= value", + local_dict={"col_data": col_data, "value": self._value}, + ) + elif self._kind == "range": + self._mask = ne.evaluate( + "(col_data >= low) & (col_data <= high)", + local_dict={ + "col_data": col_data, + "low": self._value[0], + "high": self._value[1], + }, + ) + else: + raise ValueError(f"Invalid kind {self._kind}") def to_bool(self, hsp_mask): diff --git a/src/sp_validation/pseudo_cl.py b/src/sp_validation/pseudo_cl.py index c9355ec9..34cbc68d 100644 --- a/src/sp_validation/pseudo_cl.py +++ b/src/sp_validation/pseudo_cl.py @@ -280,3 +280,32 @@ def get_pseudo_cls_catalog( cl_all = wsp.decouple_cell(cl_coupled) return ell_eff, cl_all, wsp + + +# NaMaster spin-2 × spin-2 spectrum order: EE, EB, BE, BB. +_NMT_EE = 0 + + +def bandpower_window_from_workspace(wsp): + """Extract the bandpower window matrix ``W`` for a spin-2×spin-2 workspace. + + NaMaster's ``get_bandpower_windows()`` returns a four-index array + ``(n_cl_out, n_bpw, n_cl_in, n_ell)`` describing how each output bandpower + is built from the input multipoles across the EE/EB/BE/BB spectra. SACC's + ``BandpowerWindow`` model (one window per bandpower, shared across the + stored spectra) needs the per-spectrum *decoupling* window, i.e. the + diagonal EE←EE block (equal to BB←BB and EB←EB, verified identical). + + Returns + ------- + window_ells : np.ndarray + Multipoles the window spans, ``arange(n_ell)`` — the ``ell`` axis of + ``compute_coupled_cell``. + window_weights : np.ndarray + ``W`` of shape ``(n_ell, n_bpw)`` — one column per bandpower, the layout + :func:`sp_validation.sacc_io.add_pseudo_cl` expects. + """ + bpw = wsp.get_bandpower_windows() # (n_cl_out, n_bpw, n_cl_in, n_ell) + diagonal = bpw[_NMT_EE, :, _NMT_EE, :] # (n_bpw, n_ell) + window_ells = np.arange(diagonal.shape[1], dtype=float) + return window_ells, diagonal.T diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py new file mode 100644 index 00000000..ccddab97 --- /dev/null +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -0,0 +1,269 @@ +"""Integration tests for the ``assemble_sacc.py`` workflow script. + +The pure assembler (``sacc_writers.assemble_analysis_sacc``) is covered in +``test_sacc_writers.py``. This file exercises the *script seam* the DAG uses: +``assemble_sacc.assemble_sacc`` loads per-statistic ``.sacc`` part *files* in +CANONICAL order, injects the born-cov-less ξ± / pseudo-Cℓ blocks (real CosmoCov +/ NaMaster covariance, or a flagged diagonal placeholder), and writes one +``{version}.sacc`` whose points and covariance blocks land in canonical order. + +The script lives under ``workflow/scripts`` (off the package path); it is loaded +by file path exactly as the lightcone/ASTRA CLI path imports it. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest + +from sp_validation import sacc_io as sio +from sp_validation.cosmo_val import sacc_writers as sw + + +def _load_assemble_module(): + """Import ``workflow/scripts/assemble_sacc.py`` by file path.""" + repo_root = next( + p for p in Path(__file__).resolve().parents if (p / "pyproject.toml").exists() + ) + path = repo_root / "workflow" / "scripts" / "assemble_sacc.py" + spec = importlib.util.spec_from_file_location("assemble_sacc", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +asm = _load_assemble_module() + + +def _nz(seed=0, n=40): + rng = np.random.default_rng(seed) + return np.linspace(0.01, 2.0, n), rng.uniform(0.1, 1.0, n) + + +def _spd(n, seed): + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _theta(n=6): + return np.geomspace(1.0, 100.0, n) + + +META = {"catalogue_version": "vSYNTH", "npatch": 1} + + +def _write_parts(tmp_path, *, with_pseudo_cl=True, cov_less=("xi_coarse",)): + """Write per-statistic parts to disk; return the ``{name: path}`` mapping. + + Parts named in ``cov_less`` are written without a covariance (mimicking the + born-cov-less ξ± coarse / pseudo-Cℓ parts); the rest carry their own block. + """ + nz = {0: _nz()} + theta = _theta() + ell = np.array([30.0, 60.0, 90.0]) + + class _Wsp: + def get_bandpower_windows(self): + w = np.zeros((4, 3, 4, 20)) + for out in range(4): + for b in range(3): + w[out, b, out, b * 6 : b * 6 + 6] = 1.0 + return w + + xi = sw.xi_to_sacc( + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + if "xi_coarse" not in cov_less: + xi.add_covariance(_spd(len(xi.mean), 1)) + + cl_all = np.vstack( + [np.arange(3) * 1e-9, np.arange(3) * 2e-9, np.zeros(3), np.arange(3) * 3e-9] + ) + cl = sw.pseudo_cl_to_sacc( + nz, + META, + ell, + cl_all, + _Wsp(), + covariance=None if "pseudo_cl" in cov_less else _spd(9, 2), + ) + + co = sw.cosebis_to_sacc( + nz, + META, + { + "En": np.arange(1, 6) * 1e-6, + "Bn": np.arange(1, 6) * 1e-7, + "cov": _spd(10, 3), + }, + (1.0, 100.0), + ) + + eb_arrays = { + key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS) + } + eb = sw.pure_eb_to_sacc(nz, META, theta, eb_arrays, covariance=_spd(36, 4)) + + rho = {"theta": theta} + tau = {"theta": theta} + rng = np.random.default_rng(5) + for k in sw.RHO_K: + for suffix in ("p", "m"): + rho[f"rho_{k}_{suffix}"] = rng.normal(size=6) * 1e-6 + rho[f"varrho_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, 6) + for k in sw.TAU_K: + for suffix in ("p", "m"): + tau[f"tau_{k}_{suffix}"] = rng.normal(size=6) * 1e-6 + tau[f"vartau_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, 6) + rt = sw.rho_tau_to_sacc(nz, META, rho, tau) + + parts = { + "xi_coarse": xi, + "pseudo_cl": cl, + "cosebis": co, + "pure_eb": eb, + "rho_tau": rt, + } + if not with_pseudo_cl: + parts.pop("pseudo_cl") + + paths = {} + for name, part in parts.items(): + p = tmp_path / f"{name}.sacc" + sio.save(part, str(p)) + paths[name] = str(p) + return paths + + +def test_assemble_sacc_placeholder_canonical_order(tmp_path): + """The cov-less ξ± part gets a placeholder; every point is covered and the + blocks land in canonical order (ξ±, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ).""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + assert out.exists() + assert type(s.covariance).__name__ == "FullCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) + + # Canonical insertion order: the first data types are ξ+ then ξ−. + types_in_order = [dp.data_type for dp in s.data] + assert types_in_order[0] == sio.XI_PLUS + assert sio.XI_MINUS in types_in_order + # ξ appears before pseudo-Cℓ before COSEBIs before pure-E/B before ρ/τ. + first = {t: types_in_order.index(t) for t in set(types_in_order)} + assert first[sio.XI_PLUS] < first[sio.CL_EE] < first[sio.COSEBI_EE] + assert first[sio.COSEBI_EE] < first[sio.PURE_TYPES["xip_E"]] + assert first[sio.PURE_TYPES["xip_E"]] < first[sio.RHO_PLUS.format(k=0)] + assert first[sio.RHO_PLUS.format(k=0)] < first[sio.TAU_PLUS.format(k=0)] + + # The ξ± block is the placeholder diagonal (variance 1.0 on its own points). + tr = ("source_0", "source_0") + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + dense = s.covariance.dense + assert np.allclose(np.diag(dense[np.ix_(xi_idx, xi_idx)]), 1.0) + # ...and it does not bleed into the neighbouring COSEBIs block (cross zero). + co_idx = np.concatenate( + [s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)] + ) + assert np.allclose(dense[np.ix_(xi_idx, co_idx)], 0.0) + + +def test_assemble_sacc_injects_real_xi_covariance(tmp_path): + """A CosmoCov ξ covariance .txt is loaded into the cov-less ξ± block.""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + # ξ± part has 12 points ([ξ+; ξ−] over 6 θ); supply a matching cov .txt. + xi_cov = _spd(12, 21) + cov_path = tmp_path / "xi_cov.txt" + np.savetxt(str(cov_path), xi_cov) + + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=str(cov_path)) + tr = ("source_0", "source_0") + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + assert np.allclose(s.covariance.dense[np.ix_(xi_idx, xi_idx)], xi_cov) + + +def test_assemble_sacc_injects_pseudo_cl_covariance(tmp_path): + """The NaMaster cov FITS (COVAR_EE_EE/BB_BB/EB_EB) → block-diagonal pseudo-Cℓ + block (the live default: ξ± placeholder + real pseudo-Cℓ cov).""" + from astropy.io import fits + + paths = _write_parts(tmp_path, cov_less=("xi_coarse", "pseudo_cl")) + # pseudo-Cℓ part is 3 ell × {EE, BB, EB} = 9 points; per-spectrum 3×3 blocks. + ee, bb, eb = _spd(3, 31), _spd(3, 32), _spd(3, 33) + cov_fits = tmp_path / "pseudo_cl_cov.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + fits.ImageHDU(ee, name="COVAR_EE_EE"), + fits.ImageHDU(bb, name="COVAR_BB_BB"), + fits.ImageHDU(eb, name="COVAR_EB_EB"), + ] + ).writeto(str(cov_fits)) + + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc( + "vSYNTH", paths, str(out), pseudo_cl_cov=str(cov_fits), placeholder_var=1.0 + ) + tr = ("source_0", "source_0") + cl_idx = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + dense = s.covariance.dense + expected = np.zeros((9, 9)) + expected[0:3, 0:3], expected[3:6, 3:6], expected[6:9, 6:9] = ee, bb, eb + assert np.allclose(dense[np.ix_(cl_idx, cl_idx)], expected) + # ξ± stays the placeholder; the two blocks don't bleed into each other. + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + assert np.allclose(np.diag(dense[np.ix_(xi_idx, xi_idx)]), 1.0) + assert np.allclose(dense[np.ix_(xi_idx, cl_idx)], 0.0) + + +def test_assemble_sacc_missing_cov_raises(tmp_path): + """A cov-less part with no injected block and no placeholder fails loudly.""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + out = tmp_path / "vSYNTH.sacc" + with pytest.raises(ValueError, match="carries no covariance"): + asm.assemble_sacc("vSYNTH", paths, str(out)) + + +def test_assemble_sacc_respects_pseudo_cl_toggle(tmp_path): + """With pseudo_cl absent, assembly still succeeds and omits the Cℓ points.""" + paths = _write_parts(tmp_path, with_pseudo_cl=False, cov_less=("xi_coarse",)) + assert "pseudo_cl" not in paths + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + tr = ("source_0", "source_0") + assert len(s.indices(sio.CL_EE, tr)) == 0 + # Round-trips as a valid FullCovariance over the remaining points. + s2 = sio.load(str(out)) + assert type(s2.covariance).__name__ == "FullCovariance" + assert s2.covariance.dense.shape == (len(s2.mean), len(s2.mean)) + + +def test_assemble_sacc_expected_part_missing_raises(tmp_path): + """A typo'd input keyword drops a part from part_paths; the expected list + catches it rather than silently omitting the statistic.""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + # Simulate a rule-input typo: cosebis wired under the wrong key. + paths["cosebi"] = paths.pop("cosebis") + out = tmp_path / "vSYNTH.sacc" + with pytest.raises(ValueError, match="expected parts \\['cosebis'\\] missing"): + asm.assemble_sacc( + "vSYNTH", + paths, + str(out), + expected=["xi_coarse", "pseudo_cl", "cosebis", "pure_eb", "rho_tau"], + placeholder_var=1.0, + ) + + +def test_assemble_sacc_expected_rejects_unknown_name(tmp_path): + """A typo in the expected list itself is rejected (not a valid statistic).""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + out = tmp_path / "vSYNTH.sacc" + with pytest.raises(ValueError, match="not assemblable statistics"): + asm.assemble_sacc( + "vSYNTH", paths, str(out), expected=["cosebi"], placeholder_var=1.0 + ) diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index 68c09981..87afc416 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -1,11 +1,15 @@ -"""Back-pressure guard #2: the B-modes Snakemake workflow dry-runs. +"""Back-pressure guard #2: the paper Snakemake workflows dry-run. -The reorg is allowed to change the rule graph; this guard only asserts that -Snakemake can still parse the workflow and construct a dry run. +The reorg is allowed to change the rule graph; these guards only assert that +Snakemake can still parse each composed workflow and construct a dry run. One +guard covers papers/bmodes (config space, no cosmo_val block); a second covers +papers/cosmo_val, whose config DOES carry a cosmo_val block — so it is the only +one that includes cosmo_val.smk and hence the born-as-SACC + assemble rules. """ import os import subprocess +import sys from pathlib import Path import pytest @@ -26,31 +30,69 @@ def _repo_root() -> Path: raise RuntimeError("could not locate repo root (no pyproject.toml above test)") -@requires_candide_data -def test_bmodes_workflow_dry_runs(): - """The paper B-mode workflow must still parse and dry-run cleanly.""" - workflow_dir = _repo_root() / "papers/bmodes" - # PYTHONUNBUFFERED satisfies the Snakefile's `envvars:` declaration without - # depending on the invoking shell's environment. +def _dry_run(workflow_dir, targets, *extra_snakemake_args): + """Construct a dry run of the paper workflow at ``workflow_dir``. + + Returns the CompletedProcess. PYTHONUNBUFFERED satisfies the Snakefile's + ``envvars:`` declaration without depending on the invoking shell. A dry run + resolves the DAG only — it never dispatches jobs — so drop any inherited + SNAKEMAKE_PROFILE (e.g. the login shell's "slurm" profile), which would + otherwise force an executor plugin the test environment need not have. And + invoke snakemake through sys.executable (the interpreter pytest, hence + snakemake, lives in) — a bare python3.12 resolves off PATH to e.g. an + intel-python without snakemake. + """ env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} - result = subprocess.run( + env.pop("SNAKEMAKE_PROFILE", None) + return subprocess.run( [ - "python3.12", + sys.executable, "-m", "snakemake", - "all_tapestry", + *targets, "--dry-run", "--cores", "1", "--configfile", "config/config.yaml", + *extra_snakemake_args, ], cwd=workflow_dir, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - timeout=60, + timeout=120, check=False, ) + + +@requires_candide_data +def test_bmodes_workflow_dry_runs(): + """The paper B-mode workflow must still parse and dry-run cleanly.""" + result = _dry_run(_repo_root() / "papers/bmodes", ["all_tapestry"]) + assert result.returncode == 0, result.stdout + + +@requires_candide_data +def test_cosmo_val_workflow_assemble_dry_runs(): + """The cosmo_val workflow (the only one including cosmo_val.smk) resolves the + born-as-SACC + assemble DAG, and assemble pulls the tagged pseudo-Cl + cov. + + Targets the assemble_sacc_all rule so every version's assemble_sacc job + appears. The dry run resolves the DAG structure only — it never executes the + assemble script — so the placeholder-cov opt-in (cosmo_val.allow_placeholder_cov) + is irrelevant here; a real run would need it (or a wired --xi-cov) to proceed, + which is the fail-loud-by-default behaviour asserted in test_assemble_sacc.""" + version = "SP_v1.4.6.3_leak_corr" + result = _dry_run(_repo_root() / "papers/cosmo_val", ["assemble_sacc_all"]) assert result.returncode == 0, result.stdout + # assemble_sacc must be in the DAG and pull the tagged, blinded pseudo-Cl + # part + its NaMaster covariance (not the untagged cv_pseudo_cl diagnostic), + # plus all five per-statistic parts. + out = result.stdout + assert "rule assemble_sacc:" in out, out + assert f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" in out, out + assert f"pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits" in out, out + for part in ("_xi_coarse_", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): + assert part in out, f"missing {part} part in assemble DAG:\n{out}" diff --git a/src/sp_validation/tests/test_cli_seams.py b/src/sp_validation/tests/test_cli_seams.py new file mode 100644 index 00000000..697b3426 --- /dev/null +++ b/src/sp_validation/tests/test_cli_seams.py @@ -0,0 +1,65 @@ +"""Smoke tests for workflow CLI seams — cheap guards against signature rot. + +A CLI script that calls a workflow function with a removed/renamed kwarg +TypeErrors only at invocation time (the compute is cluster-only, so it is never +exercised by the fast suite). These tests bind the exact call each seam makes +against the current signature via ``inspect.signature(...).bind(...)`` — no +compute, no data — so a drifted kwarg (e.g. run_xi_sweep's dropped save_fits) +fails here instead of on the cluster. +""" + +import importlib.util +import inspect +from pathlib import Path + +import pytest + + +def _repo_root() -> Path: + for parent in Path(__file__).resolve().parents: + if (parent / "pyproject.toml").exists(): + return parent + raise RuntimeError("could not locate repo root (no pyproject.toml above test)") + + +def _load(path, name): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_run_xi_sweep_run_2pcf_call_binds(): + """The kwargs run_xi_sweep passes to run_2pcf must bind to its signature. + + Mirrors the call in papers/bmodes/scripts/run_xi_sweep.py — if run_2pcf drops + or renames a parameter (save_fits was removed by the SACC migration), the + bind raises TypeError here rather than on every cluster invocation. + """ + root = _repo_root() + run_2pcf_mod = _load(root / "workflow/scripts/run_2pcf.py", "run_2pcf_seam") + sig = inspect.signature(run_2pcf_mod.run_2pcf) + # Exactly the keyword set run_xi_sweep._from_cli passes (grid params spread + # from GRIDS: min_sep/max_sep/nbins/npatch). + sig.bind( + ver="V", + cat_config="/cfg.yaml", + output_dir="/out", + sacc_out="/out/V_xi_coarse_reporting.sacc", + min_sep=1.0, + max_sep=250.0, + nbins=20, + npatch=1, + ) + # And the removed kwarg must NOT bind (guards against a silent re-add). + with pytest.raises(TypeError): + sig.bind( + ver="V", + cat_config="/cfg.yaml", + output_dir="/out", + save_fits=True, + min_sep=1.0, + max_sep=250.0, + nbins=20, + npatch=1, + ) diff --git a/src/sp_validation/tests/test_config_paths_exist.py b/src/sp_validation/tests/test_config_paths_exist.py index 2db03bc1..9ae74ca0 100644 --- a/src/sp_validation/tests/test_config_paths_exist.py +++ b/src/sp_validation/tests/test_config_paths_exist.py @@ -23,14 +23,7 @@ "catalog", "catalogue", ) -# Keys whose values are never filesystem paths to check. ``extra_output`` is a -# flag, not a path. ``why``/``replace``/``with``/``drop`` are the declaration -# keys of a mask *overlay* (config/calibration/*.overlay.yaml): ``why`` is -# rationale prose and ``replace``/``with``/``drop`` are verbatim blocks of base -# config text -- content, not paths -- so the path walker must not treat them as -# files to stat. (The overlay's one real path, ``base:``, is deliberately not -# listed, so it is still validated.) -NON_PATH_KEYS = ("extra_output", "why", "replace", "with", "drop") +NON_PATH_KEYS = ("extra_output",) PATH_PREFIX_KEYS = ("nz.dndz.path",) TEXT_SUFFIXES = ( ".fits", diff --git a/src/sp_validation/tests/test_image_sims.py b/src/sp_validation/tests/test_image_sims.py deleted file mode 100644 index ef3e3b14..00000000 --- a/src/sp_validation/tests/test_image_sims.py +++ /dev/null @@ -1,253 +0,0 @@ -"""UNIT TESTS FOR THE IMAGE-SIMULATION m/c ESTIMATOR. - -Exercise ``sp_validation.image_sims.ImageSimMBias`` -- the multiplicative and -additive shear-bias estimator used by the image-simulation workflow -- and the -``sp_validation.catalog.match_catalogs_radec`` helper it relies on. - -The estimator recovers ``m`` and ``c`` from five calibrated catalogues named -``1z2z`` (reference, no input shear), ``1p2z``/``1m2z`` (input shear -``g1 = +-|g|``) and ``1z2p``/``1z2m`` (``g2 = +-|g|``). The +g and -g sims are -matched to *each other* by RA/Dec -- same galaxies, opposite input shear -- and -the bias is the object-paired ("pool") average - - m = <(e_+ - e_-) / (2 |g|) - 1> , c = <(e_+ + e_-) / 2> , - -so the intrinsic shape cancels object-by-object in ``m``. - -We build synthetic catalogues in which the measured ellipticity is exactly -``e = (1 + m_true) g_in + c_true`` at shared positions, so the recovered m/c -must equal the injected values to machine precision -- an analytic check of -the estimator maths that needs no pipeline run. - -:Author: cdaley - -""" - -import numpy as np -import numpy.testing as npt -from astropy.io import fits - -from sp_validation.catalog import match_catalogs_radec -from sp_validation.image_sims import ImageSimMBias - -# Injected truth, shared across the synthetic-recovery test. -A = 0.02 # input shear amplitude |g| -M_TRUE = 0.05 # multiplicative bias (same for both components) -C1_TRUE = 0.001 # additive bias, component 1 -C2_TRUE = -0.002 # additive bias, component 2 -N_GAL = 2000 - - -def _write_cat(path, ra, dec, e1, e2, w): - """Write a minimal calibrated shape catalogue (RA, Dec, e1, e2, w_des).""" - cols = [ - fits.Column(name=name, array=arr, format="D") - for name, arr in ( - ("RA", ra), - ("Dec", dec), - ("e1", e1), - ("e2", e2), - ("w_des", w), - ) - ] - fits.HDUList([fits.PrimaryHDU(), fits.BinTableHDU.from_columns(cols)]).writeto( - path, overwrite=True - ) - - -def _make_grid(grids_dir, num): - """Create the five sheared/reference catalogues with a known m/c.""" - rng = np.random.default_rng(0) - ra = 30.0 + rng.uniform(0, 0.1, N_GAL) - dec = rng.uniform(0, 0.1, N_GAL) - w = np.ones(N_GAL) - zero = np.zeros(N_GAL) - - def e(g_in): - return (1 + M_TRUE) * g_in + zero - - sims = { - "1z2z": (C1_TRUE + zero, C2_TRUE + zero), - "1p2z": (e(+A) + C1_TRUE, C2_TRUE + zero), - "1m2z": (e(-A) + C1_TRUE, C2_TRUE + zero), - "1z2p": (C1_TRUE + zero, e(+A) + C2_TRUE), - "1z2m": (C1_TRUE + zero, e(-A) + C2_TRUE), - } - for name, (e1, e2) in sims.items(): - sim_dir = grids_dir / f"{name}_grid_{num}" - sim_dir.mkdir(parents=True, exist_ok=True) - _write_cat(sim_dir / "cat.fits", ra, dec, e1, e2, w) - - -def test_match_catalogs_radec_identity(): - """Identical positions match one-to-one; a shifted object drops out.""" - ra = np.array([30.0, 30.01, 30.02]) - dec = np.array([10.0, 10.01, 10.02]) - # Second catalogue = first, but the last object nudged well past threshold. - ra2, dec2 = ra.copy(), dec.copy() - ra2[2] += 1.0 - idx1, idx2 = match_catalogs_radec(ra, dec, ra2, dec2, thresh_deg=0.0002) - npt.assert_array_equal(idx2, [0, 1]) - npt.assert_array_equal(idx1, [0, 1]) - - -def test_mbias_recovers_injected_values(tmp_path): - """ImageSimMBias recovers the injected m/c to machine precision.""" - num = 7 - _make_grid(tmp_path, num) - config = { - "grids_dir": str(tmp_path), - "num": num, - "catalog_name": "cat.fits", - "shear_amplitude": A, - "match_radius_deg": 0.0002, - "w_cols": ["w_des"], - "n_bootstrap": 50, - "pair_match": True, - "bootstrap_seed": 42, - } - mb = ImageSimMBias(config) - mb.load_catalogs(verbose=False) - res = mb.run(verbose=False) - - npt.assert_allclose(res["m1"], M_TRUE, atol=1e-9) - npt.assert_allclose(res["m2"], M_TRUE, atol=1e-9) - npt.assert_allclose(res["c1"], C1_TRUE, atol=1e-9) - npt.assert_allclose(res["c2"], C2_TRUE, atol=1e-9) - # Bootstrap errors are non-negative and finite. - for key in ("m1_err", "m2_err", "c1_err", "c2_err"): - assert np.isfinite(res[key]) and res[key] >= 0 - # Self-describing results: the primary scheme is mirrored at the top level - # *and* lives under ``weights[scheme]``, and the two agree exactly. - assert list(res["weights"]) == ["w_des"] - for key in ("m1", "m1_err", "c1", "c1_err", "m2", "m2_err", "c2", "c2_err"): - assert res[key] == res["weights"]["w_des"][key] - - -def test_mbias_multiple_weight_schemes_share_draws(tmp_path): - """Multi-scheme runs key results per scheme; the primary mirrors the first. - - ``none`` (unit weights) and a real weight column are computed in one run. - With uniform per-object weights in the synthetic grid the two schemes give - the *same* m/c (the weighting is a no-op), and the shared bootstrap indices - make even the errors identical -- the property that lets a scheme - comparison be a clean weighting comparison. The first entry (``none``) is - the primary result surfaced at the top level. - """ - num = 8 - _make_grid(tmp_path, num) - config = { - "grids_dir": str(tmp_path), - "num": num, - "catalog_name": "cat.fits", - "shear_amplitude": A, - "match_radius_deg": 0.0002, - "w_cols": ["none", "w_des"], - "n_bootstrap": 50, - "pair_match": True, - "bootstrap_seed": 42, - } - mb = ImageSimMBias(config) - mb.load_catalogs(verbose=False) - res = mb.run(verbose=False) - - assert list(res["weights"]) == ["none", "w_des"] - # Primary (first) scheme mirrored at the top level. - for key in ("m1", "m1_err", "c1", "c1_err", "m2", "m2_err", "c2", "c2_err"): - assert res[key] == res["weights"]["none"][key] - # Uniform grid weights make the schemes agree bit-for-bit, errors included - # (shared bootstrap draws). - assert res["weights"]["none"] == res["weights"]["w_des"] - - -def test_mbias_deprecated_w_col_still_runs(tmp_path): - """A pre-``w_cols`` config with the scalar ``w_col`` still runs. - - The deprecated single-scheme key is honoured as ``[w_col]`` when ``w_cols`` - is absent, so a legacy run config keeps working and produces the same - single-scheme result as the ``w_cols=[w_col]`` spelling. - """ - num = 9 - _make_grid(tmp_path, num) - base = { - "grids_dir": str(tmp_path), - "num": num, - "catalog_name": "cat.fits", - "shear_amplitude": A, - "match_radius_deg": 0.0002, - "n_bootstrap": 50, - "pair_match": True, - "bootstrap_seed": 42, - } - res_dep = ImageSimMBias({**base, "w_col": "w_des"}) - res_dep.load_catalogs(verbose=False) - out_dep = res_dep.run(verbose=False) - - res_new = ImageSimMBias({**base, "w_cols": ["w_des"]}) - res_new.load_catalogs(verbose=False) - out_new = res_new.run(verbose=False) - - assert list(out_dep["weights"]) == ["w_des"] - assert out_dep["weights"] == out_new["weights"] - - -def test_mbias_pool_cancels_shape_noise(tmp_path): - """The paired estimator cancels intrinsic shape noise in m. - - With realistic per-galaxy intrinsic ellipticity (sigma_e ~ 0.3) shared - between the +g and -g sims plus small independent measurement noise, the - object-paired difference cancels the intrinsic shape, so sigma(m) is set by - the measurement noise (~1e-2), not the shape noise. An *unpaired* estimator - (differencing two independently-drawn means) would instead return - sigma(m) ~ sigma_e / (2 |g| sqrt(N)) -- an order of magnitude larger. We - assert the recovered error sits well below that shape-noise floor, which is - the property the pooling exists to deliver. - """ - num = 3 - rng = np.random.default_rng(1) - ra = 30.0 + rng.uniform(0, 0.1, N_GAL) - dec = rng.uniform(0, 0.1, N_GAL) - w = np.ones(N_GAL) - sigma_e, sigma_meas = 0.3, 0.01 - e1_int = rng.normal(0, sigma_e, N_GAL) # intrinsic shape, shared across sims - e2_int = rng.normal(0, sigma_e, N_GAL) - - def measured(g1_in, g2_in): - """Measured ellipticity = intrinsic + (1 + m) * input shear + noise.""" - e1 = e1_int + (1 + M_TRUE) * g1_in + rng.normal(0, sigma_meas, N_GAL) - e2 = e2_int + (1 + M_TRUE) * g2_in + rng.normal(0, sigma_meas, N_GAL) - return e1, e2 - - sims = { - "1z2z": measured(0, 0), - "1p2z": measured(+A, 0), - "1m2z": measured(-A, 0), - "1z2p": measured(0, +A), - "1z2m": measured(0, -A), - } - for name, (e1, e2) in sims.items(): - sim_dir = tmp_path / f"{name}_grid_{num}" - sim_dir.mkdir(parents=True, exist_ok=True) - _write_cat(sim_dir / "cat.fits", ra, dec, e1, e2, w) - - config = { - "grids_dir": str(tmp_path), - "num": num, - "catalog_name": "cat.fits", - "shear_amplitude": A, - "match_radius_deg": 0.0002, - "w_cols": ["w_des"], - "n_bootstrap": 200, - "pair_match": True, - "bootstrap_seed": 42, - } - mb = ImageSimMBias(config) - mb.load_catalogs(verbose=False) - res = mb.run(verbose=False) - - shape_noise_floor = sigma_e / (2 * A * np.sqrt(N_GAL)) # the unpaired error - for comp in (1, 2): - # m recovered within a few sigma of truth... - assert abs(res[f"m{comp}"] - M_TRUE) < 5 * res[f"m{comp}_err"] - # ...and its error is far below what an unpaired estimator would give. - assert res[f"m{comp}_err"] < 0.1 * shape_noise_floor diff --git a/src/sp_validation/tests/test_mask_overlay.py b/src/sp_validation/tests/test_mask_overlay.py deleted file mode 100644 index 26244053..00000000 --- a/src/sp_validation/tests/test_mask_overlay.py +++ /dev/null @@ -1,85 +0,0 @@ -"""The image-sim mask config is a declared overlay on the data mask config. - -The image-sim calibration does not keep an independent copy of the mask / -calibration config: it keeps the *data* config (``mask_v1.X.9.yaml``) as the one -home for the shared cuts, and declares the sim-specific delta in an overlay -(``mask_v1.X.9_im_sim.overlay.yaml``). ``im_compose_mask.py`` applies the -overlay to the base and must reproduce the committed runtime file -(``mask_v1.X.9_im_sim.yaml``) **byte-for-byte**. - -This guard locks that equality, so the two artefacts cannot drift: - -* if someone edits the runtime file without updating the overlay (or vice - versa), :func:`test_compose_reproduces_runtime_byte_identical` goes red; -* if the base config changes such that an overlay anchor no longer matches, - the compose fails loudly rather than emitting a wrong file -- - :func:`test_compose_fails_loud_on_stale_anchor` locks that fail-fast. - -The runtime file is a tracked input to ``im_init``; keeping it byte-stable is -what keeps the reproduction gate bit-exact, so this test's unit is bytes, not -parsed YAML. -""" - -import importlib.util -from pathlib import Path - -import pytest - - -def _repo_root() -> Path: - """Locate the repo root by walking up to the ``pyproject.toml`` marker.""" - for parent in Path(__file__).resolve().parents: - if (parent / "pyproject.toml").exists(): - return parent - raise RuntimeError("could not locate repo root (no pyproject.toml above test)") - - -_CALIB_DIR = _repo_root() / "config" / "calibration" -_BASE = _CALIB_DIR / "mask_v1.X.9.yaml" -_OVERLAY = _CALIB_DIR / "mask_v1.X.9_im_sim.overlay.yaml" -_RUNTIME = _CALIB_DIR / "mask_v1.X.9_im_sim.yaml" - - -def _compose_module(): - """Import ``workflow/scripts/im_compose_mask.py`` (lives outside the package).""" - path = _repo_root() / "workflow" / "scripts" / "im_compose_mask.py" - spec = importlib.util.spec_from_file_location("im_compose_mask", path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def test_compose_reproduces_runtime_byte_identical(): - """compose(base, overlay) == the committed runtime file, byte-for-byte.""" - import yaml - - compose = _compose_module().compose - overlay = yaml.safe_load(_OVERLAY.read_text()) - base_text = _BASE.read_text() - - composed = compose(base_text, overlay) - - assert composed == _RUNTIME.read_text(), ( - "compose(mask_v1.X.9.yaml, overlay) no longer reproduces " - "mask_v1.X.9_im_sim.yaml byte-for-byte -- the runtime file and its " - "declared overlay have drifted; reconcile one against the other." - ) - - -def test_compose_fails_loud_on_stale_anchor(): - """A base whose text no longer carries an overlay anchor aborts, not composes. - - This is the drift-proofing: the overlay anchors to verbatim base text, so if - the base config is edited such that an anchor vanishes, the compose must die - with a clear message rather than silently emit a file missing that delta. - """ - import yaml - - module = _compose_module() - overlay = yaml.safe_load(_OVERLAY.read_text()) - # Drop the IMAFLAGS_ISO cut from the base text so its overlay anchor no - # longer matches; compose must abort (SystemExit from die()). - mangled = _BASE.read_text().replace("IMAFLAGS_ISO", "SOMETHING_ELSE") - - with pytest.raises(SystemExit, match="out of sync with the base"): - module.compose(mangled, overlay) diff --git a/src/sp_validation/tests/test_masks.py b/src/sp_validation/tests/test_masks.py deleted file mode 100644 index 50579c46..00000000 --- a/src/sp_validation/tests/test_masks.py +++ /dev/null @@ -1,63 +0,0 @@ -"""TESTS FOR THE SHARED MASK-CONDITION GRAMMAR. - -Covers ``sp_validation.masks.apply_condition`` — the single ``kind``/``value`` -evaluator shared by the object-selection ``Mask`` class and the -cosmo_inference footprint builder (issue #181) — plus an equivalence check -against ``Mask.apply()`` itself. - -:Author: cdaley - -""" - -import numpy as np -import numpy.testing as npt -import pytest - -from sp_validation.masks import Mask, apply_condition - -pytestmark = pytest.mark.fast - - -_ARRAY = np.array([1, 2, 3, 4, 5]) - - -@pytest.mark.parametrize( - "kind, value, expected", - [ - ("equal", 3, [False, False, True, False, False]), - ("not_equal", 3, [True, True, False, True, True]), - ("greater", 3, [False, False, False, True, True]), - ("greater_equal", 3, [False, False, True, True, True]), - ("less", 3, [True, True, False, False, False]), - ("less_equal", 3, [True, True, True, False, False]), - ("range", [2, 4], [False, True, True, True, False]), - ], -) -def test_apply_condition_kinds(kind, value, expected): - - npt.assert_array_equal(apply_condition(_ARRAY, kind, value), expected) - - -def test_smaller_equal_alias_matches_less_equal(): - - npt.assert_array_equal( - apply_condition(_ARRAY, "smaller_equal", 3), - apply_condition(_ARRAY, "less_equal", 3), - ) - - -def test_unknown_kind_raises(): - - with pytest.raises(ValueError): - apply_condition(_ARRAY, "not_a_real_kind", 3) - - -def test_mask_apply_matches_apply_condition(): - - dat = np.array([(1,), (2,), (3,), (4,), (5,)], dtype=[("col", "i8")]) - - my_mask = Mask("col", "test_mask", kind="greater_equal", value=3, dat=dat) - - npt.assert_array_equal( - my_mask._mask, apply_condition(dat["col"], "greater_equal", 3) - ) diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 45a2366d..637b10c2 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -57,7 +57,9 @@ import pytest import yaml +from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation +from sp_validation.cosmo_val.sacc_writers import BIN as SACC_BIN from sp_validation.rho_tau import get_params_rho_tau # These tests need the full harmonic-space stack (pymaster/NaMaster + healpy), @@ -112,6 +114,7 @@ def _write_synthetic_config(tmp_path): shear_cfg = { "path": "shear.fits", + "redshift_path": str(nz_dir / "dndz_SP_A.txt"), "w_col": "w", "e1_col": "e1", "e2_col": "e2", @@ -508,24 +511,24 @@ def test_apply_random_rotation_reproducible_with_seed(cv, cat_and_params): # calculate_pseudo_cl_catalog -- deterministic end-to-end catalog path # =========================================================================== def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): - """End-to-end catalog path: FITS round-trip of ell + EE/EB/BB. + """End-to-end catalog path: SACC round-trip of ell + EE/EB/BB. The catalog method has no random noise debiasing, so it is reproducible to - the same ~2e-12 catalog-path float noise. save_pseudo_cl stores ELL/EE/EB/BB - (it drops the BE row); we pin the round-tripped table. + the same ~2e-12 catalog-path float noise. calculate_pseudo_cl_catalog is + born-as-SACC: it writes a pseudo-Cl part (EE/BB/EB + shared bandpower + window) via pseudo_cl_to_sacc_part; we pin the round-tripped spectra read + back through sacc_io.get_pseudo_cl. """ ver = cv._test_version cv._pseudo_cls = {ver: {}} - out_path = cv._output_path(f"pseudo_cl_cat_{ver}.fits") + out_path = cv._output_path(f"pseudo_cl_{ver}.sacc") cv.calculate_pseudo_cl_catalog(ver, out_path) assert os.path.exists(out_path) - d = fits.getdata(out_path) - # FITS gives big-endian f8; normalize for value comparison. - ell = np.asarray(d["ELL"], dtype=np.float64) - ee = np.asarray(d["EE"], dtype=np.float64) - eb = np.asarray(d["EB"], dtype=np.float64) - bb = np.asarray(d["BB"], dtype=np.float64) + s = sacc_io.load(out_path) + ell, ee, bb, eb, window = sacc_io.get_pseudo_cl(s, SACC_BIN) + # A shared BandpowerWindow rides the part per the SACC layout contract. + assert window is not None npt.assert_allclose( ell, @@ -590,3 +593,32 @@ def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): params = get_params_rho_tau(cv.cc[ver], survey=ver) _, cl_prim, _ = cv.get_pseudo_cls_catalog(catalog=cat_gal, params=params) npt.assert_allclose(ee, cl_prim[0], rtol=RTOL_CAT, atol=ATOL_CAT) + + +def test_calculate_pseudo_cl_out_path_born_at_declared_name(cv): + """calculate_pseudo_cl(out_path=...) writes to the given path, not the + untagged native name — the anti-collision seam. + + The tagged producer (rule pseudo_cl, blind=A) and the untagged diagnostic + (rule cv_pseudo_cl, blind=None) both call calculate_pseudo_cl; if the tagged + one wrote the native pseudo_cl_{ver}.sacc and renamed, its skip-if-exists + could silently adopt — and the rename delete — the diagnostic's differently- + blinded file. Born-at-declared-name makes the two paths provably disjoint. + """ + ver = cv._test_version + cv._pseudo_cls = {} + tagged = cv._output_path(f"pseudo_cl_{ver}_blind=A_powspace_nbins=32.sacc") + native = cv._output_path(f"pseudo_cl_{ver}.sacc") + + cv.calculate_pseudo_cl(out_path=tagged) + + assert os.path.exists(tagged) + assert not os.path.exists(native) # no undeclared native basename touched + + +def test_calculate_pseudo_cl_out_path_rejects_multiversion(cv): + """out_path targets one part; a multi-version instance must fail loudly + rather than write every version to the same path.""" + cv.versions = [cv._test_version, "SecondVersion"] + with pytest.raises(ValueError, match="one part to one path"): + cv.calculate_pseudo_cl(out_path=cv._output_path("pseudo_cl_x.sacc")) diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py new file mode 100644 index 00000000..d886fa0a --- /dev/null +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -0,0 +1,313 @@ +"""Tests for :mod:`sp_validation.cosmo_val.sacc_writers`. + +Synthetic and fast: each ``*_to_sacc`` writer is exercised with in-memory +arrays, round-tripped through ``tmp_path``, and checked against the SACC layout +contract (data types, tags, ordering, covariance alignment). The analysis-file +assembler is verified to produce a single ``FullCovariance`` covering every +point with each per-statistic block correctly placed. One real small-nside +NaMaster round-trip proves the pseudo-Cℓ window survives the writer path. +""" + +import numpy as np +import pytest + +from sp_validation import sacc_io as sio +from sp_validation.cosmo_val import sacc_writers as sw + + +def _nz(seed=0, n=40): + rng = np.random.default_rng(seed) + return np.linspace(0.01, 2.0, n), rng.uniform(0.1, 1.0, n) + + +def _spd(n, seed): + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _theta(n=6): + return np.geomspace(1.0, 100.0, n) + + +def _roundtrip(s, tmp_path, name): + p = tmp_path / f"{name}.sacc" + sio.save(s, str(p)) + return sio.load(str(p)) + + +META = {"catalogue_version": "vSYNTH", "npatch": 1} + + +# --------------------------------------------------------------------------- # +# Per-writer parts +# --------------------------------------------------------------------------- # +def test_xi_to_sacc_coarse(tmp_path): + theta = _theta() + xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 + s = sw.xi_to_sacc( + {0: _nz()}, META, theta, xip, xim, grid="coarse", theta_nom=theta * 1.01 + ) + s2 = _roundtrip(s, tmp_path, "xic") + th, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(p, xip) and np.array_equal(m, xim) + assert s2.covariance is None # coarse part has no cov until assembly + + +def test_xi_to_sacc_fine_diagonal(tmp_path): + theta = np.geomspace(0.5, 300.0, 30) + xip, xim = np.arange(30) * 1e-5, np.arange(30) * 2e-5 + varxip, varxim = np.arange(1, 31) * 1e-12, np.arange(1, 31) * 2e-12 + s = sw.xi_to_sacc( + {0: _nz()}, + META, + theta, + xip, + xim, + grid="fine", + variances=np.concatenate([varxip, varxim]), + ) + assert type(s.covariance).__name__ == "DiagonalCovariance" + s2 = _roundtrip(s, tmp_path, "xif") + th, p, _ = sio.get_xi(s2, (0, 0), grid="fine") + assert np.array_equal(th, theta) and np.array_equal(p, xip) + assert np.array_equal( + np.diag(s2.covariance.dense), np.concatenate([varxip, varxim]) + ) + + +def test_pseudo_cl_to_sacc_window_and_rows(tmp_path): + ell = np.array([30.0, 60.0, 90.0, 120.0]) + nbp = len(ell) + # NaMaster (4, nbp): EE, EB, BE, BB. + cl_all = np.vstack( + [ + np.arange(nbp) * 1e-9, + np.arange(nbp) * 2e-9, + np.zeros(nbp), + np.arange(nbp) * 3e-9, + ] + ) + + class _Wsp: + """Stand-in workspace: (n_cl_out, nbp, n_cl_in, nell) window array.""" + + def __init__(self, nbp, nell): + w = np.zeros((4, nbp, 4, nell)) + col = np.zeros((nbp, nell)) + for b in range(nbp): + col[b, b * 3 : b * 3 + 3] = 1.0 + for out in range(4): + w[out, :, out, :] = col + self._w = w + + def get_bandpower_windows(self): + return self._w + + s = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell, cl_all, _Wsp(nbp, 24)) + s2 = _roundtrip(s, tmp_path, "cl") + ell_r, ee, bb, eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell_r, ell) + assert np.array_equal(ee, cl_all[0]) # EE row + assert np.array_equal(bb, cl_all[3]) # BB row (index 3, not 2=BE) + assert np.array_equal(eb, cl_all[1]) # EB row + assert window.weight.shape == (24, nbp) + + +def test_pseudo_cl_to_sacc_real_namaster(tmp_path): + """A real small-nside NaMaster workspace's window survives the writer.""" + pytest.importorskip("pymaster") + from sp_validation.pseudo_cl import get_pseudo_cls_map + + nside = 32 + mask = np.ones(12 * nside**2) + rng = np.random.default_rng(0) + shear = ( + rng.normal(size=12 * nside**2) + 1j * rng.normal(size=12 * nside**2) + ) * 1e-2 + ell_eff, cl_all, wsp = get_pseudo_cls_map(shear, mask, nside, "linear", ell_step=8) + s = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell_eff, cl_all, wsp) + s2 = _roundtrip(s, tmp_path, "clreal") + ell_r, ee, bb, eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell_r, ell_eff) + assert np.array_equal(ee, cl_all[0]) and np.array_equal(bb, cl_all[3]) + # window columns correspond to the bandpowers, one per ell_eff + assert window.weight.shape[1] == len(ell_eff) + + +def test_cosebis_to_sacc(tmp_path): + En, Bn = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 + result = {"En": En, "Bn": Bn, "cov": _spd(20, 7)} + s = sw.cosebis_to_sacc({0: _nz()}, META, result, (1.0, 100.0)) + s2 = _roundtrip(s, tmp_path, "co") + n, E, B = sio.get_cosebis(s2, (0, 0)) + assert np.array_equal(n, np.arange(1, 11)) + assert np.array_equal(E, En) and np.array_equal(B, Bn) + assert type(s2.covariance).__name__ == "FullCovariance" + assert np.array_equal(s2.covariance.dense, result["cov"]) + + +def test_pure_eb_to_sacc(tmp_path): + theta = _theta() + eb = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} + cov = _spd(6 * len(theta), 9) + s = sw.pure_eb_to_sacc({0: _nz()}, META, theta, eb, covariance=cov) + s2 = _roundtrip(s, tmp_path, "eb") + th, back = sio.get_pure_eb(s2, (0, 0)) + assert np.array_equal(th, theta) + for key in sio.PURE_KEYS: + assert np.array_equal(back[key], eb[key]) + assert np.array_equal(s2.covariance.dense, cov) + + +def _rho_tau_tables(nth=6, seed=0): + rng = np.random.default_rng(seed) + theta = _theta(nth) + rho = {"theta": theta} + for k in sw.RHO_K: + for suffix in ("p", "m"): + rho[f"rho_{k}_{suffix}"] = rng.normal(size=nth) * 1e-6 + rho[f"varrho_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, nth) + tau = {"theta": theta} + for k in sw.TAU_K: + for suffix in ("p", "m"): + tau[f"tau_{k}_{suffix}"] = rng.normal(size=nth) * 1e-6 + tau[f"vartau_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, nth) + return rho, tau, theta + + +def test_rho_tau_to_sacc_diagonal(tmp_path): + rho, tau, theta = _rho_tau_tables() + s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau) + s2 = _roundtrip(s, tmp_path, "rt") + for k in sw.RHO_K: + th, p, m = sio.get_rho(s2, k) + assert np.array_equal(th, theta) + assert np.array_equal(p, rho[f"rho_{k}_p"]) + assert np.array_equal(m, rho[f"rho_{k}_m"]) + for k in sw.TAU_K: + th, p, m = sio.get_tau(s2, (0, 0), k) + assert np.array_equal(p, tau[f"tau_{k}_p"]) + assert type(s2.covariance).__name__ == "DiagonalCovariance" + + +def test_rho_tau_to_sacc_tau_theory_block(tmp_path): + """The (3·nbin) plus-only CovTauTh block scatters into the τ-plus rows/cols; + τ-minus keeps a vartau diagonal, and cross plus↔minus stays zero.""" + rho, tau, theta = _rho_tau_tables() + nbin = len(theta) + n_plus = len(sw.TAU_K) * nbin # τ-plus points (k-major, one component per k) + tau_cov_th = _spd(n_plus, 11) + s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov_th=tau_cov_th) + assert type(s.covariance).__name__ == "FullCovariance" + tr = ("source_0", sio.PSF_TRACER) + tau_plus = np.concatenate( + [s.indices(sio.TAU_PLUS.format(k=k), tr) for k in sw.TAU_K] + ) + tau_minus = np.concatenate( + [s.indices(sio.TAU_MINUS.format(k=k), tr) for k in sw.TAU_K] + ) + s2 = _roundtrip(s, tmp_path, "rttau") + dense = s2.covariance.dense + # τ-plus sub-block equals the supplied theory covariance (scatter is correct). + assert np.allclose(dense[np.ix_(tau_plus, tau_plus)], tau_cov_th) + # τ-minus is diagonal from vartau; plus↔minus cross is zero. + tau_minus_var = np.concatenate([np.asarray(tau[f"vartau_{k}_m"]) for k in sw.TAU_K]) + assert np.allclose(np.diag(dense[np.ix_(tau_minus, tau_minus)]), tau_minus_var) + assert np.allclose(dense[np.ix_(tau_plus, tau_minus)], 0.0) + + +def test_rho_tau_to_sacc_tau_cov_shape_mismatch(): + rho, tau, _ = _rho_tau_tables() + with pytest.raises(ValueError, match="tau_cov_th shape"): + sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov_th=_spd(3, 1)) + + +# --------------------------------------------------------------------------- # +# Analysis-file assembly +# --------------------------------------------------------------------------- # +def _make_parts(nz): + theta = _theta() + ell = np.array([30.0, 60.0, 90.0]) + + class _Wsp: + def get_bandpower_windows(self): + w = np.zeros((4, 3, 4, 20)) + for out in range(4): + for b in range(3): + w[out, b, out, b * 6 : b * 6 + 6] = 1.0 + return w + + xi = sw.xi_to_sacc( + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + xi.add_covariance(_spd(len(xi.mean), 1)) + cl_all = np.vstack( + [np.arange(3) * 1e-9, np.arange(3) * 2e-9, np.zeros(3), np.arange(3) * 3e-9] + ) + cl = sw.pseudo_cl_to_sacc(nz, META, ell, cl_all, _Wsp(), covariance=_spd(9, 2)) + co = sw.cosebis_to_sacc( + nz, + META, + { + "En": np.arange(1, 6) * 1e-6, + "Bn": np.arange(1, 6) * 1e-7, + "cov": _spd(10, 3), + }, + (1.0, 100.0), + ) + return [xi, cl, co] + + +def test_assemble_analysis_sacc_full_covariance(tmp_path): + nz = {0: _nz()} + parts = _make_parts(nz) + s = sw.assemble_analysis_sacc(nz, META, parts) + assert type(s.covariance).__name__ == "FullCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) + # every point covered; blocks placed and cross-blocks zero + tr = ("source_0", "source_0") + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl_idx = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + co_idx = np.concatenate( + [s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)] + ) + assert len(xi_idx) + len(cl_idx) + len(co_idx) == len(s.mean) + dense = s.covariance.dense + assert np.array_equal(dense[np.ix_(xi_idx, xi_idx)], parts[0].covariance.dense) + assert np.array_equal(dense[np.ix_(cl_idx, cl_idx)], parts[1].covariance.dense) + assert np.array_equal(dense[np.ix_(co_idx, co_idx)], parts[2].covariance.dense) + assert np.array_equal( + dense[np.ix_(xi_idx, cl_idx)], np.zeros((len(xi_idx), len(cl_idx))) + ) + # round-trips + s2 = _roundtrip(s, tmp_path, "analysis") + assert type(s2.covariance).__name__ == "FullCovariance" + assert np.allclose(s2.covariance.dense, s.covariance.dense) + + +def test_assemble_analysis_sacc_requires_covariance(): + nz = {0: _nz()} + parts = _make_parts(nz) + parts.append( + sw.xi_to_sacc( + nz, META, _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + ) # no covariance + with pytest.raises(ValueError, match="own covariance block"): + sw.assemble_analysis_sacc(nz, META, parts) + + +def test_assemble_from_reloaded_parts(tmp_path): + """Parts written to disk then reloaded assemble identically (the DAG path).""" + nz = {0: _nz()} + parts = _make_parts(nz) + reloaded = [] + for i, part in enumerate(parts): + sio.save(part, str(tmp_path / f"part{i}.sacc")) + reloaded.append(sio.load(str(tmp_path / f"part{i}.sacc"))) + s = sw.assemble_analysis_sacc(nz, META, reloaded) + assert type(s.covariance).__name__ == "FullCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) diff --git a/uv-overrides.txt b/uv-overrides.txt new file mode 100644 index 00000000..813ad055 --- /dev/null +++ b/uv-overrides.txt @@ -0,0 +1,21 @@ +# uv dependency overrides — pass via `--overrides uv-overrides.txt` (or +# UV_OVERRIDE=uv-overrides.txt) to every `uv pip install` against this project. +# +# Why this file exists: firecrown declares its sampler *connectors* as hard +# dependencies, but we use firecrown only as the theory engine for Smokescreen +# blinding (`compute_theory_vector`); sampling stays with CosmoSIS in +# cosmo_inference. Of the three connector deps: +# +# - numcosmo-py exists only on conda-forge, so pip/uv resolution of firecrown +# is *impossible* without an override; +# - cosmosis ships sdist-only (full Fortran/C build with gsl/cfitsio) — a +# heavy, fragile compile in every CI image build, for a connector we never +# import; +# - cobaya is wheel-clean but equally unused. +# +# Each line below replaces the package's requirement (wherever it appears in +# the graph) with one gated on an always-false marker, dropping it from +# resolution. firecrown's likelihood/CCL core imports none of them. +numcosmo-py; python_version < "3" +cosmosis; python_version < "3" +cobaya; python_version < "3" diff --git a/uv.lock b/uv.lock deleted file mode 100644 index bef8072e..00000000 --- a/uv.lock +++ /dev/null @@ -1,4451 +0,0 @@ -version = 1 -revision = 3 -requires-python = ">=3.12" -resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'linux'", - "python_full_version < '3.14' and sys_platform == 'linux'", -] -supported-markers = [ - "sys_platform == 'linux'", -] - -[[package]] -name = "adjusttext" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b5/5c/496e506ad3313664df24a79f801719cadcd62af5999fcb299c9b08ff0d4b/adjusttext-1.4.0.tar.gz", hash = "sha256:1f73860ced8cccce3f85ee6989ca133c2579b67a7453f63dbeb38f39bf123154", size = 15852, upload-time = "2026-06-08T16:48:32.726Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/2c/897bdd17b05724c894a5b831c6b2e9853adcc2d07a70d6246c0cd5cd3912/adjusttext-1.4.0-py3-none-any.whl", hash = "sha256:6febd6484c0d45c39a22f44b2c1f4a8cd01ef58fada565cab4b629c771df79b5", size = 13262, upload-time = "2026-06-08T16:48:31.765Z" }, -] - -[[package]] -name = "aenum" -version = "3.1.17" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/07/e9/8b283567c1fef7c24d1f390b37daede8b61593d8cdaffb8e95d571699e83/aenum-3.1.17.tar.gz", hash = "sha256:a969a4516b194895de72c875ece355f17c0d272146f7fda346ef74f93cf4d5ba", size = 137648, upload-time = "2026-03-20T20:43:29.846Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/48/8d/1fe30c6fd8999b9d462547c4a1bb6690bda24af38f2913c4bec7decb81f2/aenum-3.1.17-py3-none-any.whl", hash = "sha256:8b883a37a04e74cc838ac442bdd28c266eae5bbf13e1342c7ef123ed25230139", size = 165560, upload-time = "2026-03-20T20:43:27.681Z" }, -] - -[[package]] -name = "alabaster" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz", hash = "sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e", size = 24210, upload-time = "2024-07-26T18:15:03.762Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, -] - -[[package]] -name = "annotated-types" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, -] - -[[package]] -name = "anyio" -version = "4.14.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "idna", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3b/72/5562aabb8dd7181e8e860622a38bea08d17842b99ecd4c91f84ac95251b0/anyio-4.14.1.tar.gz", hash = "sha256:8d648a3544c1a700e3ff78615cd679e4c5c3f149904287e73687b2596963629e", size = 254831, upload-time = "2026-06-24T20:56:06.017Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/7b/90df4a0a816d98d6ea26f559d87836d494a2cf1fcf063be67df50a7bcc30/anyio-4.14.1-py3-none-any.whl", hash = "sha256:4e5533c5b8ff0a24f5d7a176cbe6877129cd183893f66b537f8f227d10527d72", size = 124875, upload-time = "2026-06-24T20:56:04.413Z" }, -] - -[[package]] -name = "argon2-cffi" -version = "25.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argon2-cffi-bindings", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, -] - -[[package]] -name = "argon2-cffi-bindings" -version = "25.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441, upload-time = "2025-07-30T10:02:05.147Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/d3/942aa10782b2697eee7af5e12eeff5ebb325ccfb86dd8abda54174e377e4/argon2_cffi_bindings-25.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1c70058c6ab1e352304ac7e3b52554daadacd8d453c1752e547c76e9c99ac44", size = 86558, upload-time = "2025-07-30T10:01:43.943Z" }, - { url = "https://files.pythonhosted.org/packages/0d/82/b484f702fec5536e71836fc2dbc8c5267b3f6e78d2d539b4eaa6f0db8bf8/argon2_cffi_bindings-25.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2fd3bfbff3c5d74fef31a722f729bf93500910db650c925c2d6ef879a7e51cb", size = 92364, upload-time = "2025-07-30T10:01:44.887Z" }, - { url = "https://files.pythonhosted.org/packages/c9/c1/a606ff83b3f1735f3759ad0f2cd9e038a0ad11a3de3b6c673aa41c24bb7b/argon2_cffi_bindings-25.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c4f9665de60b1b0e99bcd6be4f17d90339698ce954cfd8d9cf4f91c995165a92", size = 85637, upload-time = "2025-07-30T10:01:46.225Z" }, - { url = "https://files.pythonhosted.org/packages/44/b4/678503f12aceb0262f84fa201f6027ed77d71c5019ae03b399b97caa2f19/argon2_cffi_bindings-25.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ba92837e4a9aa6a508c8d2d7883ed5a8f6c308c89a4790e1e447a220deb79a85", size = 91934, upload-time = "2025-07-30T10:01:47.203Z" }, - { url = "https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e021e87faa76ae0d413b619fe2b65ab9a037f24c60a1e6cc43457ae20de6dc6", size = 81246, upload-time = "2025-07-30T10:01:54.145Z" }, - { url = "https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e924cfc503018a714f94a49a149fdc0b644eaead5d1f089330399134fa028a", size = 87126, upload-time = "2025-07-30T10:01:55.074Z" }, - { url = "https://files.pythonhosted.org/packages/72/70/7a2993a12b0ffa2a9271259b79cc616e2389ed1a4d93842fac5a1f923ffd/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87b72589133f0346a1cb8d5ecca4b933e3c9b64656c9d175270a000e73b288d", size = 80343, upload-time = "2025-07-30T10:01:56.007Z" }, - { url = "https://files.pythonhosted.org/packages/78/9a/4e5157d893ffc712b74dbd868c7f62365618266982b64accab26bab01edc/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99", size = 86777, upload-time = "2025-07-30T10:01:56.943Z" }, -] - -[[package]] -name = "argparse-dataclass" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1a/ff/a2e4e328075ddef2ac3c9431eb12247e4ba707a70324894f1e6b4f43c286/argparse_dataclass-2.0.0.tar.gz", hash = "sha256:09ab641c914a2f12882337b9c3e5086196dbf2ee6bf0ef67895c74002cc9297f", size = 6395, upload-time = "2023-06-11T20:32:54.465Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/66/e6c0a808950ba5a4042e2fcedd577fc7401536c7db063de4d7c36be06f84/argparse_dataclass-2.0.0-py3-none-any.whl", hash = "sha256:3ffc8852a88d9d98d1364b4441a712491320afb91fb56049afd8a51d74bb52d2", size = 8762, upload-time = "2023-06-11T20:32:52.724Z" }, -] - -[[package]] -name = "array-api-compat" -version = "1.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/86/12/65d297d3e425cb7fe5247b7ec14b4ed83539e9b5e89b65c15ea7ee274182/array_api_compat-1.15.0.tar.gz", hash = "sha256:53c5f922491bf15f62847afafc4e39eedfae57d218988fefb8cce39c2a9b3dea", size = 129305, upload-time = "2026-06-07T20:53:24.964Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/16/1a8fd2b19544b84575cf84ef7aa3ad4c173b756d5f087c91f85d1b295777/array_api_compat-1.15.0-py3-none-any.whl", hash = "sha256:7b1b9c53269061403fd5f45a8de349f16e7887653328bfa0c5f2d45299ff0a8e", size = 79113, upload-time = "2026-06-07T20:53:23.621Z" }, -] - -[[package]] -name = "arrow" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-dateutil", marker = "sys_platform == 'linux'" }, - { name = "tzdata", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b9/33/032cdc44182491aa708d06a68b62434140d8c50820a087fac7af37703357/arrow-1.4.0.tar.gz", hash = "sha256:ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7", size = 152931, upload-time = "2025-10-18T17:46:46.761Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl", hash = "sha256:749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205", size = 68797, upload-time = "2025-10-18T17:46:45.663Z" }, -] - -[[package]] -name = "asteval" -version = "1.0.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b1/71/bec6758ab41ada3b6870cb194ea16d2e76bb530562c8a8cd7d2167012bea/asteval-1.0.9.tar.gz", hash = "sha256:4f7323d63d9fbf89d7e1d5f8bb59b6c093970c15bb5926e6e67d0195858aa230", size = 55237, upload-time = "2026-06-11T18:40:36.086Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/d7/a71b12b625c44cfcb95d41397752edccc0a731903081609cffd6720f671d/asteval-1.0.9-py3-none-any.whl", hash = "sha256:60513ea5d608d27bdb0cf3785dc993a6c3adeaee45f1bded0953a09e904e461f", size = 23320, upload-time = "2026-06-11T18:40:35.068Z" }, -] - -[[package]] -name = "astropy" -version = "8.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy-iers-data", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pyerfa", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0e/c4/21be4313ddfde5f60e0607fd307f367b9e0f0bf153a89b10cbd036dd8cfd/astropy-8.0.1.tar.gz", hash = "sha256:45ca31d5b91fa294cd590a4791a32db94de7f9c8a343155f4d5877baa82351da", size = 7152500, upload-time = "2026-07-05T07:24:48.482Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/3b/d29035da0a08e3b0b03d63abe9ce69b900d8ef7863dc557a2e34ebf56a4e/astropy-8.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9a4044396c15969bb029b648521f7c25d294e982dbd23e7c2c7dd328178b8b98", size = 10414634, upload-time = "2026-07-05T07:24:36.129Z" }, - { url = "https://files.pythonhosted.org/packages/fa/14/6f4427419f8a02e1d0a885fb64d8d4454c5d443627aa7cb1a3a36e16abb7/astropy-8.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fa11d56855e10107ea2231a6b6a33dbf1edbea6890adf34634c1f1d8f25c5a5a", size = 10452095, upload-time = "2026-07-05T07:24:38.424Z" }, - { url = "https://files.pythonhosted.org/packages/b2/ef/a5cf36a402c4511a776405f12d0b35196f55f4312ce407012a2fbcf1e4e0/astropy-8.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5b482bc6c57c966e6c6234410a1d9afbcf92bcac858cf287812f8c99ddc3fafc", size = 10407732, upload-time = "2026-07-05T07:24:40.774Z" }, -] - -[[package]] -name = "astropy-healpix" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/bb/fb3dcca212b1d57ed26b33eabb9e1cc18fcadbfc2f2976b20530b7d80b90/astropy_healpix-2.0.0.tar.gz", hash = "sha256:6ce7646dc5b1c09a7ed24b1f23d3e2fce8e3570387220d26a24487ab001b3b0b", size = 112064, upload-time = "2026-06-29T20:02:46.407Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/d3/6055bf7486e0fc980a2c842fc757918f79b012c3181d6f03b3fe230a2f65/astropy_healpix-2.0.0-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6bf86b2cdd9615de0031bc13b33fb72bcf085c5a2c78a6cf27c7a02b9ec05be7", size = 191108, upload-time = "2026-06-29T20:02:34.106Z" }, - { url = "https://files.pythonhosted.org/packages/d0/9e/0bf5fdb204527abb6ee1ae8f77ce867b1150203864a47c19d0e5bd3ae63a/astropy_healpix-2.0.0-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd3614d685ec2cd629f4e2bbb0e1db625fe878dde778b335ec07ab4754809d78", size = 193204, upload-time = "2026-06-29T20:02:35.261Z" }, - { url = "https://files.pythonhosted.org/packages/4a/1a/5b8680a2a68df29a38baebd9659c4c4b42f5885c7fcfeb03dc336afb0053/astropy_healpix-2.0.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8ffc56532822414fc6ba9b5e80b7897f98c96762d4ae69f14f5797e18946c26e", size = 188492, upload-time = "2026-06-29T20:02:36.721Z" }, - { url = "https://files.pythonhosted.org/packages/7e/43/174bf249286926518e77e29a52179a5c971544fa29affe8dee6ab33bcd5b/astropy_healpix-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:727588aebabb0bdf7f27aeb793a721090e9c62ab44c5dfe083c5a5ec4ad91df8", size = 198639, upload-time = "2026-06-29T20:02:41.547Z" }, - { url = "https://files.pythonhosted.org/packages/95/e1/2e57926e5c1e6a7b648b980656a3efd94ee58909fda577f7bf4fd701dca5/astropy_healpix-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31c4f998f83e7be4245fa75a48767d7749b2f233ee61e134573106dfbe53c73c", size = 201015, upload-time = "2026-06-29T20:02:42.721Z" }, - { url = "https://files.pythonhosted.org/packages/93/96/ce466e4f53b59b8e0a89fd09824cce2532c5b9146279b7397d844f78cadf/astropy_healpix-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3eca64fb1640dcb6b5633db73a33b2d208f64c5339a454ca38f538be3cdc0a4b", size = 195935, upload-time = "2026-06-29T20:02:43.868Z" }, -] - -[[package]] -name = "astropy-iers-data" -version = "0.2026.7.6.1.1.20" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7f/e3/d32d2401c53fa1aaf118c963c51c4eb9cd10d074a72a52430904dc8ef46e/astropy_iers_data-0.2026.7.6.1.1.20.tar.gz", hash = "sha256:bc38de235802c9f006c8a04693706e2adcc9e7ac1cbf415dc042b5f0ac14f1f3", size = 1939850, upload-time = "2026-07-06T01:02:02.379Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8c/a3/1d816863674c610234afef9a444d4eb562689a3fa1a872ac5687b4e0eac4/astropy_iers_data-0.2026.7.6.1.1.20-py3-none-any.whl", hash = "sha256:7aa50fa7f6d13258357d1e17fa97f3a8c2f04b9a7429d056e986a91cc46576a0", size = 1994649, upload-time = "2026-07-06T01:02:00.977Z" }, -] - -[[package]] -name = "asttokens" -version = "3.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/be/a5/8e3f9b6771b0b408517c82d97aed8f2036509bc247d46114925e32fe33f0/asttokens-3.0.1.tar.gz", hash = "sha256:71a4ee5de0bde6a31d64f6b13f2293ac190344478f081c3d1bccfcf5eacb0cb7", size = 62308, upload-time = "2025-11-15T16:43:48.578Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl", hash = "sha256:15a3ebc0f43c2d0a50eeafea25e19046c68398e487b9f1f5b517f7c0f40f976a", size = 27047, upload-time = "2025-11-15T16:43:16.109Z" }, -] - -[[package]] -name = "async-lru" -version = "2.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e8/1f/989ecfef8e64109a489fff357450cb73fa73a865a92bd8c272170a6922c2/async_lru-2.3.0.tar.gz", hash = "sha256:89bdb258a0140d7313cf8f4031d816a042202faa61d0ab310a0a538baa1c24b6", size = 16332, upload-time = "2026-03-19T01:04:32.413Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/e2/c2e3abf398f80732e58b03be77bde9022550d221dd8781bf586bd4d97cc1/async_lru-2.3.0-py3-none-any.whl", hash = "sha256:eea27b01841909316f2cc739807acea1c623df2be8c5cfad7583286397bb8315", size = 8403, upload-time = "2026-03-19T01:04:30.883Z" }, -] - -[[package]] -name = "attrs" -version = "26.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, -] - -[[package]] -name = "babel" -version = "2.18.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, -] - -[[package]] -name = "beautifulsoup4" -version = "4.15.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "soupsieve", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, -] - -[[package]] -name = "bleach" -version = "6.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "webencodings", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/48/3c/e12ac860709702bd5ebeb9b56a4fe334f1001246ee1b8f2b7ee28912df7d/bleach-6.4.0.tar.gz", hash = "sha256:4202482733d85cedd04e59fcb2f89f4e4c7c385a78d3c3c23c30446843a37452", size = 204857, upload-time = "2026-06-05T13:01:13.734Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/9d/40b6267367182187139a4000b82a3b287d84d745bccd808e75d916920e9d/bleach-6.4.0-py3-none-any.whl", hash = "sha256:4b6b6a54fff2e69a3dde9d21cc6301220bee3c3cb792187d11403fd795031081", size = 165109, upload-time = "2026-06-05T13:01:12.504Z" }, -] - -[package.optional-dependencies] -css = [ - { name = "tinycss2", marker = "sys_platform == 'linux'" }, -] - -[[package]] -name = "blosc2" -version = "4.8.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx", extra = ["http2"], marker = "sys_platform == 'linux'" }, - { name = "msgpack", marker = "sys_platform == 'linux'" }, - { name = "ndindex", marker = "sys_platform == 'linux'" }, - { name = "numexpr", marker = "platform_machine != 'wasm32' and sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pydantic", marker = "sys_platform == 'linux'" }, - { name = "rich", marker = "sys_platform == 'linux'" }, - { name = "threadpoolctl", marker = "platform_machine != 'wasm32' and sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c1/26/c5b32a553f002e91886213ec91ff56a2dd8555e5e78eb4c289addb208642/blosc2-4.8.0.tar.gz", hash = "sha256:9789c3248052be59e63d6debafec4cd59d0eb4a6fd1cbaced36865d5d40ffa63", size = 5681856, upload-time = "2026-07-10T09:38:33.592Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/79/54ea19add4199547d684f3a90c3cc0568b6d915403b15c3e4bc73d4bbf3b/blosc2-4.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:95b04eb8a769e98de9640915f9c5c0a3074714762958896440e5a479a96bf83e", size = 6704345, upload-time = "2026-07-10T09:37:56.942Z" }, - { url = "https://files.pythonhosted.org/packages/e3/b0/aa277d5952ac4c673924454277de0800c780bcdb15917e303dac8ea3840f/blosc2-4.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:12668fe516f18f783877457ddcc5072d80edf4939914e27acf900ef65e970e69", size = 7208868, upload-time = "2026-07-10T09:37:58.649Z" }, - { url = "https://files.pythonhosted.org/packages/ab/1b/765d2d38609d11e727f63c44f30abacccd59132e74f6e082b3741656f108/blosc2-4.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90b1a86b9044b3b242d78d5ac1d62abb8af7b1da355eb8eb45978539f9df2eac", size = 6708353, upload-time = "2026-07-10T09:38:05.91Z" }, - { url = "https://files.pythonhosted.org/packages/ee/aa/c72f0ff0c1cfa7e3a06530453f9b5388e4bb6266b73608f58232f181d68c/blosc2-4.8.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c17ec006d2770a4710552b0f4f8ceb40654fda08fe5047b27ccb7b0f396c3c6e", size = 7214012, upload-time = "2026-07-10T09:38:07.685Z" }, - { url = "https://files.pythonhosted.org/packages/6e/54/d7b8a01b43751840d75908d57601f03711a7cf260d46140f4c09dff5f9d6/blosc2-4.8.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:2af9162e7712323b21ad4d6502326c21459b6851b758b0421aa0be2e8b9dc0c1", size = 2300187, upload-time = "2026-07-10T09:38:09.407Z" }, - { url = "https://files.pythonhosted.org/packages/2a/2d/0c9961009c422f366555574af4653f3c13ba73fe075efb31e7af42151af5/blosc2-4.8.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0bb7d94bffdef8304454bbb373b8e9bf41966f8ddb58c449782c649d4d6a395c", size = 6724604, upload-time = "2026-07-10T09:38:16.688Z" }, - { url = "https://files.pythonhosted.org/packages/6d/8e/cb14d6e21fa44255739e2dac8d66ec7fb3d7aee348f198cc943e45d84d76/blosc2-4.8.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec3fc9ad1b2528b471670233e9cf631be104b8526cf4b958a74099f6756c98f6", size = 7217824, upload-time = "2026-07-10T09:38:18.951Z" }, - { url = "https://files.pythonhosted.org/packages/e0/1f/ea276e70d09ed406aba4012a47c7595ed772b25fc096fba8b9eac3494db9/blosc2-4.8.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:da018d11be4ac995aeb7b1a406c5cafce8c9f8a3ca8a540fe872a482de198b0d", size = 2297787, upload-time = "2026-07-10T09:38:20.871Z" }, - { url = "https://files.pythonhosted.org/packages/2f/b7/cba879eed1f510b3f034fbeea7db81cb535c822fe96ac62b5e47012c0b9b/blosc2-4.8.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8574afd38d20f953e9114d7362535ef0019a4c85f2a672e6f3b36dcd00a77f32", size = 6669949, upload-time = "2026-07-10T09:38:27.542Z" }, - { url = "https://files.pythonhosted.org/packages/07/5a/012f58a518bebf0d11931a5a12ba08955aa959cca59764ad38a8d82c9a1b/blosc2-4.8.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f8b459e22059a3a794480df2c72ff008ab6c95b9799fa2979c1dd7830cf44e2", size = 7186485, upload-time = "2026-07-10T09:38:29.411Z" }, -] - -[[package]] -name = "cadcutils" -version = "1.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "distro", marker = "sys_platform == 'linux'" }, - { name = "lxml", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pyopenssl", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, - { name = "setuptools", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fe/44/4964823c72c9e6a93e26de0118cead37fa3c3600b4677f0c06788c903c80/cadcutils-1.6.2.tar.gz", hash = "sha256:6e2d7822756d48a363bc5f857cd717f80aab760554df86518b957f8826a7a906", size = 94673, upload-time = "2026-06-22T17:35:47.303Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/a4/963d9e7f8a2c385a42b77b2791a67f1bff32aa9c48299af2e6cafb8f3a5a/cadcutils-1.6.2-py3-none-any.whl", hash = "sha256:a95323d422731b41f9d24c2f06785ea2231cf69d1cc6ccacc4175df72b988877", size = 164697, upload-time = "2026-06-22T17:35:45.6Z" }, -] - -[[package]] -name = "camb" -version = "1.6.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "sympy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/ad/f2e6446fbd94f7adc1474e1f4e40d2e7fa7e5e7143640cb8557d492f0a73/camb-1.6.6.tar.gz", hash = "sha256:9856202a5c05570256e52377b20431891c7b08b2e9c334e141fd08d2a085516f", size = 799344, upload-time = "2026-03-11T16:46:55.819Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/f6/9131f388386c50982b1b2992424deca857e7c019f4014dab0770de690a84/camb-1.6.6-py3-none-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eded2dd7f19226ed4717535c07a029fc958b7c581549250da33a768e12c87b13", size = 1592029, upload-time = "2026-03-11T16:46:44.925Z" }, - { url = "https://files.pythonhosted.org/packages/41/53/9d0b3cf1a2a07a882adac9565074ac6974fb8b031c31da438d407db22c89/camb-1.6.6-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bb8e0c85b3e33d78ca0676c84b0b60b54dfa72d4e520a805166cab0458e81094", size = 2258716, upload-time = "2026-03-11T16:46:46.458Z" }, -] - -[[package]] -name = "certifi" -version = "2026.6.17" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c9/c7/424b75da314c1045981bd9777432fad05a9e0c69daa4ed7e308bbaffe405/certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", size = 134594, upload-time = "2026-06-17T10:31:07.894Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" }, -] - -[[package]] -name = "cffi" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy' and sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/c0/d1ec30ffb370f748f2fb54425972bfef9871e0132e82fb589c46b6676049/cffi-2.1.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d", size = 214815, upload-time = "2026-07-06T21:32:48.557Z" }, - { url = "https://files.pythonhosted.org/packages/1b/dc/5620cf930688be01f2d673804291de757a934c90b946dbdc3d84130c2ea4/cffi-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7", size = 222429, upload-time = "2026-07-06T21:32:49.848Z" }, - { url = "https://files.pythonhosted.org/packages/4b/a4/77b53abbf7a1e0beb9637edbef2a94d15f9c822f591e85d439ffd91519a6/cffi-2.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b", size = 210315, upload-time = "2026-07-06T21:32:51.221Z" }, - { url = "https://files.pythonhosted.org/packages/58/0c/f528df19cc94b675087324d4760d9e6d5bfae97d6217aa4fac43de4f5fcc/cffi-2.1.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7", size = 208859, upload-time = "2026-07-06T21:32:52.512Z" }, - { url = "https://files.pythonhosted.org/packages/62/f2/c9522a81c32132799a1972c39f5c5f8b4c8b9f00488a23feaa6c06f07741/cffi-2.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66", size = 221844, upload-time = "2026-07-06T21:32:53.704Z" }, - { url = "https://files.pythonhosted.org/packages/6e/28/bd53988b9833e8f8ad539d26f4c07a6b3f6bcb1e9e02e7ca038250b3428d/cffi-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe", size = 225287, upload-time = "2026-07-06T21:32:54.907Z" }, - { url = "https://files.pythonhosted.org/packages/79/99/0d0fd37f055224085f42bbb2c022d002e17dde4a97972822327b07d84101/cffi-2.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b", size = 223681, upload-time = "2026-07-06T21:32:56.329Z" }, - { url = "https://files.pythonhosted.org/packages/96/88/a996879e2eeccb815f6e3a5967b12a308257412acec882039d386bd2aa7b/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda", size = 194331, upload-time = "2026-07-06T21:33:03.697Z" }, - { url = "https://files.pythonhosted.org/packages/58/85/7ae00d5c8dd6266f4e944c3db630f3c5c9a98b61d469c714d848b1d8138a/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b", size = 196966, upload-time = "2026-07-06T21:33:05.353Z" }, - { url = "https://files.pythonhosted.org/packages/a0/1c/4ed5a0e5bdca6cbc275556de3328dd1b76fd0c11cc13c88fe66d1d8715f2/cffi-2.1.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db", size = 214747, upload-time = "2026-07-06T21:33:09.671Z" }, - { url = "https://files.pythonhosted.org/packages/3a/a6/e879bb68cc23a2bc9ba8f4b7d8019f0c2694bad2ab6c4a3701d429439f58/cffi-2.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f", size = 222392, upload-time = "2026-07-06T21:33:10.896Z" }, - { url = "https://files.pythonhosted.org/packages/88/f6/01890cfd63c08f8eb96a8319b0443690197d240a8bd6346048cf7bde9190/cffi-2.1.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d", size = 210285, upload-time = "2026-07-06T21:33:12.251Z" }, - { url = "https://files.pythonhosted.org/packages/a6/cf/2b684132056f438567b61e19d690dd31cd0921ace051e0a458be6074369e/cffi-2.1.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:47ff3a8bfd8cb9da1af7524b965127095055654c177fcfc7578debcb015eecd0", size = 208801, upload-time = "2026-07-06T21:33:13.617Z" }, - { url = "https://files.pythonhosted.org/packages/6f/08/f2e7d62c460faae0926f2d6e423694aa409ced3bc1fe2927a0a6e5f05416/cffi-2.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:799416bae98336e400981ff6e532d67d5c709cfb30afb79865a1315f94b0e224", size = 221808, upload-time = "2026-07-06T21:33:15.466Z" }, - { url = "https://files.pythonhosted.org/packages/38/37/04f54b8e63a02f3d908332c9effbf8c366167c6f733ed8a3d4f79b7e2a1e/cffi-2.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:961be50688f7fba2fa65f63712d3b9b341a22311f5253460ce933f52f0de1c8c", size = 225241, upload-time = "2026-07-06T21:33:16.869Z" }, - { url = "https://files.pythonhosted.org/packages/a9/d6/c72eecca433cd3e681c65ed313ab4835d9d4a379704d0f628a6a05f51c2e/cffi-2.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bf5c6cf48238b0eb4c086978c492ad1cbc22373fc5b2d7353b3a598ce6db887a", size = 223588, upload-time = "2026-07-06T21:33:18.239Z" }, - { url = "https://files.pythonhosted.org/packages/d8/f0/81478e482afa03f6d18dc8f2afb5edc45b3080853b634b5ed91961be0998/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a", size = 194142, upload-time = "2026-07-06T21:33:23.657Z" }, - { url = "https://files.pythonhosted.org/packages/7d/95/8de304305cd9204974b0ca051b86d307cafca13aa575a0ef1b44d92c0d8c/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3", size = 196819, upload-time = "2026-07-06T21:33:25.007Z" }, - { url = "https://files.pythonhosted.org/packages/2e/d2/065fcae1c73979fac8e054462478d0ff8a29c40cdc2ed7ea5676a061df53/cffi-2.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6", size = 222353, upload-time = "2026-07-06T21:33:29.178Z" }, - { url = "https://files.pythonhosted.org/packages/ed/a5/e8bbb1ce5b3ac2f53ad6a10bde44318a5a8d99d4f4a000d44a6e39aeb3e4/cffi-2.1.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913", size = 210051, upload-time = "2026-07-06T21:33:30.534Z" }, - { url = "https://files.pythonhosted.org/packages/28/ed/c127d3ac36e899c965e3361357c3befacd6578c03f40125183e41c3b219e/cffi-2.1.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d", size = 208630, upload-time = "2026-07-06T21:33:31.753Z" }, - { url = "https://files.pythonhosted.org/packages/cc/d7/97d3136f81db489ec8d1d67748c110d6c994268fd7528014aa9f2b085e4e/cffi-2.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d53d10f7da99ae46f7373b9150393e9c5eab9b224909982b43832668de4779f5", size = 221593, upload-time = "2026-07-06T21:33:33.044Z" }, - { url = "https://files.pythonhosted.org/packages/d3/27/93195977168ee63aed233a1a0993a2178798654d1f4bddcdd321d6fd3b21/cffi-2.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c351efb95e832a853a29361675f33a7ce53de1a109cd73fd47af0712213aa4ce", size = 225146, upload-time = "2026-07-06T21:33:34.224Z" }, - { url = "https://files.pythonhosted.org/packages/b3/c1/6dbd291ee2ae5a50a034aa057207081f545923bbf15dad4511e985aafff5/cffi-2.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:dbf7c7a88e2bac086f06d14577332760bdeecc42bdec8ac4077f6260557d9326", size = 223240, upload-time = "2026-07-06T21:33:35.57Z" }, - { url = "https://files.pythonhosted.org/packages/14/d0/117dcd9209255ad8571fbc8c92ef32593a1d294dcec91ddc4e4db50606f2/cffi-2.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98", size = 223899, upload-time = "2026-07-06T21:33:39.514Z" }, - { url = "https://files.pythonhosted.org/packages/b6/3d/f20f8b886b254e3ad10e15cd4186d3aed49f3e6a35ab37aab9f8f25f7c03/cffi-2.1.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13", size = 211652, upload-time = "2026-07-06T21:33:40.851Z" }, - { url = "https://files.pythonhosted.org/packages/28/3b/fad54de07260b93ddeef4b96d0131d57ea900675df1d410ae1deee52d7a6/cffi-2.1.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d", size = 210755, upload-time = "2026-07-06T21:33:42.183Z" }, - { url = "https://files.pythonhosted.org/packages/cc/82/3d5c705acb7abbba9bbd7d79b8e62e0f25b6120eb7ae6ac49f1b721722fe/cffi-2.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ac0f1a2d0cfa7eea3f2aaf006ab6e70e8feeb16b75d65b7e5939982ca2f11056", size = 223933, upload-time = "2026-07-06T21:33:43.603Z" }, - { url = "https://files.pythonhosted.org/packages/6c/d0/47e338384ab6b1004241002fa616301020cea4fc95f283506565d252f276/cffi-2.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c16914df9fb7f500e440e6875fa23ff5e0b31db01fa9c06af98d59a91f0dc2e4", size = 226749, upload-time = "2026-07-06T21:33:45.046Z" }, - { url = "https://files.pythonhosted.org/packages/70/25/65bd5b58ea4bfdfc15cde02cb5365f89ef8ab8b2adfb8fe5c4bd4233382f/cffi-2.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5ecbd0499275d57506d397eebe1981cee87b47fcd9ef5c22cab7ed7644a39a94", size = 225703, upload-time = "2026-07-06T21:33:46.374Z" }, - { url = "https://files.pythonhosted.org/packages/55/c7/8c8c50cb11c6750051daf12164098a9a6f027ac4356967fd4d800a07f242/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c", size = 194121, upload-time = "2026-07-06T21:33:56.109Z" }, - { url = "https://files.pythonhosted.org/packages/99/e2/67680bf19a6b60d2bb7ff83baefa2a4c3d2d7dc0f3277034b802e1fc504c/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001", size = 196820, upload-time = "2026-07-06T21:33:57.288Z" }, - { url = "https://files.pythonhosted.org/packages/ef/c3/ad299dc38f3583f8d916b299f028af418a9ec98bc695fcbebeae7420691c/cffi-2.1.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699", size = 222342, upload-time = "2026-07-06T21:34:01.814Z" }, - { url = "https://files.pythonhosted.org/packages/eb/d8/df4543cc087245044ed02ef3ad8e0a26619d0075ac7a77a12dc81177851b/cffi-2.1.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022", size = 210073, upload-time = "2026-07-06T21:34:03.255Z" }, - { url = "https://files.pythonhosted.org/packages/2c/0e/fac738d73728c6cea2a88a2883dca54892496cbba88a1dc1f2909cb8a6f5/cffi-2.1.0-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0", size = 208551, upload-time = "2026-07-06T21:34:04.433Z" }, - { url = "https://files.pythonhosted.org/packages/e6/3f/0b04a700dd64f465c93020253a793a82c9b4dff9961f48facd0df945d9b8/cffi-2.1.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d3538f9c0e50670f4deb93dbb696576e60590369cae2faf7de681e597a8a1f1", size = 221649, upload-time = "2026-07-06T21:34:06.157Z" }, - { url = "https://files.pythonhosted.org/packages/5d/7c/b7379a5704c79eda57ce075869ba70a0368d1c850f803b3c0d078d39dcaf/cffi-2.1.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:8f9ec95b8a043d3dfbc74d9abc6f7baf524dd27a8dc160b0a32ff9cdab650c28", size = 225203, upload-time = "2026-07-06T21:34:07.489Z" }, - { url = "https://files.pythonhosted.org/packages/5a/02/d5e6c43ea85c41bda2a184a3418f195fe7cf602967a8d2b94e085b83deef/cffi-2.1.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:af5e2915d41fe6c961694d7bfdc8562942638200f3ce2765dfb8b745cf997629", size = 223263, upload-time = "2026-07-06T21:34:08.712Z" }, - { url = "https://files.pythonhosted.org/packages/e0/27/1d0b408497e41a74795af122d7b603c418c5fed0171450f899afd04e594f/cffi-2.1.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d", size = 223904, upload-time = "2026-07-06T21:34:12.606Z" }, - { url = "https://files.pythonhosted.org/packages/8b/31/e115c985105dd7ffb32444505f18ceb874bb42d992af05d5dced7ecf1980/cffi-2.1.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8", size = 211554, upload-time = "2026-07-06T21:34:13.987Z" }, - { url = "https://files.pythonhosted.org/packages/5a/67/9e6e09409336d9e515c58367e7cfcf4f89df06ad25252675595a58eb59d5/cffi-2.1.0-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd", size = 210795, upload-time = "2026-07-06T21:34:15.972Z" }, - { url = "https://files.pythonhosted.org/packages/19/e5/d3cc82a4a0be7902af279c04181ad038449c096734464a5ae1de3e1401bd/cffi-2.1.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0611e7ebf90573a535ebdc33ae9da222d037853983e13359f580fab781ca017f", size = 223843, upload-time = "2026-07-06T21:34:17.509Z" }, - { url = "https://files.pythonhosted.org/packages/b9/65/b434abc97ce7cecc2c640fde160507c0ecc7e21544b483ba3325d2e2ea17/cffi-2.1.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:86cf8755a791f72c85dc287128cc62d4f24d392e3f1e15837245623f4a33cccc", size = 226773, upload-time = "2026-07-06T21:34:19.05Z" }, - { url = "https://files.pythonhosted.org/packages/b5/9f/d4dc66ca651eb1145a133314cda721abf13cfac3d28c4a0402263ae6ad75/cffi-2.1.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:ba00f661f8ba35d075c937174e27c2c421cec3942fd2e0ea3e66996757c0fdd9", size = 225719, upload-time = "2026-07-06T21:34:20.576Z" }, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, - { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, - { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, - { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, - { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, - { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, - { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, - { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, - { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, - { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" }, - { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" }, - { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" }, - { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" }, - { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" }, - { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" }, - { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" }, - { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" }, - { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" }, - { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" }, - { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" }, - { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" }, - { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" }, - { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" }, - { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" }, - { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" }, - { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" }, - { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" }, - { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" }, - { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" }, - { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" }, - { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" }, - { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" }, - { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" }, - { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" }, - { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" }, - { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" }, - { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, -] - -[[package]] -name = "click" -version = "8.4.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, -] - -[[package]] -name = "clmm" -version = "1.16.10" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "healpy", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "qp-prob", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/db/73/7ff76d487cdbcddb9c80ec03313db1b11f48fe4fcb5e1609b6eca5a73159/clmm-1.16.10.tar.gz", hash = "sha256:48273784419c038a89619b798a140153ad1e451390ff56074f3b6825c91ebe32", size = 72475, upload-time = "2025-12-04T17:30:11.47Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/a6/2a730fd1b23bb67c31c83449b12e74d6a3369681e63a098f02e353bdb77f/clmm-1.16.10-py3-none-any.whl", hash = "sha256:97e70d1bb7f45f994ab19c07b405688d984bd28399d6e31187012e8a0bf2feb6", size = 89434, upload-time = "2025-12-04T17:30:10.009Z" }, -] - -[[package]] -name = "cloudpickle" -version = "3.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, -] - -[[package]] -name = "colorama" -version = "0.4.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, -] - -[[package]] -name = "comm" -version = "0.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4c/13/7d740c5849255756bc17888787313b61fd38a0a8304fc4f073dfc46122aa/comm-0.2.3.tar.gz", hash = "sha256:2dc8048c10962d55d7ad693be1e7045d891b7ce8d999c97963a5e3e99c055971", size = 6319, upload-time = "2025-07-25T14:02:04.452Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, -] - -[[package]] -name = "conda-inject" -version = "1.3.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/a8/8dc86113c65c949cc72d651461d6e4c544b3302a85ed14a5298829e6a419/conda_inject-1.3.2.tar.gz", hash = "sha256:0b8cde8c47998c118d8ff285a04977a3abcf734caf579c520fca469df1cd0aac", size = 3635, upload-time = "2024-05-27T12:20:58.873Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/4c/fc30b69fb4062aee57e3ab7ff493647c4220144908f0839c619f912045bf/conda_inject-1.3.2-py3-none-any.whl", hash = "sha256:6e641b408980c2814e3e527008c30749117909a21ff47392f07ef807da93a564", size = 4133, upload-time = "2024-05-27T12:20:57.332Z" }, -] - -[[package]] -name = "configargparse" -version = "1.7.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3f/0b/30328302903c55218ffc5199646d0e9d28348ff26c02ba77b2ffc58d294a/configargparse-1.7.5.tar.gz", hash = "sha256:e3f9a7bb6be34d66b2e3c4a2f58e3045f8dfae47b0dc039f87bcfaa0f193fb0f", size = 53548, upload-time = "2026-03-11T02:19:38.144Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/19/3ba5e1b0bcc7b91aeab6c258afd70e4907d220fed3972febe38feb40db30/configargparse-1.7.5-py3-none-any.whl", hash = "sha256:1e63fdffedf94da9cd435fc13a1cd24777e76879dd2343912c1f871d4ac8c592", size = 27692, upload-time = "2026-03-11T02:19:36.442Z" }, -] - -[[package]] -name = "connection-pool" -version = "0.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/df/c9b4e25dce00f6349fd28aadba7b6c3f7431cc8bd4308a158fbe57b6a22e/connection_pool-0.0.3.tar.gz", hash = "sha256:bf429e7aef65921c69b4ed48f3d48d3eac1383b05d2df91884705842d974d0dc", size = 3795, upload-time = "2020-09-17T02:48:28.824Z" } - -[[package]] -name = "contourpy" -version = "1.3.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/1c/a12359b9b2ca3a845e8f7f9ac08bdf776114eb931392fcad91743e2ea17b/contourpy-1.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7", size = 332653, upload-time = "2025-07-26T12:01:24.155Z" }, - { url = "https://files.pythonhosted.org/packages/63/12/897aeebfb475b7748ea67b61e045accdfcf0d971f8a588b67108ed7f5512/contourpy-1.3.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8", size = 379536, upload-time = "2025-07-26T12:01:25.91Z" }, - { url = "https://files.pythonhosted.org/packages/43/8a/a8c584b82deb248930ce069e71576fc09bd7174bbd35183b7943fb1064fd/contourpy-1.3.3-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea", size = 384397, upload-time = "2025-07-26T12:01:27.152Z" }, - { url = "https://files.pythonhosted.org/packages/cc/8f/ec6289987824b29529d0dfda0d74a07cec60e54b9c92f3c9da4c0ac732de/contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1", size = 362601, upload-time = "2025-07-26T12:01:28.808Z" }, - { url = "https://files.pythonhosted.org/packages/05/0a/a3fe3be3ee2dceb3e615ebb4df97ae6f3828aa915d3e10549ce016302bd1/contourpy-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7", size = 1331288, upload-time = "2025-07-26T12:01:31.198Z" }, - { url = "https://files.pythonhosted.org/packages/33/1d/acad9bd4e97f13f3e2b18a3977fe1b4a37ecf3d38d815333980c6c72e963/contourpy-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411", size = 1403386, upload-time = "2025-07-26T12:01:33.947Z" }, - { url = "https://files.pythonhosted.org/packages/73/23/90e31ceeed1de63058a02cb04b12f2de4b40e3bef5e082a7c18d9c8ae281/contourpy-1.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:348ac1f5d4f1d66d3322420f01d42e43122f43616e0f194fc1c9f5d830c5b286", size = 334672, upload-time = "2025-07-26T12:01:41.942Z" }, - { url = "https://files.pythonhosted.org/packages/ed/93/b43d8acbe67392e659e1d984700e79eb67e2acb2bd7f62012b583a7f1b55/contourpy-1.3.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:655456777ff65c2c548b7c454af9c6f33f16c8884f11083244b5819cc214f1b5", size = 381234, upload-time = "2025-07-26T12:01:43.499Z" }, - { url = "https://files.pythonhosted.org/packages/46/3b/bec82a3ea06f66711520f75a40c8fc0b113b2a75edb36aa633eb11c4f50f/contourpy-1.3.3-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:644a6853d15b2512d67881586bd03f462c7ab755db95f16f14d7e238f2852c67", size = 385169, upload-time = "2025-07-26T12:01:45.219Z" }, - { url = "https://files.pythonhosted.org/packages/4b/32/e0f13a1c5b0f8572d0ec6ae2f6c677b7991fafd95da523159c19eff0696a/contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4debd64f124ca62069f313a9cb86656ff087786016d76927ae2cf37846b006c9", size = 362859, upload-time = "2025-07-26T12:01:46.519Z" }, - { url = "https://files.pythonhosted.org/packages/33/71/e2a7945b7de4e58af42d708a219f3b2f4cff7386e6b6ab0a0fa0033c49a9/contourpy-1.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a15459b0f4615b00bbd1e91f1b9e19b7e63aea7483d03d804186f278c0af2659", size = 1332062, upload-time = "2025-07-26T12:01:48.964Z" }, - { url = "https://files.pythonhosted.org/packages/12/fc/4e87ac754220ccc0e807284f88e943d6d43b43843614f0a8afa469801db0/contourpy-1.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca0fdcd73925568ca027e0b17ab07aad764be4706d0a925b89227e447d9737b7", size = 1403932, upload-time = "2025-07-26T12:01:51.979Z" }, - { url = "https://files.pythonhosted.org/packages/ae/15/e59f5f3ffdd6f3d4daa3e47114c53daabcb18574a26c21f03dc9e4e42ff0/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e4e6b05a45525357e382909a4c1600444e2a45b4795163d3b22669285591c1ae", size = 326751, upload-time = "2025-07-26T12:02:00.343Z" }, - { url = "https://files.pythonhosted.org/packages/0f/81/03b45cfad088e4770b1dcf72ea78d3802d04200009fb364d18a493857210/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab3074b48c4e2cf1a960e6bbeb7f04566bf36b1861d5c9d4d8ac04b82e38ba20", size = 375486, upload-time = "2025-07-26T12:02:02.128Z" }, - { url = "https://files.pythonhosted.org/packages/0c/ba/49923366492ffbdd4486e970d421b289a670ae8cf539c1ea9a09822b371a/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c3d53c796f8647d6deb1abe867daeb66dcc8a97e8455efa729516b997b8ed99", size = 388106, upload-time = "2025-07-26T12:02:03.615Z" }, - { url = "https://files.pythonhosted.org/packages/9f/52/5b00ea89525f8f143651f9f03a0df371d3cbd2fccd21ca9b768c7a6500c2/contourpy-1.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50ed930df7289ff2a8d7afeb9603f8289e5704755c7e5c3bbd929c90c817164b", size = 352548, upload-time = "2025-07-26T12:02:05.165Z" }, - { url = "https://files.pythonhosted.org/packages/32/1d/a209ec1a3a3452d490f6b14dd92e72280c99ae3d1e73da74f8277d4ee08f/contourpy-1.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4feffb6537d64b84877da813a5c30f1422ea5739566abf0bd18065ac040e120a", size = 1322297, upload-time = "2025-07-26T12:02:07.379Z" }, - { url = "https://files.pythonhosted.org/packages/bc/9e/46f0e8ebdd884ca0e8877e46a3f4e633f6c9c8c4f3f6e72be3fe075994aa/contourpy-1.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2b7e9480ffe2b0cd2e787e4df64270e3a0440d9db8dc823312e2c940c167df7e", size = 1391023, upload-time = "2025-07-26T12:02:10.171Z" }, - { url = "https://files.pythonhosted.org/packages/b1/71/f93e1e9471d189f79d0ce2497007731c1e6bf9ef6d1d61b911430c3db4e5/contourpy-1.3.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22e9b1bd7a9b1d652cd77388465dc358dafcd2e217d35552424aa4f996f524f5", size = 335810, upload-time = "2025-07-26T12:02:18.9Z" }, - { url = "https://files.pythonhosted.org/packages/91/f9/e35f4c1c93f9275d4e38681a80506b5510e9327350c51f8d4a5a724d178c/contourpy-1.3.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a22738912262aa3e254e4f3cb079a95a67132fc5a063890e224393596902f5a4", size = 382871, upload-time = "2025-07-26T12:02:20.418Z" }, - { url = "https://files.pythonhosted.org/packages/b5/71/47b512f936f66a0a900d81c396a7e60d73419868fba959c61efed7a8ab46/contourpy-1.3.3-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:afe5a512f31ee6bd7d0dda52ec9864c984ca3d66664444f2d72e0dc4eb832e36", size = 386264, upload-time = "2025-07-26T12:02:21.916Z" }, - { url = "https://files.pythonhosted.org/packages/04/5f/9ff93450ba96b09c7c2b3f81c94de31c89f92292f1380261bd7195bea4ea/contourpy-1.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f64836de09927cba6f79dcd00fdd7d5329f3fccc633468507079c829ca4db4e3", size = 363819, upload-time = "2025-07-26T12:02:23.759Z" }, - { url = "https://files.pythonhosted.org/packages/3e/a6/0b185d4cc480ee494945cde102cb0149ae830b5fa17bf855b95f2e70ad13/contourpy-1.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1fd43c3be4c8e5fd6e4f2baeae35ae18176cf2e5cced681cca908addf1cdd53b", size = 1333650, upload-time = "2025-07-26T12:02:26.181Z" }, - { url = "https://files.pythonhosted.org/packages/43/d7/afdc95580ca56f30fbcd3060250f66cedbde69b4547028863abd8aa3b47e/contourpy-1.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6afc576f7b33cf00996e5c1102dc2a8f7cc89e39c0b55df93a0b78c1bd992b36", size = 1404833, upload-time = "2025-07-26T12:02:28.782Z" }, - { url = "https://files.pythonhosted.org/packages/66/06/8a475c8ab718ebfd7925661747dbb3c3ee9c82ac834ccb3570be49d129f4/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d304906ecc71672e9c89e87c4675dc5c2645e1f4269a5063b99b0bb29f232d13", size = 326748, upload-time = "2025-07-26T12:02:37.193Z" }, - { url = "https://files.pythonhosted.org/packages/b4/a3/c5ca9f010a44c223f098fccd8b158bb1cb287378a31ac141f04730dc49be/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ca658cd1a680a5c9ea96dc61cdbae1e85c8f25849843aa799dfd3cb370ad4fbe", size = 375554, upload-time = "2025-07-26T12:02:38.894Z" }, - { url = "https://files.pythonhosted.org/packages/80/5b/68bd33ae63fac658a4145088c1e894405e07584a316738710b636c6d0333/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ab2fd90904c503739a75b7c8c5c01160130ba67944a7b77bbf36ef8054576e7f", size = 388118, upload-time = "2025-07-26T12:02:40.642Z" }, - { url = "https://files.pythonhosted.org/packages/40/52/4c285a6435940ae25d7410a6c36bda5145839bc3f0beb20c707cda18b9d2/contourpy-1.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7301b89040075c30e5768810bc96a8e8d78085b47d8be6e4c3f5a0b4ed478a0", size = 352555, upload-time = "2025-07-26T12:02:42.25Z" }, - { url = "https://files.pythonhosted.org/packages/24/ee/3e81e1dd174f5c7fefe50e85d0892de05ca4e26ef1c9a59c2a57e43b865a/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2a2a8b627d5cc6b7c41a4beff6c5ad5eb848c88255fda4a8745f7e901b32d8e4", size = 1322295, upload-time = "2025-07-26T12:02:44.668Z" }, - { url = "https://files.pythonhosted.org/packages/3c/b2/6d913d4d04e14379de429057cd169e5e00f6c2af3bb13e1710bcbdb5da12/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fd6ec6be509c787f1caf6b247f0b1ca598bef13f4ddeaa126b7658215529ba0f", size = 1391027, upload-time = "2025-07-26T12:02:47.09Z" }, -] - -[[package]] -name = "cosmo-numba" -version = "1.0.1.dev6+g188d272c6" -source = { git = "https://github.com/aguinot/cosmo-numba.git?rev=main#188d272c67d6d699d7cff7a7a53ded8dec759bb0" } -dependencies = [ - { name = "mpmath", marker = "sys_platform == 'linux'" }, - { name = "numba", marker = "sys_platform == 'linux'" }, - { name = "numbaquadpack", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "rocket-fft", marker = "sys_platform == 'linux'" }, - { name = "sympy", marker = "sys_platform == 'linux'" }, -] - -[[package]] -name = "cosmology" -version = "2022.10.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c9/38/04099bb2b6a626bd69a3f117afc60f2c47c08899e0730e0e48c898bf4745/cosmology-2022.10.9.tar.gz", hash = "sha256:0c2857c9bf1fdd09f1f11ab5765df0389a4101f3a900fa11251c3f37696f02d4", size = 8488, upload-time = "2022-10-10T10:18:16.229Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/2c/c80db37593ad593e3f6c60705fbdf690b33f22c89edd5507d2731346cd87/cosmology-2022.10.9-py3-none-any.whl", hash = "sha256:3903658c2474177a1a1c75771ae14458d93200c516f8fc6c3f4d776f3287b288", size = 9341, upload-time = "2022-10-10T10:18:14.928Z" }, -] - -[[package]] -name = "coverage" -version = "7.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/8b/adeb62ea8951f13c4c7fef2e7a85e1a06b499c8d8237ea589d496029e53f/coverage-7.15.0.tar.gz", hash = "sha256:9ac3fe7a1435986463eaa8ee253ae2f2a268709ba4ae5c7dd1f52a05391ad78f", size = 925362, upload-time = "2026-07-02T13:10:50.535Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/17/99fa688541ae1d6e84543a0e544f83de0c944815b63e9e7b1ed411d15036/coverage-7.15.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e4d0bb73455bf97ab243a8f12c37c686ccf1c13bb614b7b85f1d062f06f42b2c", size = 252705, upload-time = "2026-07-02T13:09:05.059Z" }, - { url = "https://files.pythonhosted.org/packages/fb/02/6a95a5cd83b74839017ef9cf48d2d8c9ae60af919e17a3f336e6f9f1b7bd/coverage-7.15.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:20d9ccc4ebd0edc434d86dfd2a1dd2a8efa6b6b3073d0485a394fee86459ebb4", size = 255441, upload-time = "2026-07-02T13:09:06.559Z" }, - { url = "https://files.pythonhosted.org/packages/67/f2/406f6c57d600f68185942422c4c00f1a3255d60aee6e5fd961425cd9987e/coverage-7.15.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20c8a976c365c8cb12f0cbd099508772ea41fb5fa80657a8506df0e11bd278c5", size = 256556, upload-time = "2026-07-02T13:09:08.197Z" }, - { url = "https://files.pythonhosted.org/packages/74/8e/d3fa48489c15ecdec1ba48fd61f68798555dddd2f6716f9ad42adeb1a2a9/coverage-7.15.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f948fd5ba1b9cbca91f0ae08b4c1ce2b139509149a435e2585d056d57d70bf01", size = 258815, upload-time = "2026-07-02T13:09:09.691Z" }, - { url = "https://files.pythonhosted.org/packages/47/2e/2d40ddd110462c6a2769677cf7f1c119a52b45f568978fc6c98e4cc0dd0f/coverage-7.15.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f58185f06edf6ad68ec9fb155d63ef650c82f3fbd7e1770e2867751fb13158f4", size = 253117, upload-time = "2026-07-02T13:09:11.212Z" }, - { url = "https://files.pythonhosted.org/packages/51/c0/310782f0d7c3cb2b5ac05ba8d205fe91f24a36f6bf3256098f1782181c38/coverage-7.15.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:02adc79a920c73c647c5d117f55747df7f2de94571884758ce8bc58e04f0a796", size = 254475, upload-time = "2026-07-02T13:09:13.029Z" }, - { url = "https://files.pythonhosted.org/packages/86/f7/702da6c275f8ae6ade423d2877243122932c9b27f5403003b9ef8c927d12/coverage-7.15.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6eb7c300fbed667fd6e3588eba71c1904cdb06110ca6fdf908c26bdd88b8e382", size = 252619, upload-time = "2026-07-02T13:09:14.699Z" }, - { url = "https://files.pythonhosted.org/packages/fb/84/c5b15a7e5ecba4e56218d772d99fe80a63e63f8d11f12783723a6005ab45/coverage-7.15.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:b5fb23fa2de9dce1f5c36c09066d8fcda16cd96e8e26686caa2d7cb9b567d65c", size = 256689, upload-time = "2026-07-02T13:09:16.103Z" }, - { url = "https://files.pythonhosted.org/packages/95/2f/c8b07559b57701230c61b23a953858c052890c12ef568d81780c6c46e92e/coverage-7.15.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cec79341dbe6281484024979976d0c7f22beae08b4a254655decd25d42cbe766", size = 252189, upload-time = "2026-07-02T13:09:17.828Z" }, - { url = "https://files.pythonhosted.org/packages/6b/80/6d2f049dd3fd3dbfd60b62ba6b2162a04009e2c002ce70b24cf3878dec7a/coverage-7.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6c664c5444b1d970b1b2a450e21fb19ee5c9cfdf151ded2dda37260031cca0da", size = 254059, upload-time = "2026-07-02T13:09:19.304Z" }, - { url = "https://files.pythonhosted.org/packages/7b/19/3a80b97d3b2a5c77a01ae359c6bed20c13738fe3d9380f08616d4fec0281/coverage-7.15.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bbcbb317c2e5ded5b21104af81c29f391be2af98d065693ffbe8d23949b948e5", size = 252227, upload-time = "2026-07-02T13:09:28.543Z" }, - { url = "https://files.pythonhosted.org/packages/a1/fa/b70062750686bd7da454da27927622f48bbac6990ac7a4c4a4653e7b0036/coverage-7.15.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:27f31ecb458da3f859aab3f15ada871eb7a7768807d88df4a9f186bb17737970", size = 254823, upload-time = "2026-07-02T13:09:30.177Z" }, - { url = "https://files.pythonhosted.org/packages/a9/09/dad6a75a2e561b9dc5086a8c5257a7591d584246f67e23e70d2995b89ab6/coverage-7.15.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fb759be317fdc62e0f56bffdf61cfcb45c7761ad6b71e3e583e71a67ae753c", size = 256059, upload-time = "2026-07-02T13:09:31.979Z" }, - { url = "https://files.pythonhosted.org/packages/e6/e7/b5d2941fa9564573d44b693a871ff3156f0c42cbefe977a09fa7fdc59971/coverage-7.15.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5cf007add5ab4bb8fa9f4c77e3732127c9e6cad501d7db43355fbfafca0be84", size = 258190, upload-time = "2026-07-02T13:09:34.035Z" }, - { url = "https://files.pythonhosted.org/packages/7c/1d/8e895bcde3c57ccd46d896dda5f2b3d5df761a1b0c6c9d450d175dedc632/coverage-7.15.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc78d9843bd576fbe2118248258d485e968dc535f95ed504a7b0867ba9b51389", size = 252456, upload-time = "2026-07-02T13:09:35.765Z" }, - { url = "https://files.pythonhosted.org/packages/14/4c/f6997da343ddeb959be82c3b05322793f92c071ad45f7cb8a96336e2dd5f/coverage-7.15.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a263060f1de0b4b74b4e089c2a70b8003b3781c733329a9c8fd54995328f9950", size = 254192, upload-time = "2026-07-02T13:09:37.445Z" }, - { url = "https://files.pythonhosted.org/packages/17/27/a0bc09d032267b9da89d95a2d874cfbef2a5aebbf0e87cf7aba221d79a99/coverage-7.15.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c48decf16e0dfd5b049c7d5e82200c23c08126719142998d4f172444e3d0529e", size = 252153, upload-time = "2026-07-02T13:09:39.422Z" }, - { url = "https://files.pythonhosted.org/packages/54/c0/77fc233d9fba07b244c40948c53fe27308b8f21732fb3417f87fbd6fd992/coverage-7.15.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:08fb028000ed0aaa0a4cbdfbb98be7cb42f370db973fbbb469733505ab20e13e", size = 256310, upload-time = "2026-07-02T13:09:41.006Z" }, - { url = "https://files.pythonhosted.org/packages/d5/24/601cecfb5825becacb8d45219a018a3b55b9dbaec624efdb0ea249d08be2/coverage-7.15.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb7dc0c3b7d8a1077abea0b8546ebc5e26d6ef6ecefc2f0f5ad2b8a53bdad837", size = 251974, upload-time = "2026-07-02T13:09:42.733Z" }, - { url = "https://files.pythonhosted.org/packages/47/1e/6f45e5a5b3d5484318d368702af6716b5ab8913b0428bec981a562fcf296/coverage-7.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6cb3602054ccbe9f0d8c2dc04bbeba90d5719236e2cd06e042ddd6d3fc7b6e37", size = 253745, upload-time = "2026-07-02T13:09:44.376Z" }, - { url = "https://files.pythonhosted.org/packages/a7/ce/22bae91e0b75445f68d365c7643ed0aa4880bbf77450ee74ca65bdae53a7/coverage-7.15.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:769e8ece11a596315ebf5aa7ec383aeeed016c091d2bf6363ffb996d41529092", size = 252286, upload-time = "2026-07-02T13:09:54.996Z" }, - { url = "https://files.pythonhosted.org/packages/dd/1e/bec5e32aa508615d9d7a2790effb25fb4dc28606e995816afe400b25ece3/coverage-7.15.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:65a6b6164ee5c39e2f3803f314292d6c61a607ba7fee253d1e03c42dc3903502", size = 254789, upload-time = "2026-07-02T13:09:56.678Z" }, - { url = "https://files.pythonhosted.org/packages/17/29/0e865435b4354e4a7c03b1b7920046d31d0a273d55decefea27e011cb9bf/coverage-7.15.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:75128817f95a5c45bb01d65fd2d8b9cb54bbe03d81608fb70e3e14b437ad56c2", size = 256135, upload-time = "2026-07-02T13:09:58.343Z" }, - { url = "https://files.pythonhosted.org/packages/84/ff/33a870b58a13325d62fc0a6c8f01fa0ff667cef60c7498e2382a147dfa18/coverage-7.15.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9887bb428fe2d4cd4bee89bac1a6c9932f484afd5b36fbd4ff6ea5f825bb1f5e", size = 258449, upload-time = "2026-07-02T13:10:00.057Z" }, - { url = "https://files.pythonhosted.org/packages/18/7b/6fffe596bf3ddba8462758d02c5dad730fd91055a6634aa2e4226229181a/coverage-7.15.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0bfc0be1f702042207a93a00523b1065ee1fe951e96edf311581c0bbc2e34888", size = 252313, upload-time = "2026-07-02T13:10:01.946Z" }, - { url = "https://files.pythonhosted.org/packages/58/1b/11468dd6c1676ab831a70cb9a8d4e198e8607fa0b7220ab918b73fe9bfbd/coverage-7.15.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f64627d55def5a43282d70e08396672692f77e4da610a5bb8bb4060b432b6859", size = 254142, upload-time = "2026-07-02T13:10:04.065Z" }, - { url = "https://files.pythonhosted.org/packages/79/41/29328e21d16b1b95092c30dd700e08cf915bd3734f836df8f3bdb0e8fa9f/coverage-7.15.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:2c6f0fa473003905c6d5bac328ee4eba9fbea654f15bc24b8a3274b23363fa99", size = 252108, upload-time = "2026-07-02T13:10:06.11Z" }, - { url = "https://files.pythonhosted.org/packages/9b/de/05ccfb990439655b35afbfd8e0d13fe66677565a7d4eb38c3f5ef2635e1c/coverage-7.15.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2bcf9afaf064172c6ec3c58a325a9957ad1178c05dd934e25f253321776e0676", size = 256385, upload-time = "2026-07-02T13:10:08.141Z" }, - { url = "https://files.pythonhosted.org/packages/51/0e/486828a3d2695ea7a2609f17ff572f6b01905e608379440a11da4b8dffbe/coverage-7.15.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:baf06bc987115d6fb938d403f7eab684a057766c490367999a2b71a6883110c6", size = 251923, upload-time = "2026-07-02T13:10:10.179Z" }, - { url = "https://files.pythonhosted.org/packages/18/c7/03582b6715f078e5e558354c87616d945b9894cda2dace8e4009b17035e4/coverage-7.15.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0405f2ff97b1c4c0e782cb32e02f32369bcf2e6b618b591d67e1ea754575dfe", size = 253580, upload-time = "2026-07-02T13:10:12.052Z" }, - { url = "https://files.pythonhosted.org/packages/6d/b4/c0ca3028f42c9a08e51feb4561ef1192e5de99797cd1db5b04590c215bda/coverage-7.15.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:94c9686bfe8a9a6810297aecbd99beaa3445f9e8dc2f80b1382cca0d86b64461", size = 263267, upload-time = "2026-07-02T13:10:23.261Z" }, - { url = "https://files.pythonhosted.org/packages/5f/aa/a375e3846e5d3c013dc600b2a3231089055c73d77f5393dd2192a8d64da6/coverage-7.15.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9bd671c25f9d85f09d7ec481d0e43d5139f486c06a37139847a7ce569788af72", size = 265390, upload-time = "2026-07-02T13:10:25.152Z" }, - { url = "https://files.pythonhosted.org/packages/92/e1/5783cdabb797305e1c9e4809fea496d31834c51fa772514f73dc148bcfc9/coverage-7.15.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:110cbdf8d2e216577312cf06ccf85539c0e5a5420ef747e4a4719b5e483c88cd", size = 267811, upload-time = "2026-07-02T13:10:27.249Z" }, - { url = "https://files.pythonhosted.org/packages/85/31/96d8bbf58b8e9193bc8389574a91a0db48355ee98feb66aa6bf8d1b32eea/coverage-7.15.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c5d4619214f1d9993e7b00a8600d14614b7e9d84e89507460b126aa5e6559e5", size = 268928, upload-time = "2026-07-02T13:10:29.242Z" }, - { url = "https://files.pythonhosted.org/packages/5e/7a/5294567e811a1cb7eda93140c628fa050d66189da28da320f93d1d815c73/coverage-7.15.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:781a704516e2d8346fbbd5be6c6f3412dd824785146528b3a01816f26c081007", size = 262378, upload-time = "2026-07-02T13:10:31.107Z" }, - { url = "https://files.pythonhosted.org/packages/69/3f/3f48538421f899f28946f90a3d272136a4686e1abf461cc9249a783ee0f3/coverage-7.15.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd4a1b44bcb65ee29e947ac92bbee04956df3a6bfc6143641bb6cae7ede00fc9", size = 265263, upload-time = "2026-07-02T13:10:32.942Z" }, - { url = "https://files.pythonhosted.org/packages/ce/d3/092df15efcab8a9c1467ee960eb8019bbad3f9300d115d89ea6195f369ff/coverage-7.15.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0e4950c9d6d3e39c64c991814ff315e2d0b9cb8152363594212c9e55208c0a8f", size = 262866, upload-time = "2026-07-02T13:10:35.104Z" }, - { url = "https://files.pythonhosted.org/packages/e5/ab/0254d2b88665efb2c57ad368cc77ab5de3435bd8d5add4729c1b0e79431e/coverage-7.15.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:fe9c87ff42e5472d80d21704972e1f96e104a0a599d77c5e35db5a3c562e2571", size = 266599, upload-time = "2026-07-02T13:10:37.05Z" }, - { url = "https://files.pythonhosted.org/packages/a8/79/1cfa4023e489ce6fbc7be4a5d442dbc375edb4f4fda39a352cedb53263c2/coverage-7.15.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f00d5ae1dd2fe13fb8186e3e7d37bcbd8b25c0d764ff7d1b32cef9be058510a8", size = 261714, upload-time = "2026-07-02T13:10:38.966Z" }, - { url = "https://files.pythonhosted.org/packages/b7/eb/fee5c8665656be63f497418d410484637c438172568688e8ac92e06574e7/coverage-7.15.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:363ab38cc78b615f11c9cac3cf1d7eef950c18b9fdedfb9066f59461dcf84d68", size = 264025, upload-time = "2026-07-02T13:10:40.789Z" }, - { url = "https://files.pythonhosted.org/packages/52/30/21b2ad45959cd50e909e02ebac1e30b4ceb7162e91c11d4c570223a458b7/coverage-7.15.0-py3-none-any.whl", hash = "sha256:56da6a4cbe8f7e9e80bd072ca9cefe67d7106a440a7ec06519ec6507ac94ad19", size = 212632, upload-time = "2026-07-02T13:10:48.641Z" }, -] - -[[package]] -name = "cryptography" -version = "49.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1f/99/d1c90d6041656cc6ee229dc99cd67fd0cd5aec3c5f7d72fffc27cc750054/cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493", size = 854345, upload-time = "2026-06-12T20:02:30.512Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/09/41/3797cfaf69cae04a13ee78ebd83f0678d9c02b4779d21ce24445326f1a69/cryptography-49.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db", size = 4692978, upload-time = "2026-06-12T20:01:21.305Z" }, - { url = "https://files.pythonhosted.org/packages/e6/8b/43011f7ebe515a8aa20d61f290a326cd890c2e738e16e59eaff8d9c3a412/cryptography-49.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325", size = 4716422, upload-time = "2026-06-12T20:01:48.566Z" }, - { url = "https://files.pythonhosted.org/packages/4a/91/01ce7303a4579e6d3a6abef01bd322848e9ea7a219adcabc5048b9033571/cryptography-49.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2", size = 4700503, upload-time = "2026-06-12T20:02:47.091Z" }, - { url = "https://files.pythonhosted.org/packages/62/99/a2c95cf8293f07491e9e27c20cc4dcd18176d944e674679adeb1d0173fd6/cryptography-49.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b", size = 5309779, upload-time = "2026-06-12T20:02:08.987Z" }, - { url = "https://files.pythonhosted.org/packages/20/2c/0622f20ff02b2ef32558733443805dc82fd4c275be01b2d19d14676f3a1b/cryptography-49.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6", size = 4749683, upload-time = "2026-06-12T20:02:03.335Z" }, - { url = "https://files.pythonhosted.org/packages/a3/5b/c5246635d5fd3b64e0d45ae10e99fd32fe9676a79915ccfe5a61ba9af1a5/cryptography-49.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c", size = 4337874, upload-time = "2026-06-12T20:02:54.323Z" }, - { url = "https://files.pythonhosted.org/packages/6d/88/05563c7fe2e914e87d1a536d06fe83e66b4e1d95cb593e05aea375531da8/cryptography-49.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7", size = 4700283, upload-time = "2026-06-12T20:01:34.822Z" }, - { url = "https://files.pythonhosted.org/packages/c4/b6/d7696e4e890d6ae1469935164c9e5215c557671cb78d6e3f458ccceaa632/cryptography-49.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68", size = 5265844, upload-time = "2026-06-12T20:01:24.09Z" }, - { url = "https://files.pythonhosted.org/packages/a9/3c/f3ad17eecc1a57b0ba236dc01f90e783c51f4a2f35f64777cc4f47a184b2/cryptography-49.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9", size = 4749290, upload-time = "2026-06-12T20:01:30.848Z" }, - { url = "https://files.pythonhosted.org/packages/4f/01/339573cf1023163a400b0b5d16f6d507de413b9f60be6fd1b77feeaf6737/cryptography-49.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f", size = 4834612, upload-time = "2026-06-12T20:01:29.246Z" }, - { url = "https://files.pythonhosted.org/packages/71/fd/577302e213a1be9468f92d1afef66fcf1ef83d516819d9992ca547f592bd/cryptography-49.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459", size = 4980804, upload-time = "2026-06-12T20:01:42.853Z" }, - { url = "https://files.pythonhosted.org/packages/86/12/c48a424f38db03027be9f7ed5c7dc5de9933dbee992865f98b13727a009d/cryptography-49.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d", size = 4678835, upload-time = "2026-06-12T20:02:48.743Z" }, - { url = "https://files.pythonhosted.org/packages/68/28/8a3ad4653662c93fc44dc4e5d8fd374c25c42e07b34bbfbadf49cf57a5a8/cryptography-49.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa", size = 4697239, upload-time = "2026-06-12T20:02:56.03Z" }, - { url = "https://files.pythonhosted.org/packages/a8/b2/2193fc74f81aee4f9b62733133b73b5176718932ed8f2e4b03fa040480a6/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb", size = 4685593, upload-time = "2026-06-12T20:02:50.666Z" }, - { url = "https://files.pythonhosted.org/packages/47/f1/1d3eaa243bfc5de4a187b22aa8c048b3e4980bfbe830ac46e6bac2e66947/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d", size = 5289961, upload-time = "2026-06-12T20:01:46.468Z" }, - { url = "https://files.pythonhosted.org/packages/58/39/2d51306721330c486495853eda1c567880ff036de15a14c4b74f399934af/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561", size = 4731145, upload-time = "2026-06-12T20:02:16.832Z" }, - { url = "https://files.pythonhosted.org/packages/17/50/983e838c7fd0d87fd8c969bcdd328edaf5f756e38df5281637424c155873/cryptography-49.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122", size = 4321719, upload-time = "2026-06-12T20:02:52.611Z" }, - { url = "https://files.pythonhosted.org/packages/a7/f5/8f571d7e27c55bce9f76f026143bcb1e040a4233149ecca0bea5fa5dd5f7/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505", size = 4685209, upload-time = "2026-06-12T20:02:07.282Z" }, - { url = "https://files.pythonhosted.org/packages/e7/84/0e27016a6fc5a0886f797018b26aa42f40c09a82332bff77822a451deaaa/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866", size = 5246285, upload-time = "2026-06-12T20:01:32.439Z" }, - { url = "https://files.pythonhosted.org/packages/11/2d/5e1fb307cb5931881516b464c98774b3f2c36b5d4bb9a2830253cf553cad/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8", size = 4730441, upload-time = "2026-06-12T20:02:01.469Z" }, - { url = "https://files.pythonhosted.org/packages/e4/c0/bff5a02ee731d207d6a1ed51732549d8c53d2bc8da1d10ec6f2844201d68/cryptography-49.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3", size = 4815869, upload-time = "2026-06-12T20:01:36.574Z" }, - { url = "https://files.pythonhosted.org/packages/b9/26/814681d14248d95d73d5c3eea0c39a94eb8302df966f670a2c60de90974b/cryptography-49.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27", size = 4960948, upload-time = "2026-06-12T20:02:18.688Z" }, - { url = "https://files.pythonhosted.org/packages/3d/df/40577043ca124e17012f408ddddaeb213b856336ac82ddb3bc915f39e29f/cryptography-49.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4", size = 4692429, upload-time = "2026-06-12T20:01:53.628Z" }, - { url = "https://files.pythonhosted.org/packages/2c/99/2d13299eb3dd27b02dcfaafcc91d6b5cb3329f7cbd6d8f51921acd566c1a/cryptography-49.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18", size = 4700968, upload-time = "2026-06-12T20:02:45.383Z" }, - { url = "https://files.pythonhosted.org/packages/a5/4d/9c0cd02f95e2602dd5e563da149ee0830abef3537be8b34dc56281ebe27a/cryptography-49.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69", size = 4697758, upload-time = "2026-06-12T20:01:41.13Z" }, - { url = "https://files.pythonhosted.org/packages/24/01/186c825898477d77e2324d5360fefe622ff1d8d1963ec0554e2cada8ec77/cryptography-49.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64", size = 5298863, upload-time = "2026-06-12T20:02:24.579Z" }, - { url = "https://files.pythonhosted.org/packages/b8/7b/62cbbab75d0659865bf0273790031544a0b16c8072d258f9428dcd8190dc/cryptography-49.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21", size = 4735983, upload-time = "2026-06-12T20:01:50.14Z" }, - { url = "https://files.pythonhosted.org/packages/6c/72/3e798c064bc39e471008075d0f9bc9daf77a80879c092e4a8e170c585ed4/cryptography-49.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9", size = 4334173, upload-time = "2026-06-12T20:01:44.743Z" }, - { url = "https://files.pythonhosted.org/packages/f0/ee/6fca21d1ac73e06f8bef71940abfd4d2f6472b4bca284d770f32bd4086f6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc", size = 4697298, upload-time = "2026-06-12T20:02:20.918Z" }, - { url = "https://files.pythonhosted.org/packages/67/d0/a5fcd3515f0bae49a7b6d0413cc1bdccdcc1fc0047037a0d480642cdc5d6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8", size = 5254338, upload-time = "2026-06-12T20:02:22.737Z" }, - { url = "https://files.pythonhosted.org/packages/a0/84/84fe36f19caf857d61cb7fc9c63035a47ffabd84ea12d1d393148efa3615/cryptography-49.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36", size = 4735650, upload-time = "2026-06-12T20:02:41.389Z" }, - { url = "https://files.pythonhosted.org/packages/6c/a0/db537264e234f7273a73ec020873d6d6b39dfd8a53db78b550ca8320440e/cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e", size = 4834820, upload-time = "2026-06-12T20:01:51.847Z" }, - { url = "https://files.pythonhosted.org/packages/93/77/8df9eb486495979bccecd1062e2eaf435250e84437040295b57d09048b0b/cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b", size = 4967968, upload-time = "2026-06-12T20:02:12.524Z" }, -] - -[[package]] -name = "cs-util" -version = "0.2.2" -source = { git = "https://github.com/CosmoStat/cs_util.git?rev=develop#bedcdbf50ac57fc4d0b72a62b251373cd1807824" } -dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "camb", marker = "sys_platform == 'linux'" }, - { name = "datetime", marker = "sys_platform == 'linux'" }, - { name = "healpy", marker = "sys_platform == 'linux'" }, - { name = "healsparse", marker = "sys_platform == 'linux'" }, - { name = "keyring", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pyccl", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "swig", marker = "sys_platform == 'linux'" }, - { name = "vos", marker = "sys_platform == 'linux'" }, -] - -[[package]] -name = "cycler" -version = "0.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a9/95/a3dbbb5028f35eafb79008e7522a75244477d2838f38cbb722248dabc2a8/cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c", size = 7615, upload-time = "2023-10-07T05:32:18.335Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, -] - -[[package]] -name = "dask" -version = "2026.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click", marker = "sys_platform == 'linux'" }, - { name = "cloudpickle", marker = "sys_platform == 'linux'" }, - { name = "fsspec", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "partd", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "toolz", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4d/b5/aa50877159f1efd65a744107e0662a15d2b80fa0c427d980313367f49493/dask-2026.7.0.tar.gz", hash = "sha256:b039970642ce97a063070bae2763dada8aae27e3cbd8ff6d894072f920a1e252", size = 11548914, upload-time = "2026-07-06T16:58:36.496Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/aa/ceeb81d9dc886d445416310f0eff5c3978f20224b4143e96e36b7e85b011/dask-2026.7.0-py3-none-any.whl", hash = "sha256:68def8b467529ab8425ef9fcc6735dcaefaaf715d5e19ae219485f99a394dbef", size = 1496878, upload-time = "2026-07-06T16:58:34.597Z" }, -] - -[package.optional-dependencies] -array = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] - -[[package]] -name = "dask-image" -version = "2026.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dask", extra = ["array"], marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pims", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "tifffile", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/49/e592a13a5e1efdcdb8f1faab7c4e309c61648792e276f9ced5fb79381b33/dask_image-2026.5.0.tar.gz", hash = "sha256:ed6b462277e691b2c12b0890ba801a0f9a00cc1894b0aa71b195a7a1419b2b00", size = 80457, upload-time = "2026-05-27T14:05:57.383Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/5b/15d6d6ff8697b188787609be059fe4f07f99fc00f43f68e9e1540fa8733e/dask_image-2026.5.0-py3-none-any.whl", hash = "sha256:acf86cd7f0f1e97804198d30b7cc931efd29f9dec86c65ec004b50405c3f5227", size = 43814, upload-time = "2026-05-27T14:05:56.237Z" }, -] - -[[package]] -name = "datetime" -version = "6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytz", marker = "sys_platform == 'linux'" }, - { name = "zope-interface", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/77/32/decbfd165e9985ba9d8c2d34a39afe5aeba2fc3fe390eb6e9ef1aab98fa8/datetime-6.0.tar.gz", hash = "sha256:c1514936d2f901e10c8e08d83bf04e6c9dbd7ca4f244da94fec980980a3bc4d5", size = 64167, upload-time = "2025-11-25T08:00:34.586Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/7a/ea0f3e3ea74be36fc7cf54f966cde732a3de72697983cdb5646b0a4dacde/datetime-6.0-py3-none-any.whl", hash = "sha256:d19988f0657a4e72c9438344157254a8dcad6aea8cd5ae70a5d1b5a75e5dc930", size = 52637, upload-time = "2025-11-25T08:00:33.077Z" }, -] - -[[package]] -name = "debugpy" -version = "1.8.21" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f2/aa/12037145b7a56eaa5b29b41872f7a21b538e807e13f32c4d3c46e59be084/debugpy-1.8.21.tar.gz", hash = "sha256:a3c53278e84c94e11bd87c53970ec391d1a67396c8b22609fcac576520e611a6", size = 1697577, upload-time = "2026-06-01T19:30:35.156Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/09/59324b903599031ff9faaec1758292409f6561a0ec2492fe4b703327705a/debugpy-1.8.21-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:c193d474f0a211191f2b4449d2d06157c689013035bd952f3b617e0ef422b176", size = 3968900, upload-time = "2026-06-01T19:30:52.341Z" }, - { url = "https://files.pythonhosted.org/packages/48/57/412421516afc3055fa577516f00beec3d663f9b0ab330639547ae6c57720/debugpy-1.8.21-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:ecbd158386c31ffe71d46f72d44d56e66331ab9b16cad649156d514368f23ab2", size = 3962096, upload-time = "2026-06-01T19:30:59.235Z" }, - { url = "https://files.pythonhosted.org/packages/8c/2d/6e7ec524984a1702777868de49a4c53202bddac2a432a76a093469587750/debugpy-1.8.21-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:3d6922439bf33fd38a3e2c447869ebc7b97da5cd3d329ff1ef9bc06c4903437e", size = 3927115, upload-time = "2026-06-01T19:31:05.863Z" }, - { url = "https://files.pythonhosted.org/packages/95/51/67e7cf11a53e40694f720457d5b3a1cdaaa3d5a9a633e482f225456b93ff/debugpy-1.8.21-py2.py3-none-any.whl", hash = "sha256:b1e37d333663c8851516a47364ef473da127f9caebe4417e6df6f5825a7e9a92", size = 5352888, upload-time = "2026-06-01T19:31:25.186Z" }, -] - -[[package]] -name = "decorator" -version = "5.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/60/8b/32f9823da46cde7df2087faa08cd98d01b908f8dcab982cdba9c84e85355/decorator-5.3.1.tar.gz", hash = "sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82", size = 58084, upload-time = "2026-05-18T06:03:28.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365, upload-time = "2026-05-18T06:03:26.517Z" }, -] - -[[package]] -name = "defusedxml" -version = "0.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, -] - -[[package]] -name = "deprecated" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wrapt", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, -] - -[[package]] -name = "dill" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, -] - -[[package]] -name = "distro" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, -] - -[[package]] -name = "docstring-parser" -version = "0.18.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, -] - -[[package]] -name = "docutils" -version = "0.22.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ae/b6/03bb70946330e88ffec97aefd3ea75ba575cb2e762061e0e62a213befee8/docutils-0.22.4.tar.gz", hash = "sha256:4db53b1fde9abecbb74d91230d32ab626d94f6badfc575d6db9194a49df29968", size = 2291750, upload-time = "2025-12-18T19:00:26.443Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, -] - -[[package]] -name = "donfig" -version = "0.8.1.post1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, -] - -[[package]] -name = "dpath" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/ce/e1fd64d36e4a5717bd5e6b2ad188f5eaa2e902fde871ea73a79875793fc9/dpath-2.2.0.tar.gz", hash = "sha256:34f7e630dc55ea3f219e555726f5da4b4b25f2200319c8e6902c394258dd6a3e", size = 28266, upload-time = "2024-06-12T22:08:03.686Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/d1/8952806fbf9583004ab479d8f58a9496c3d35f6b6009ddd458bdd9978eaf/dpath-2.2.0-py3-none-any.whl", hash = "sha256:b330a375ded0a0d2ed404440f6c6a715deae5313af40bbb01c8a41d891900576", size = 17618, upload-time = "2024-06-12T22:08:01.881Z" }, -] - -[[package]] -name = "ducc0" -version = "0.41.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cf/e7/849dd1a079b994184fa53905d47278f86d0dd3dcfb6ff2b0e3eb7d6a1c1b/ducc0-0.41.0.tar.gz", hash = "sha256:bac084745bbdb243482a4aec3ecc857bdd46faec298c67e48bc0bb90350dbabd", size = 353129, upload-time = "2026-03-26T18:56:04.669Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/8e/3a28ebdef748b9b59c22d9a801116a9ea073d0ce66d9a738bf5db6c29e0c/ducc0-0.41.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94718e321194313232ae6ba73c528a3847d5eae212a9fc1ef67471e7113e6492", size = 4563683, upload-time = "2026-03-26T18:54:28.643Z" }, - { url = "https://files.pythonhosted.org/packages/11/72/5132178a63fbbbc8868c10b07cbacee138f248ac6f319040537e5a45b3a9/ducc0-0.41.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c17565f332745f08dd012334e07a4dcbb591c9591c2b14f51a1d57ad40018f53", size = 5601347, upload-time = "2026-03-26T18:54:34.054Z" }, - { url = "https://files.pythonhosted.org/packages/17/97/191420d245383304f20f96254774f14b96239e52a0eac44cab93c870941e/ducc0-0.41.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ca66a1806983545460c5447f053f15f6d9f8037b16b128b2a925f0410bcee06d", size = 4563741, upload-time = "2026-03-26T18:54:47.684Z" }, - { url = "https://files.pythonhosted.org/packages/1f/e3/a8783330d7225def7cafe33ed9a2fd7e29eff8f91e7312306037048199a2/ducc0-0.41.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d058b71274f9d60d6edae70cc31d6f5a72e29b0ecc7229725905682566d1e605", size = 5602698, upload-time = "2026-03-26T18:54:53.816Z" }, - { url = "https://files.pythonhosted.org/packages/67/ca/fce0c5395faca711cd4b7203bc001e820288608e6957f81ee183218509ed/ducc0-0.41.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e8c2c49fbfae0c6f7f4ec759a46b936e00c0d4398eeba8f6cfafd562b7dbe55", size = 4563768, upload-time = "2026-03-26T18:55:05.598Z" }, - { url = "https://files.pythonhosted.org/packages/47/a4/70b5a521697c7482ceca8b00f4406bef5a111d37142a719b2fa4942252f6/ducc0-0.41.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:13f98aefcce65b3416e86fa998a0a81ee0ebd9f0feef2588f44c6694b2a50bf0", size = 5604100, upload-time = "2026-03-26T18:55:11.133Z" }, - { url = "https://files.pythonhosted.org/packages/0d/88/022d053b65c9acd69b2de2a9400afd3916ec3a409f7c5776dc906f86b228/ducc0-0.41.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:583d59d5eb4f23c3eb7c6f8a2b6f32da72e3868e9aa711598978a76a25425f96", size = 4579841, upload-time = "2026-03-26T18:55:19.689Z" }, - { url = "https://files.pythonhosted.org/packages/9a/91/428eca0cc8a1142b9952bef02c06bfe54526d730dd070de873424ab8c0bf/ducc0-0.41.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a3a71fd23661ddd6a7f6b44433e778d35a25b3f1492c9239217cdfedbe8178b1", size = 5613706, upload-time = "2026-03-26T18:55:24.971Z" }, -] - -[[package]] -name = "emcee" -version = "3.1.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cb/53/1045ee878cb24281387079f8ee4f0ade1622c6aae1ed1fd91a53e4fa5b19/emcee-3.1.6.tar.gz", hash = "sha256:11af4daf6ab8f9ca69681e3c29054665db7bbd87fd4eb8e437d2c3a1248c637d", size = 2871117, upload-time = "2024-04-19T10:03:19.555Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/ef/2196b9bf88ffa1bde45853c72df021fbd07a8fa91a0f59a22d14a050dc04/emcee-3.1.6-py2.py3-none-any.whl", hash = "sha256:f2d63752023bdccf744461450e512a5b417ae7d28f18e12acd76a33de87580cb", size = 47351, upload-time = "2024-04-19T10:03:17.522Z" }, -] - -[[package]] -name = "executing" -version = "2.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/28/c14e053b6762b1044f34a13aab6859bbf40456d37d23aa286ac24cfd9a5d/executing-2.2.1.tar.gz", hash = "sha256:3632cc370565f6648cc328b32435bd120a1e4ebb20c77e3fdde9a13cd1e533c4", size = 1129488, upload-time = "2025-09-01T09:48:10.866Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, -] - -[[package]] -name = "fastjsonschema" -version = "2.21.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/b5/23b216d9d985a956623b6bd12d4086b60f0059b27799f23016af04a74ea1/fastjsonschema-2.21.2.tar.gz", hash = "sha256:b1eb43748041c880796cd077f1a07c3d94e93ae84bba5ed36800a33554ae05de", size = 374130, upload-time = "2025-08-14T18:49:36.666Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/a8/20d0723294217e47de6d9e2e40fd4a9d2f7c4b6ef974babd482a59743694/fastjsonschema-2.21.2-py3-none-any.whl", hash = "sha256:1c797122d0a86c5cace2e54bf4e819c36223b552017172f32c5c024a6b77e463", size = 24024, upload-time = "2025-08-14T18:49:34.776Z" }, -] - -[[package]] -name = "fitsio" -version = "1.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/34/6b/5658778e070faaa194f1a1133527cedfa948d173daed29d61f25c5487ab6/fitsio-1.4.1.tar.gz", hash = "sha256:2b2bf0bfd37790114b44283931c59a14b1fec25a653bf65c4fc555b8d3b24de9", size = 4148217, upload-time = "2026-07-10T21:21:31.585Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/a0/86e2bde46aa5559b7b9544208def3045675d055f0d132b235ff325279a9b/fitsio-1.4.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:de7ac1df97ce68626dac1da5014e6b396cff2789d1c617c36df03d6bc6d98a2d", size = 881494, upload-time = "2026-07-10T21:21:09.496Z" }, - { url = "https://files.pythonhosted.org/packages/a5/53/f7f644b812e572c57015ef5215f982922d9cdac8d239549c136ba48f2850/fitsio-1.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:34df9129decab3ae85377601668b82442159ed704800bd1ed047428bc532c96d", size = 810043, upload-time = "2026-07-10T21:21:11.254Z" }, - { url = "https://files.pythonhosted.org/packages/83/5d/d71993140df9f939e81d2a5531f26d15cbd700855348a756921b0ad24235/fitsio-1.4.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:161806de2bb9971e271379558616d2cef795ce9590923270ec3bbdadea0d0b2f", size = 913505, upload-time = "2026-07-10T21:21:16.695Z" }, - { url = "https://files.pythonhosted.org/packages/ef/ae/e89778256e0aee3e5b51a620cdf9d91ac89f348504045461bee28006275d/fitsio-1.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5ed27e5b8aaf1ae51718c6624d00fec1d6054fddde8d602eaea3bb6686bcfb0a", size = 815241, upload-time = "2026-07-10T21:21:18.018Z" }, - { url = "https://files.pythonhosted.org/packages/7c/47/7f64a60d883a4b69a0a15c1ec60ba9313d8e774af83ac2b00f57d511b76d/fitsio-1.4.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:1ab4eadb13caf362630d45bce7cf7e57e407d3c971f361f639ca9c316f460bb2", size = 913536, upload-time = "2026-07-10T21:21:22.556Z" }, - { url = "https://files.pythonhosted.org/packages/de/46/9f45b3c8bd371057f1333aec8f92879cb786fe9ed05b6b0c2643fd228558/fitsio-1.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2627eeec3dff4b692402babb88aec3cccbf73a3ee22d19b4accfe76491bd67a8", size = 815239, upload-time = "2026-07-10T21:21:23.61Z" }, - { url = "https://files.pythonhosted.org/packages/8b/3b/9a6d92637db9a603a18681f60689c30208f2485016ed91280e42b927df57/fitsio-1.4.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:5f499149738fdce6afa3324a76fd65a762082647d8b1d30a356dc002dd4a9178", size = 928341, upload-time = "2026-07-10T21:21:28.075Z" }, - { url = "https://files.pythonhosted.org/packages/0a/fa/eeb997744bb1c2c9308dab9b492bcdb1843e4eb3dc5d1697d2f0a312c8b7/fitsio-1.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c2a0dba7a868b4b1716d7468431fda51bc89ee8d8f2a643ac28f9200d99d2e2b", size = 815548, upload-time = "2026-07-10T21:21:29.202Z" }, -] - -[[package]] -name = "flt" -version = "2026.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/80/66/538256582e373d81a944c0ee3a12fd0e4dd0f39d10136353c4ce914d7061/flt-2026.1.tar.gz", hash = "sha256:4bf2818d4a9137f5f715fd2ff13acfbf0175639373155cd89ea2b788edf00093", size = 12817, upload-time = "2026-03-20T22:47:29.754Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/0f/9911ee41dbb2eb043d6252aa8f9bade8995f7b7ee01406e049088c96acf1/flt-2026.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2962fd252df62d77bb2861859173ffc7cc436921a234a52c17eab7839bcce588", size = 479324, upload-time = "2026-03-20T22:46:55.38Z" }, - { url = "https://files.pythonhosted.org/packages/7c/86/c525fef761b2bf9cd961fa6e06a4e255ea9eca529d37e22098d8baa900f0/flt-2026.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b73c4aac4874808dc66e2f7a3cbcdd20a65d0ef14d6f2d378a79930ff3d0bad", size = 478225, upload-time = "2026-03-20T22:46:56.902Z" }, - { url = "https://files.pythonhosted.org/packages/1e/14/9beb3e8a64536741b6f06f62324305171391f9b6d84b804c42e0519f472e/flt-2026.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e02e75d2c8d6441cd7de2d7a13e62bb8feb687cee2d70c4bd57c43c5c78dc630", size = 483678, upload-time = "2026-03-20T22:47:02.783Z" }, - { url = "https://files.pythonhosted.org/packages/95/05/086fd21633043fe19dbe972cab79a680199286c57ba66a88f899c1f51dc7/flt-2026.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ef91c6dd6e453062b5f5d3cbb0c489bd8b3ee1440ad9e5c940d22921664bb69a", size = 483418, upload-time = "2026-03-20T22:47:04.284Z" }, - { url = "https://files.pythonhosted.org/packages/f1/93/a24b0937fa882fc513d399713d6780262ac5beacdef16a3638370669c960/flt-2026.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28c43c464fffa6edbee4bcaaf07565fdd8046ef649764d5647b0de9b2faa19a7", size = 479350, upload-time = "2026-03-20T22:47:10.738Z" }, - { url = "https://files.pythonhosted.org/packages/1a/d9/54d75cabcacbbfa79b3c54515167f5a3b2dcfcb7af1ec23e8df46bc59e52/flt-2026.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f011917cb43d3590c5fdb8e622a9641b5a42e5519d6fea27b4553aa8fc6c8f32", size = 480496, upload-time = "2026-03-20T22:47:11.894Z" }, - { url = "https://files.pythonhosted.org/packages/60/94/a12bc6dd3136b187dacbd5b820537c0e5507e8f63f5c7db10019eeb54394/flt-2026.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:999f1ffbea353fd77b1797ecd8b69a7bae7bb098723ae0dca51a05fc6e4df6dc", size = 480306, upload-time = "2026-03-20T22:47:17.933Z" }, - { url = "https://files.pythonhosted.org/packages/c0/7b/700fc7cfc5033090b27e14e69d9678b1973642f684cdfeffdcf08e561a98/flt-2026.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:10575dc192697bd024ab22f7948fa7f850c40ca6a036e80fe2e96afec6e5c703", size = 473443, upload-time = "2026-03-20T22:47:19.19Z" }, -] - -[[package]] -name = "fonttools" -version = "4.63.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/84/69/c97f2c18e0db87d2c7b15da1974dace76ae938f1cfa22e2727a648b7ed43/fonttools-4.63.0.tar.gz", hash = "sha256:caeb583deeb5168e694b65cda8b4ee62abedfa66cf88488734466f2366b9c4e0", size = 3597189, upload-time = "2026-05-14T12:04:30.958Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/04/0b91d8e916e92ad1fac9e4624760baf0fd5ff2ead614c2f68fb21373f03f/fonttools-4.63.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef3048ef05dbb552b89817713d9cac912e00d0fde4a3105c00d29e52e10c89af", size = 5044298, upload-time = "2026-05-14T12:03:18.085Z" }, - { url = "https://files.pythonhosted.org/packages/77/c7/2342da9830e3e9d4870305ca5d2091d2a83284f2953079b7bdd3b5e029d8/fonttools-4.63.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:58dc6bb86a78d782f00f9190ca02c119cf5bbe2807536e361e18d42019f877d8", size = 4999800, upload-time = "2026-05-14T12:03:20.161Z" }, - { url = "https://files.pythonhosted.org/packages/e6/6d/67fe16c48d7ce050979b33f47e0d28a318f02da030602e944c34f7a16ef3/fonttools-4.63.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee08ebfa58f6e1aeff5697ab9582105bb620008c1caafb681e4c557e7483027b", size = 4982666, upload-time = "2026-05-14T12:03:22.87Z" }, - { url = "https://files.pythonhosted.org/packages/f2/00/3bbab338c07c71fa56269953845e92c951a61457bbbb0f1022551ea266d9/fonttools-4.63.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:27fdc65af8da6f88b9c6121c47a464cbe359fcfff7ff6fc2d37a1f395d755b78", size = 5133598, upload-time = "2026-05-14T12:03:25.168Z" }, - { url = "https://files.pythonhosted.org/packages/8e/40/e76320afa1df918e146155ef239b1719ee266092e96f5423bfd075affba1/fonttools-4.63.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e874792a8212b44583ea02189d9e693906b2f78b261f372f95d6c563210ac1d", size = 5024840, upload-time = "2026-05-14T12:03:36.745Z" }, - { url = "https://files.pythonhosted.org/packages/ce/36/0b805d8c485f872f65a509cbe3b58a5d0d17bee855333b54a150c79d3061/fonttools-4.63.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:22135da48a348785c5e2d5d2d9d6bec5ed44adacbaeb9db12d9493bf6c6bfa68", size = 4975801, upload-time = "2026-05-14T12:03:38.833Z" }, - { url = "https://files.pythonhosted.org/packages/c8/26/2cee03d0aa083ab022da5c07aff9ed3f689da1defb81ad6917c9627896da/fonttools-4.63.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ccf41f2efdf56994d22d73bef4ced1052161958169428d06ba9724ea9e9a64be", size = 4965009, upload-time = "2026-05-14T12:03:41.494Z" }, - { url = "https://files.pythonhosted.org/packages/7e/48/cc4b66d9058c0d0982c833fad10127c4b0e9324606aafa41382295ca4102/fonttools-4.63.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9ced0bd02ac751dd6319b0da88aaef24414e3b0dbc32bb4f24944821a3741a27", size = 5105892, upload-time = "2026-05-14T12:03:43.525Z" }, - { url = "https://files.pythonhosted.org/packages/dd/87/64cfa18a7a1621d17b7f4502b2b0ed8a135a90c3db51ea590ee99043e76b/fonttools-4.63.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b2248c5decb223562f7902ff6325077a073f608ee8e33e88ad88db734eb9f49", size = 5010526, upload-time = "2026-05-14T12:03:54.647Z" }, - { url = "https://files.pythonhosted.org/packages/36/e1/a8933a72c45a87177fbde2696e0d0755c8c9062f8c077a961c6215fa27b1/fonttools-4.63.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:308f957cdeaf8abe4e5f2f124902ef405448af92c90f80e302a3b771c2e6116b", size = 4923946, upload-time = "2026-05-14T12:03:56.984Z" }, - { url = "https://files.pythonhosted.org/packages/27/60/872e6e233b8c5e8b41413796ff18b7fe479661bd40147e071b450dfad7a1/fonttools-4.63.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bf00f21eb5fb721dbaf73d1e9da6d02a1af7768f2ebcf9798be98beab8ba90f6", size = 4962489, upload-time = "2026-05-14T12:03:59.443Z" }, - { url = "https://files.pythonhosted.org/packages/30/c4/83c24f2ec38b90cfda84bf4b1a1f49df80e84a1db4e7ac6e0d41bf23bc39/fonttools-4.63.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c1aaa4b9c75798400ac043ce04d74e7830376c85095a5a6ed7cba2f17a266bf4", size = 5071870, upload-time = "2026-05-14T12:04:02.122Z" }, - { url = "https://files.pythonhosted.org/packages/ea/2b/0aa8db70f18cf52e49b4ed5ecec68547f981160bf5ded3b5aed6faa0a6f9/fonttools-4.63.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0eac00b9118c3c2f87d272e45341871c5b3066baa3c86897fa634a7c3fb59096", size = 5148649, upload-time = "2026-05-14T12:04:12.747Z" }, - { url = "https://files.pythonhosted.org/packages/7f/63/18e4369c25043096f1048e0c9915951adc4f842bd81c6b18155824d6fa99/fonttools-4.63.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51394295f1a51de8b5f30bdb1e1b9a4231536c7064ef5c6e211eec19fa36036f", size = 4932147, upload-time = "2026-05-14T12:04:14.806Z" }, - { url = "https://files.pythonhosted.org/packages/a1/3f/67f3eac2ffd8a98446c5022f8ed3864eac878a5ff7af8df4c8286dba16cc/fonttools-4.63.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9e12f105d2b6342c559c298afb674006bb2893afc7102dcf8a1b55b0486b4e40", size = 5027237, upload-time = "2026-05-14T12:04:17.675Z" }, - { url = "https://files.pythonhosted.org/packages/1a/ba/4e6214cb38a7b04779e97bb7636de9a5c7f20af7018d03dee0b64c08510a/fonttools-4.63.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:796f27556dbe094c4824f75ca85267e4df776c79036c8441469a4df37038c196", size = 5053933, upload-time = "2026-05-14T12:04:20.818Z" }, - { url = "https://files.pythonhosted.org/packages/2c/47/c99d5268f354002ce80f8d029cd9d7d872969da1de8b93d32de4dc56d6f4/fonttools-4.63.0-py3-none-any.whl", hash = "sha256:445af2eab030a16b9171ea8bdda7ebf7d96bda2df88ee182a464252f6e05e20d", size = 1164562, upload-time = "2026-05-14T12:04:29.092Z" }, -] - -[[package]] -name = "fqdn" -version = "1.5.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/30/3e/a80a8c077fd798951169626cde3e239adeba7dab75deb3555716415bd9b0/fqdn-1.5.1.tar.gz", hash = "sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f", size = 6015, upload-time = "2021-03-11T07:16:29.08Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl", hash = "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014", size = 9121, upload-time = "2021-03-11T07:16:28.351Z" }, -] - -[[package]] -name = "fsspec" -version = "2026.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/10/a1/ae4e3e5003468d6391d2c77b6fa1cd73bd5d13511d81c642d7b28ac90ed4/fsspec-2026.6.0.tar.gz", hash = "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a", size = 313646, upload-time = "2026-06-16T01:57:28.105Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" }, -] - -[[package]] -name = "future" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a7/b2/4140c69c6a66432916b26158687e821ba631a4c9273c474343badf84d3ba/future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05", size = 1228490, upload-time = "2024-02-21T11:52:38.461Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/71/ae30dadffc90b9006d77af76b393cb9dfbfc9629f339fc1574a1c52e6806/future-1.0.0-py3-none-any.whl", hash = "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216", size = 491326, upload-time = "2024-02-21T11:52:35.956Z" }, -] - -[[package]] -name = "getdist" -version = "1.6.1" -source = { git = "https://github.com/benabed/getdist.git?rev=113cd22a9a0d013b6f72fe734be81f260f3d3be5#113cd22a9a0d013b6f72fe734be81f260f3d3be5" } -dependencies = [ - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, -] - -[[package]] -name = "gitdb" -version = "4.0.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "smmap", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, -] - -[[package]] -name = "gitpython" -version = "3.1.50" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "gitdb", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, -] - -[[package]] -name = "glass" -version = "2025.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cosmology", marker = "sys_platform == 'linux'" }, - { name = "healpix", marker = "sys_platform == 'linux'" }, - { name = "healpy", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "transformcl", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a5/d6/26cd58e75834671259f26b9287a584d726e6023da35ddf7ca3b7f2c393fb/glass-2025.1.tar.gz", hash = "sha256:7b1aa2394e16010f7f1b4243f49e7e12d7a4dd28fcbf3e3f7cf25ce4905a8615", size = 48533, upload-time = "2025-02-21T18:43:48.046Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/e7/3e89bc5147a3e84245c783098afffbbacbcdf5678ba1aacc2c9c035434da/glass-2025.1-py3-none-any.whl", hash = "sha256:d7919a7d19e05ab8da4e52dbfbcfb5bd1ed2b72e0cc3afde39ec94ced22883c5", size = 47185, upload-time = "2025-02-21T18:43:46.649Z" }, -] - -[[package]] -name = "glass-ext-camb" -version = "2023.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "camb", marker = "sys_platform == 'linux'" }, - { name = "glass", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b7/1a/18bb47c0f7cc267c2f8c95273055b82b1975e4dccbfea6051e82a4d1f513/glass.ext.camb-2023.6.tar.gz", hash = "sha256:f7b6e521b2d45d3b68400353ce6ab1747c772631832a257567de028aa2060509", size = 2989, upload-time = "2023-06-30T22:27:31.511Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/71/50/e92d9db0d26966bad6c4c441822c345784af592767747ee563a65ee24450/glass.ext.camb-2023.6-py3-none-any.whl", hash = "sha256:f46015ecedf87cffabf008d21ac06be5d7a55557027f12c54e258ee11b85c44a", size = 3634, upload-time = "2023-06-30T22:27:30.171Z" }, -] - -[[package]] -name = "google-crc32c" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, - { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, - { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, - { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, - { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, - { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, -] - -[[package]] -name = "greenlet" -version = "3.5.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e2/f1/fbbfef6af0bad0548f09bc28948ea3c275b4edb19e17fc5ca9900a6a634d/greenlet-3.5.3.tar.gz", hash = "sha256:a61efc018fd3eb317eeca31aba90ee9e7f26f22884a79b6c6ec715bf71bb62f1", size = 200270, upload-time = "2026-06-26T19:28:24.832Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/73/815dd90131c1b71ebdf53dbc7c276cafec2a1173b97559f97aba72724a87/greenlet-3.5.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efa9f765dd09f9d0cdac651ffdf631ee59ec5dc6ee7a73e0c012ba9c52fbdf5b", size = 604761, upload-time = "2026-06-26T19:07:10.114Z" }, - { url = "https://files.pythonhosted.org/packages/9f/57/079cfe76bcef36b153b25607ee91c6fcb58f17f8b23c86bbbeabe0c88d72/greenlet-3.5.3-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7faba15ac005376e02a0384504e0243be3370ce010296a44a820feb342b505ab", size = 617044, upload-time = "2026-06-26T19:10:07.25Z" }, - { url = "https://files.pythonhosted.org/packages/37/87/b4d095775a3fb1bcafbb483fc206b27ebb785724c83051447737085dc54e/greenlet-3.5.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:87142215824be6ac05e2e8e2786eec307ccbc27c36723c3881959df654af6861", size = 614244, upload-time = "2026-06-26T18:32:17.594Z" }, - { url = "https://files.pythonhosted.org/packages/8a/70/7559b609683650fa2b95b8ab84b4ab0b26556a635d19675e12aa832d826d/greenlet-3.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:215275b1b49320987352e6c1b054acca0064f965a2c66992bed9a6f7d913f149", size = 1574210, upload-time = "2026-06-26T19:09:03.077Z" }, - { url = "https://files.pythonhosted.org/packages/ae/73/be55392074c60fc37655ca40fa6022457bfbf6718e9e342a7b0b41f96dd2/greenlet-3.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6b1b0eed82364b0e32c4ea0f221452d33e6bb17ae094d9f72aed9851812747ea", size = 1638627, upload-time = "2026-06-26T18:31:44.748Z" }, - { url = "https://files.pythonhosted.org/packages/d6/fa/5401ac78021c826a25b6dde0c705e0a8f29b617509f9185a31dac15fbe1b/greenlet-3.5.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2d185dd1621757e70c3861cceffd5317ab4e7ed7eb09c82994828468527ade5", size = 607435, upload-time = "2026-06-26T19:07:11.412Z" }, - { url = "https://files.pythonhosted.org/packages/e9/76/1dc144a2e56e65d36405078ed774224375ea520a1870a6e46e08bb4ac7bf/greenlet-3.5.3-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1c514a468149bf8fbbab874188a3535cd8a48a3e353eb53a3d424296f8dbacd3", size = 619787, upload-time = "2026-06-26T19:10:08.396Z" }, - { url = "https://files.pythonhosted.org/packages/bf/87/c298cee62df1de4ad7fec32abda73526cff347fd143a6ed4ac369246668a/greenlet-3.5.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:915f887cf2682b66419b879423a2e072634aa7b7dce6f3ada4957cfced3f1e9a", size = 616786, upload-time = "2026-06-26T18:32:19.128Z" }, - { url = "https://files.pythonhosted.org/packages/9e/2e/e6f009885ed0705ccf33fe0583c117cfd03cde77e31a596dd5785a30762b/greenlet-3.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:766cfd421c13e450feb340cd472a3ed9957d438727b7b4593ad7c76c5d2b0deb", size = 1574316, upload-time = "2026-06-26T19:09:04.273Z" }, - { url = "https://files.pythonhosted.org/packages/ef/fe/43fd110b01e40da0adb7c90ac7ea744bef2d43dca00de5095fd2351c2a68/greenlet-3.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2ecda9ec22edf38fa389369eaed8c3d37c05f3c54e69f69438dbb2cc1de1458b", size = 1638614, upload-time = "2026-06-26T18:31:46.297Z" }, - { url = "https://files.pythonhosted.org/packages/82/2f/146d218299046a43d1f029fd544b3d110d0f175a09c715c7e8da4a4a345d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df0a0628d1597eb0897b62f55d1343f772405fd25f3b2a796c76874b0c2e22e8", size = 654096, upload-time = "2026-06-26T19:07:12.71Z" }, - { url = "https://files.pythonhosted.org/packages/a0/cc/04738cafb3f45fa991ea44f9de94c47dcec964f5a972300988a6751f49d9/greenlet-3.5.3-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ebd933a6adabc298bab47731a130fe6bfb888bd934eee37810f151159544540d", size = 666304, upload-time = "2026-06-26T19:10:09.503Z" }, - { url = "https://files.pythonhosted.org/packages/ce/aa/4e0dad5e605c270c784ab911c43da6adb136ccd4d81180f763ca429a723d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b9d501b40e80b70e32323c799dd9b420a5577a9601469d362ae1ffb690f3a7c", size = 663635, upload-time = "2026-06-26T18:32:20.802Z" }, - { url = "https://files.pythonhosted.org/packages/d1/50/13efdbea246fe3d3b735e191fec08fb50809f53cd2383ebe123d0809e44b/greenlet-3.5.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a1fad1d11e7d6aab184107baa8e4ece11ccba3ec9599cd7efa5ff4d70d43256a", size = 1621252, upload-time = "2026-06-26T19:09:05.647Z" }, - { url = "https://files.pythonhosted.org/packages/f7/22/c0a336ae4a1410fd5f5121098e5bfbf1865f64c5ef80b4b5412886c4a332/greenlet-3.5.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fad5aec764399f1b5cc347ad250a59660f20c8f8888ea6bae1f93b769cce1154", size = 1684824, upload-time = "2026-06-26T18:31:47.738Z" }, - { url = "https://files.pythonhosted.org/packages/00/92/715c44721abe2b4d1ae9abde4179411868a5bff312479f54e105d372f131/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19131729ae0ddc3c2e1ef85e650169b5e37ee32e400f215f78b94d7b0d567310", size = 653382, upload-time = "2026-06-26T19:07:14.209Z" }, - { url = "https://files.pythonhosted.org/packages/a0/83/37a10372a1090a6624cca8e74c12df1a36c2dc36429ed0255b7fb1aeee23/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1540dd8e5fc2a5aec40fbb98ef8e149fa47c89a4b4a1cf2575a14d3d1869d7a8", size = 659401, upload-time = "2026-06-26T19:10:10.876Z" }, - { url = "https://files.pythonhosted.org/packages/db/e2/d1509cad4207da559cc42986ecdd8fc67ad0d1bba2bf03023c467fd5e0f3/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e81fa194a1d20967877bdf9c7794db2bc99063e5be36aee710c08f04c5bb087f", size = 656969, upload-time = "2026-06-26T18:32:22.272Z" }, - { url = "https://files.pythonhosted.org/packages/86/7d/eaf70de20aadca3a5884aec58362861c64ce45e7b277f47ed026926a3b89/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:55cf4d777485d43110e47133cbba6d74a8885a87ec1227ef0267f9ee80c5aa21", size = 1617822, upload-time = "2026-06-26T19:09:06.893Z" }, - { url = "https://files.pythonhosted.org/packages/8a/f9/414d38fc400ae4350d4185eaad1827676f7cf5287b9136e0ed1cbbe20a7f/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:12a248ba75f6a9a236375f52296c498c89ff1d8badf32deb9eca7abd5853f7da", size = 1677983, upload-time = "2026-06-26T18:31:49.396Z" }, - { url = "https://files.pythonhosted.org/packages/4f/ca/69db42d447a1378043e2c8f19c09cbbd1263371505053c496b49066d3d16/greenlet-3.5.3-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78dbef602fda6d97d957eb7937f70c9ce9e9527330347f8f6b6f9e554a9e7a47", size = 659747, upload-time = "2026-06-26T19:07:15.565Z" }, - { url = "https://files.pythonhosted.org/packages/a8/0b/af7ac2ef8dd41e3da1a40dda6305c23b9a03e13ba975ec916357b50f8575/greenlet-3.5.3-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f73857adb8fee13fa56c172bd11262f888c0c648f9fea113e777bb2c7904a81", size = 670419, upload-time = "2026-06-26T19:10:12.293Z" }, - { url = "https://files.pythonhosted.org/packages/51/1e/1d51640cacbfc455dbe9f9a9f594c49e4e244f63b9971a2f4764e46cc53d/greenlet-3.5.3-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:232fec92e823addaf02d9472cf7381e24a1d046a6ced1103c5caa4c21b9dfc1d", size = 668787, upload-time = "2026-06-26T18:32:24.298Z" }, - { url = "https://files.pythonhosted.org/packages/21/66/4030d5b0b5894500023f003bb054d9bb354dfbd1e186c3a296759172f5f5/greenlet-3.5.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2421c3564da9429d5586d46ca31ebb26516b5498a802cf65c041a8e8a8980d34", size = 1626305, upload-time = "2026-06-26T19:09:08.281Z" }, - { url = "https://files.pythonhosted.org/packages/0e/50/5221371c7550108dfa3c378debc41d032aa9c78e89abb01d8011cfc93289/greenlet-3.5.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e0f0d160f0b2e558e6c75f7930967183255dc9735e5f5b8cae58ee09c9576d8b", size = 1688631, upload-time = "2026-06-26T18:31:51.278Z" }, - { url = "https://files.pythonhosted.org/packages/57/66/b3bfae3e220a9b63ea539a0eea681800c69ab1aada757eae8789f183e7ce/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:629b614d2b786e89c50440e246f33eea78f58a962d0bdbbcc809e6d13605903f", size = 657221, upload-time = "2026-06-26T19:07:16.973Z" }, - { url = "https://files.pythonhosted.org/packages/7b/81/b6d4d73a709684fc77e7fa034d7c2fe82cffa9fc920fadcaa659c2626213/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2b2e857ae16f5f72142edf75f9f176fe7526ba19a2841df1420516f83831c9f2", size = 663226, upload-time = "2026-06-26T19:10:13.723Z" }, - { url = "https://files.pythonhosted.org/packages/f5/07/e210b02b589f16e74ff48b730690e4a34ffe984219fce4f3c1a0e7ec8545/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e515757e2e36bcbf1fad09a46e1557e8b1ae1797d4b44d09da7deed88ad28608", size = 660802, upload-time = "2026-06-26T18:32:26.081Z" }, - { url = "https://files.pythonhosted.org/packages/eb/2e/5303eb3fa06bca089060f479707182a93e360683bc252acf846c3090d34e/greenlet-3.5.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b363d46ed1ea431825fdb01471bb024fc08399bad1572a616e853c7684415adb", size = 1622157, upload-time = "2026-06-26T19:09:09.527Z" }, - { url = "https://files.pythonhosted.org/packages/54/70/50de47a488f14df260b50ae34fb5d56016e308b098eab02c878b5223c26a/greenlet-3.5.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e44da2f5bbdaabaf7d80b73dbb430c7035771e9f244e3c8b769715c9d8fa0a16", size = 1681159, upload-time = "2026-06-26T18:31:52.986Z" }, -] - -[[package]] -name = "gsl" -version = "0.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/36/a8a99ab67918beb6ee8722fb6c27512796a5ff8ed4c10036f787eef51569/gsl-0.0.3.tar.gz", hash = "sha256:85a6071e00a284510c963b97ab74ef8a3e7d1786c5f52dc2ff444cbb7d94ee3c", size = 44404, upload-time = "2018-05-30T09:02:51.203Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/1e/14d4c4657bc7130b900c65033861ee0db508d2d9206f4223f5da7c855acc/gsl-0.0.3-py3-none-any.whl", hash = "sha256:081c77375bf65cb759a3e5129275b8806be18648452f50f7c420f40dac60529a", size = 39432, upload-time = "2018-05-30T09:02:49.93Z" }, -] - -[[package]] -name = "h11" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, -] - -[[package]] -name = "h2" -version = "4.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "hpack", marker = "sys_platform == 'linux'" }, - { name = "hyperframe", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" }, -] - -[[package]] -name = "h5py" -version = "3.16.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/db/33/acd0ce6863b6c0d7735007df01815403f5589a21ff8c2e1ee2587a38f548/h5py-3.16.0.tar.gz", hash = "sha256:a0dbaad796840ccaa67a4c144a0d0c8080073c34c76d5a6941d6818678ef2738", size = 446526, upload-time = "2026-03-06T13:49:08.07Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/89/84/06281c82d4d1686fde1ac6b0f307c50918f1c0151062445ab3b6fa5a921d/h5py-3.16.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ff24039e2573297787c3063df64b60aab0591980ac898329a08b0320e0cf2527", size = 5198852, upload-time = "2026-03-06T13:48:07.482Z" }, - { url = "https://files.pythonhosted.org/packages/9e/e9/1a19e42cd43cc1365e127db6aae85e1c671da1d9a5d746f4d34a50edb577/h5py-3.16.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:dfc21898ff025f1e8e67e194965a95a8d4754f452f83454538f98f8a3fcb207e", size = 5405250, upload-time = "2026-03-06T13:48:09.628Z" }, - { url = "https://files.pythonhosted.org/packages/b7/8e/9790c1655eabeb85b92b1ecab7d7e62a2069e53baefd58c98f0909c7a948/h5py-3.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:698dd69291272642ffda44a0ecd6cd3bda5faf9621452d255f57ce91487b9794", size = 5190108, upload-time = "2026-03-06T13:48:11.26Z" }, - { url = "https://files.pythonhosted.org/packages/51/d7/ab693274f1bd7e8c5f9fdd6c7003a88d59bedeaf8752716a55f532924fbb/h5py-3.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2b2c02b0a160faed5fb33f1ba8a264a37ee240b22e049ecc827345d0d9043074", size = 5419216, upload-time = "2026-03-06T13:48:13.322Z" }, - { url = "https://files.pythonhosted.org/packages/da/1e/6172269e18cc5a484e2913ced33339aad588e02ba407fafd00d369e22ef3/h5py-3.16.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:099f2525c9dcf28de366970a5fb34879aab20491589fa89ce2863a84218bb524", size = 5193860, upload-time = "2026-03-06T13:48:24.071Z" }, - { url = "https://files.pythonhosted.org/packages/bd/98/ef2b6fe2903e377cbe870c3b2800d62552f1e3dbe81ce49e1923c53d1c5c/h5py-3.16.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9300ad32dea9dfc5171f94d5f6948e159ed93e4701280b0f508773b3f582f402", size = 5400417, upload-time = "2026-03-06T13:48:25.728Z" }, - { url = "https://files.pythonhosted.org/packages/bc/81/5b62d760039eed64348c98129d17061fdfc7839fc9c04eaaad6dee1004e4/h5py-3.16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:171038f23bccddfc23f344cadabdfc9917ff554db6a0d417180d2747fe4c75a7", size = 5185214, upload-time = "2026-03-06T13:48:27.436Z" }, - { url = "https://files.pythonhosted.org/packages/28/c4/532123bcd9080e250696779c927f2cb906c8bf3447df98f5ceb8dcded539/h5py-3.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7e420b539fb6023a259a1b14d4c9f6df8cf50d7268f48e161169987a57b737ff", size = 5414598, upload-time = "2026-03-06T13:48:29.49Z" }, - { url = "https://files.pythonhosted.org/packages/58/a5/4964bc0e91e86340c2bbda83420225b2f770dcf1eb8a39464871ad769436/h5py-3.16.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:e2c04d129f180019e216ee5f9c40b78a418634091c8782e1f723a6ca3658b965", size = 5198886, upload-time = "2026-03-06T13:48:38.879Z" }, - { url = "https://files.pythonhosted.org/packages/f1/16/d905e7f53e661ce2c24686c38048d8e2b750ffc4350009d41c4e6c6c9826/h5py-3.16.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e4360f15875a532bc7b98196c7592ed4fc92672a57c0a621355961cafb17a6dd", size = 5404883, upload-time = "2026-03-06T13:48:41.324Z" }, - { url = "https://files.pythonhosted.org/packages/4b/f2/58f34cb74af46d39f4cd18ea20909a8514960c5a3e5b92fd06a28161e0a8/h5py-3.16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3fae9197390c325e62e0a1aa977f2f62d994aa87aab182abbea85479b791197c", size = 5192039, upload-time = "2026-03-06T13:48:43.117Z" }, - { url = "https://files.pythonhosted.org/packages/ce/ca/934a39c24ce2e2db017268c08da0537c20fa0be7e1549be3e977313fc8f5/h5py-3.16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:43259303989ac8adacc9986695b31e35dba6fd1e297ff9c6a04b7da5542139cc", size = 5421526, upload-time = "2026-03-06T13:48:44.838Z" }, - { url = "https://files.pythonhosted.org/packages/f5/a1/af87f64b9f986889884243643621ebbd4ac72472ba8ec8cec891ac8e2ca1/h5py-3.16.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:0f456f556e4e2cebeebd9d66adf8dc321770a42593494a0b6f0af54a7567b242", size = 5074061, upload-time = "2026-03-06T13:48:54.089Z" }, - { url = "https://files.pythonhosted.org/packages/cc/d0/146f5eaff3dc246a9c7f6e5e4f42bd45cc613bce16693bcd4d1f7c958bf5/h5py-3.16.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:3e6cb3387c756de6a9492d601553dffea3fe11b5f22b443aac708c69f3f55e16", size = 5279216, upload-time = "2026-03-06T13:48:56.75Z" }, - { url = "https://files.pythonhosted.org/packages/a1/9d/12a13424f1e604fc7df9497b73c0356fb78c2fb206abd7465ce47226e8fd/h5py-3.16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8389e13a1fd745ad2856873e8187fd10268b2d9677877bb667b41aebd771d8b7", size = 5070068, upload-time = "2026-03-06T13:48:59.169Z" }, - { url = "https://files.pythonhosted.org/packages/41/8c/bbe98f813722b4873818a8db3e15aa3e625b59278566905ac439725e8070/h5py-3.16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:346df559a0f7dcb31cf8e44805319e2ab24b8957c45e7708ce503b2ec79ba725", size = 5300253, upload-time = "2026-03-06T13:49:02.033Z" }, -] - -[[package]] -name = "healpix" -version = "2025.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/43/1a/c1af658a2d06cdd2734b085b0eee34d51646f091ed13b1669d3a596de57c/healpix-2025.1.tar.gz", hash = "sha256:733f16f30c15d911c8209f4045f0b0934e5fcafa63b8b39d62e425a036adccea", size = 22744, upload-time = "2025-07-07T16:38:42.402Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/9e/a14baf09831eef05592a91b8c0c29918025b5069511c9300c4ce64ef5f25/healpix-2025.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1a1d41e0c2a98ef3d3cf48e18ca7edadb599641f1754e28e8df0a0a96471fdd4", size = 70836, upload-time = "2025-07-07T16:38:22.512Z" }, - { url = "https://files.pythonhosted.org/packages/0b/06/7946ec2c0fd5a4fb2b077c9c49358fb32577b21c4e27c7457464716780d3/healpix-2025.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2e347f3c76407d7c8018cd8da7eccb4aebd715e46b8f60111dcf161e969bd25f", size = 69891, upload-time = "2025-07-07T16:38:23.692Z" }, - { url = "https://files.pythonhosted.org/packages/81/cb/82b05f0779924b73c2522f440a65bd4d27af2099aec876c44168bea01136/healpix-2025.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e8cacfd6359564c12c7bc81633e92fa7d6e74fffb834a1e08a38e88cb9ac3c50", size = 70865, upload-time = "2025-07-07T16:38:29.898Z" }, - { url = "https://files.pythonhosted.org/packages/fa/0c/fc40bad841a2c6d45b8e59973135be667c8f4b2000c870b374a3a746a66a/healpix-2025.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa546ca045be0661dfc42f7a4e7996f2134dd07d0d5639ae8d7d1a6b293470a2", size = 69939, upload-time = "2025-07-07T16:38:31.257Z" }, -] - -[[package]] -name = "healpy" -version = "1.19.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e3/4e/6f5328f375f88c8c38314f24f82b7e1486ec0358c557779500382db1503c/healpy-1.19.0.tar.gz", hash = "sha256:28e839cb885a23d36c77fc3423a3cb9271a07fda94085bd12fc329f941130ec5", size = 4075006, upload-time = "2025-12-02T08:27:19.766Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/51/c07439b92bc05966006897762b7291d7d2c83fde8807b8d53a48e140a8b4/healpy-1.19.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:730095eef4bbe3c94039edb330d1800285cf43c7e16a81ce5b87edb31a8ffa42", size = 8239086, upload-time = "2025-12-02T08:26:53.076Z" }, - { url = "https://files.pythonhosted.org/packages/61/c6/42745aeff172755e55af2487491c7a7475fe1068f152850cc14d64e04f30/healpy-1.19.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:89dbe14b270b5479a9652d1ab9f28fc7fce34253549db665f2e91e6cea588883", size = 8255685, upload-time = "2025-12-02T08:26:54.361Z" }, - { url = "https://files.pythonhosted.org/packages/9d/86/2a223bd92dde578de34f07095c71152c84b158644782c4057ca9b25e53b5/healpy-1.19.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:29bb5f7acde68ca5850c3a59ba33cacf4c3768c9e85f7013a99aa5fbd127ce84", size = 9042629, upload-time = "2025-12-02T08:26:55.773Z" }, - { url = "https://files.pythonhosted.org/packages/18/6d/037a7bb889c92b3e3f792de6a692e636de0577ea62bdd0038b1eb68270af/healpy-1.19.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:126bef3ae6594f461435ac85c6bcb32e427eca43f743881d00186bc3a55da1d8", size = 9376051, upload-time = "2025-12-02T08:26:57.516Z" }, - { url = "https://files.pythonhosted.org/packages/de/bb/314bf8c493701b5842cdd74a1b5b34cea1522444042efad5b7cb17b8d159/healpy-1.19.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:086bae2510ac60f7cf9bbb520e1d1b8864cc8fca74b46b16e24857e74d08f896", size = 8198878, upload-time = "2025-12-02T08:27:03.119Z" }, - { url = "https://files.pythonhosted.org/packages/f6/d4/a60ed9a50768ff5e896dd94d878496ae16767925ea32c49d5a4189ab818a/healpy-1.19.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:920c0a1c6749c05c8ad9522a5a2630f7bc83124c5742ef50f91b9f5e6a1bdcc7", size = 8205835, upload-time = "2025-12-02T08:27:04.416Z" }, - { url = "https://files.pythonhosted.org/packages/6c/fd/5f4c989b53423bbad07c91a4b9d52de5cde9f1154e3d2b145b397e1b8cd8/healpy-1.19.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:26e3e6c50f2d256c9218f0fb0624406a93c4f22bc66deebf51a0f31fd5594b89", size = 8991846, upload-time = "2025-12-02T08:27:06.123Z" }, - { url = "https://files.pythonhosted.org/packages/d5/df/00636a5d2f1141b1fa9646069436c4fc68b35ac2c53ef520f8812b900f05/healpy-1.19.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bc3c243985abf1f1b5d91da12a23ceed49181dd9d0e46ed362babdc92c814aa8", size = 9325719, upload-time = "2025-12-02T08:27:08.138Z" }, - { url = "https://files.pythonhosted.org/packages/c5/ca/f583967e1395e41608225d0a6d29e582236f615d83023d730e63bac89baa/healpy-1.19.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b3cf9c627af77a3fc670d6240d8a83485f373c274945ca8ad156fc2a25d6eb61", size = 8172544, upload-time = "2025-12-02T08:27:13.684Z" }, - { url = "https://files.pythonhosted.org/packages/85/ae/80a5e0d36ee51e72b27fb8e11d0d43dfb3f8e2ccc315a38865a2ccd5ed73/healpy-1.19.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:c22360ad7ec6f16bdbb650dececf52d21f7f0e03a943eb1a1046fdb9c51a3c5e", size = 8198134, upload-time = "2025-12-02T08:27:15.156Z" }, - { url = "https://files.pythonhosted.org/packages/36/7b/34b9954fdf76b1f5ab0a76228650c2575b189fb6f9f7865c98477a46086d/healpy-1.19.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1a479826c1d6ae8312476e079a3f5c7aa3e40911d74f2d7069adcd62e98604ad", size = 8986426, upload-time = "2025-12-02T08:27:16.417Z" }, - { url = "https://files.pythonhosted.org/packages/94/df/bb3ef51f423c62576fb0775fe72f0d2028a10e3ffb20b4fc76146ea8757b/healpy-1.19.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a103e327102e124785c1b5bdd379c6e5469137633212b56a6196bf04ddd100ff", size = 9307512, upload-time = "2025-12-02T08:27:18.03Z" }, -] - -[[package]] -name = "healsparse" -version = "1.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "hpgeom", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e3/33/36520faabae128fe9a4a5cbeb0916be864d2eb34cf7410d52dc532af044e/healsparse-1.13.0.tar.gz", hash = "sha256:c21e4eb075c686bd306bdc1b40e039134c4ca60ccfa4e7e73845c107a5b16544", size = 133589, upload-time = "2026-07-07T18:33:05.481Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/2f/7674b91031a12a9b3cd69f3bb675d3b1eac59657139e23bc528ed6d33d0a/healsparse-1.13.0-py3-none-any.whl", hash = "sha256:3b475a61d7701526246a592ed980758f94764caba4355429bf470a2f9b812964", size = 70673, upload-time = "2026-07-07T18:33:04.514Z" }, -] - -[[package]] -name = "hpack" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/5b/fcabf6028144a8723726318b07a32c2f3314acdff6265743cf08a344b18e/hpack-4.2.0.tar.gz", hash = "sha256:0895cfa3b5531fc65fe439c05eb65144f123bf7a394fcaa56aa423548d8e45c0", size = 51300, upload-time = "2026-06-23T18:34:46.667Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/71/b4/4a9fcfb2aef6ba44d9073ecd301443aa00b3dac95de5619f2a7de7ec8a91/hpack-4.2.0-py3-none-any.whl", hash = "sha256:858ac0b02280fa582b5080d68db0899c62a80375e0e5413a74970c5e518b6986", size = 34246, upload-time = "2026-06-23T18:34:45.472Z" }, -] - -[[package]] -name = "hpgeom" -version = "1.5.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/07/72/27640de7d85a566b8ecd27e7894f371a19669c56c268756e18111a137df8/hpgeom-1.5.4.tar.gz", hash = "sha256:85ac73e267c11f3f248920d3541a0e6ffab821cae54f0d44414d1939dfb91200", size = 153906, upload-time = "2026-02-24T19:03:01.954Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/7f/bae5e1a44d76fa7396c4a004e2e86acb5fe70d8aaaa7f91190c65eb09e97/hpgeom-1.5.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d544a9834477b0aedf61ee934ce4598170cf14f16742200cd1729103edb5f506", size = 192550, upload-time = "2026-02-24T19:02:49.454Z" }, - { url = "https://files.pythonhosted.org/packages/68/4d/8b83dcbbd08bd7e713583490c14db848515a909515644037c80d20e2b80c/hpgeom-1.5.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7af0216ff1db0a49a771356562088c49185104a10502b68ab26c8ca2816d17b0", size = 197180, upload-time = "2026-02-24T19:02:50.453Z" }, - { url = "https://files.pythonhosted.org/packages/75/dc/4c0c6d7eba1d42be23794b03e6048af4de0d92c5805f4f2ed29bc1ca0f16/hpgeom-1.5.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d7058b3396fb7deb8b8637ac7f2861bfaa213574c50b9e771bf80f4d1e9064f3", size = 192643, upload-time = "2026-02-24T19:02:54.045Z" }, - { url = "https://files.pythonhosted.org/packages/8d/9a/41ba036857a5489d2c1e6f742903aae452d874a48c5b24889de79e02e668/hpgeom-1.5.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e160d40a18b8e80b7e233b4459adcbd4aea5d1fb48cefe7f07e8fc7dfbdcd9f3", size = 197257, upload-time = "2026-02-24T19:02:55.356Z" }, - { url = "https://files.pythonhosted.org/packages/a7/83/45d4208eb33432195df1998d5b5d75afc52d52cdec5ccdec80fa9d3fb2cc/hpgeom-1.5.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:47fe1c0c334d9722110a0d6346d2e4222aaed4454138996f9b2638a78ec98b60", size = 192606, upload-time = "2026-02-24T19:02:58.915Z" }, - { url = "https://files.pythonhosted.org/packages/1d/67/ab80b95519d453e72cd65ccc6cd411f23a4d997f492ed9ad0dbc51bce0fa/hpgeom-1.5.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fce969f1311a2f44c4f1c58fca1302b472ef798e702725a75bc16a342f924a66", size = 197071, upload-time = "2026-02-24T19:03:00.063Z" }, -] - -[[package]] -name = "html2text" -version = "2025.4.15" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/27/e158d86ba1e82967cc2f790b0cb02030d4a8bef58e0c79a8590e9678107f/html2text-2025.4.15.tar.gz", hash = "sha256:948a645f8f0bc3abe7fd587019a2197a12436cd73d0d4908af95bfc8da337588", size = 64316, upload-time = "2025-04-15T04:02:30.045Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/84/1a0f9555fd5f2b1c924ff932d99b40a0f8a6b12f6dd625e2a47f415b00ea/html2text-2025.4.15-py3-none-any.whl", hash = "sha256:00569167ffdab3d7767a4cdf589b7f57e777a5ed28d12907d8c58769ec734acc", size = 34656, upload-time = "2025-04-15T04:02:28.44Z" }, -] - -[[package]] -name = "httpcore" -version = "1.0.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi", marker = "sys_platform == 'linux'" }, - { name = "h11", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, -] - -[[package]] -name = "httpx" -version = "0.28.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio", marker = "sys_platform == 'linux'" }, - { name = "certifi", marker = "sys_platform == 'linux'" }, - { name = "httpcore", marker = "sys_platform == 'linux'" }, - { name = "idna", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, -] - -[package.optional-dependencies] -http2 = [ - { name = "h2", marker = "sys_platform == 'linux'" }, -] - -[[package]] -name = "humanfriendly" -version = "10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794, upload-time = "2021-09-17T21:40:39.897Z" }, -] - -[[package]] -name = "hyperframe" -version = "6.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, -] - -[[package]] -name = "idna" -version = "3.18" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, -] - -[[package]] -name = "imageio" -version = "2.37.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pillow", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/84/93bcd1300216ea50811cee96873b84a1bebf8d0489ffaf7f2a3756bab866/imageio-2.37.3.tar.gz", hash = "sha256:bbb37efbfc4c400fcd534b367b91fcd66d5da639aaa138034431a1c5e0a41451", size = 389673, upload-time = "2026-03-09T11:31:12.573Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/fa/391e437a34e55095173dca5f24070d89cbc233ff85bf1c29c93248c6588d/imageio-2.37.3-py3-none-any.whl", hash = "sha256:46f5bb8522cd421c0f5ae104d8268f569d856b29eb1a13b92829d1970f32c9f0", size = 317646, upload-time = "2026-03-09T11:31:10.771Z" }, -] - -[[package]] -name = "imagesize" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/e6/7bf14eeb8f8b7251141944835abd42eb20a658d89084b7e1f3e5fe394090/imagesize-2.0.0.tar.gz", hash = "sha256:8e8358c4a05c304f1fccf7ff96f036e7243a189e9e42e90851993c558cfe9ee3", size = 1773045, upload-time = "2026-03-03T14:18:29.941Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/53/fb7122b71361a0d121b669dcf3d31244ef75badbbb724af388948de543e2/imagesize-2.0.0-py2.py3-none-any.whl", hash = "sha256:5667c5bbb57ab3f1fa4bc366f4fbc971db3d5ed011fd2715fd8001f782718d96", size = 9441, upload-time = "2026-03-03T14:18:27.892Z" }, -] - -[[package]] -name = "immutables" -version = "0.21" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/69/41/0ccaa6ef9943c0609ec5aa663a3b3e681c1712c1007147b84590cec706a0/immutables-0.21.tar.gz", hash = "sha256:b55ffaf0449790242feb4c56ab799ea7af92801a0a43f9e2f4f8af2ab24dfc4a", size = 89008, upload-time = "2024-10-10T00:55:01.434Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/52/cb9e2bb7a69338155ffabbd2f993c968c750dd2d5c6c6eaa6ebb7bfcbdfa/immutables-0.21-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b48b116aaca4500398058b5a87814857a60c4cb09417fecc12d7da0f5639b73d", size = 104270, upload-time = "2024-10-10T00:54:21.912Z" }, - { url = "https://files.pythonhosted.org/packages/0f/a4/25df835a9b9b372a4a869a8a1ac30a32199f2b3f581ad0e249f7e3d19eed/immutables-0.21-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dad7c0c74b285cc0e555ec0e97acbdc6f1862fcd16b99abd612df3243732e741", size = 104864, upload-time = "2024-10-10T00:54:22.956Z" }, - { url = "https://files.pythonhosted.org/packages/4a/51/b548fbc657134d658e179ee8d201ae82d9049aba5c3cb2d858ed2ecb7e3f/immutables-0.21-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e44346e2221a5a676c880ca8e0e6429fa24d1a4ae562573f5c04d7f2e759b030", size = 99733, upload-time = "2024-10-10T00:54:23.99Z" }, - { url = "https://files.pythonhosted.org/packages/47/db/d7b1e0e88faf07fe9a88579a86f58078a9a37fff871f4b3dbcf28cad9a12/immutables-0.21-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b10139b529a460e53fe8be699ebd848c54c8a33ebe67763bcfcc809a475a26f", size = 101698, upload-time = "2024-10-10T00:54:25.734Z" }, - { url = "https://files.pythonhosted.org/packages/d2/d0/a5fb7c164ddb298ec37537e618b70dfa30c7cae9fac01de374c36489cbc9/immutables-0.21-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:583d2a63e444ce1538cc2bda56ae1f4a1a11473dbc0377c82b516bc7eec3b81e", size = 104334, upload-time = "2024-10-10T00:54:31.284Z" }, - { url = "https://files.pythonhosted.org/packages/f3/a5/5fda0ee4a261a85124011ac0750fec678f00e1b2d4a5502b149a3b4d86d9/immutables-0.21-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b274a52da9b106db55eceb93fc1aea858c4e6f4740189e3548e38613eafc2021", size = 104898, upload-time = "2024-10-10T00:54:32.295Z" }, - { url = "https://files.pythonhosted.org/packages/93/fa/d46bfe92f2c66d35916344176ff87fa839aac9c16849652947e722b7a15f/immutables-0.21-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:338bede057250b33716a3e4892e15df0bf5a5ddbf1d67ead996b3e680b49ef9e", size = 99966, upload-time = "2024-10-10T00:54:34.046Z" }, - { url = "https://files.pythonhosted.org/packages/d7/f5/2a19e2e095f7a39d8d77dcc10669734d2d99773ce00c99bdcfeeb7d714e6/immutables-0.21-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8781c89583b68f604cf30f0978b722165824c3075888639fde771bf1a3e12dc0", size = 101773, upload-time = "2024-10-10T00:54:35.851Z" }, -] - -[[package]] -name = "importlib-metadata" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "zipp", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7", size = 27789, upload-time = "2026-03-20T06:42:55.665Z" }, -] - -[[package]] -name = "importlib-resources" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e4/06/b56dfa750b44e86157093bc8fca0ab81dccbf5260510de4eaf1cb69b5b99/importlib_resources-7.1.0.tar.gz", hash = "sha256:0722d4c6212489c530f2a145a34c0a7a3b4721bc96a15fada5930e2a0b760708", size = 44985, upload-time = "2026-04-12T16:36:09.232Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/db/55a262f3606bebcae07cc14095338471ad7c0bbcaa37707e6f0ee49725b7/importlib_resources-7.1.0-py3-none-any.whl", hash = "sha256:1bd7b48b4088eddb2cd16382150bb515af0bd2c70128194392725f82ad2c96a1", size = 37232, upload-time = "2026-04-12T16:36:08.219Z" }, -] - -[[package]] -name = "iniconfig" -version = "2.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, -] - -[[package]] -name = "ipykernel" -version = "7.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "comm", marker = "sys_platform == 'linux'" }, - { name = "debugpy", marker = "sys_platform == 'linux'" }, - { name = "ipython", marker = "sys_platform == 'linux'" }, - { name = "jupyter-client", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "matplotlib-inline", marker = "sys_platform == 'linux'" }, - { name = "nest-asyncio2", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "psutil", marker = "sys_platform == 'linux'" }, - { name = "pyzmq", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3d/c4/e4a38f579de4225a561305666f7541cdabb30075def2aa1ac17bd73c1fb5/ipykernel-7.3.0.tar.gz", hash = "sha256:9acaaaf97d16355166e4085afe9d225bfbdf2b7ef520f9df3be8f2b248275e09", size = 184899, upload-time = "2026-06-10T08:41:25.481Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/02/77b271f5dc58bfbc0b577c877b2365d1ffea2afe66a80c13f2312820348c/ipykernel-7.3.0-py3-none-any.whl", hash = "sha256:897eb64da762549ef610698fca5e9675195ec6ac8ec7f19d81ce1ca20c876057", size = 120583, upload-time = "2026-06-10T08:41:23.648Z" }, -] - -[[package]] -name = "ipython" -version = "9.15.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "decorator", marker = "sys_platform == 'linux'" }, - { name = "ipython-pygments-lexers", marker = "sys_platform == 'linux'" }, - { name = "jedi", marker = "sys_platform == 'linux'" }, - { name = "matplotlib-inline", marker = "sys_platform == 'linux'" }, - { name = "pexpect", marker = "sys_platform == 'linux'" }, - { name = "prompt-toolkit", marker = "sys_platform == 'linux'" }, - { name = "psutil", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, - { name = "stack-data", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/53/59/165d3b4d75cc34add3122c4417ecb229085140ac573103c223cd01dde96f/ipython-9.15.0.tar.gz", hash = "sha256:da2819ce2aa83135257df830660b1176d986c3d2876db24df01974fa955b2756", size = 4442580, upload-time = "2026-06-26T11:03:35.913Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/40/3a/948263ca3b9d65bb2b1b0c521b3a49fad5d59ada58724bd87d2bd5ff3f36/ipython-9.15.0-py3-none-any.whl", hash = "sha256:515ad9c3cdf0c932a5a9f6245419e8aba706b7bd03c3e1d3a1c83d9351d6aa6e", size = 630895, upload-time = "2026-06-26T11:03:33.809Z" }, -] - -[[package]] -name = "ipython-genutils" -version = "0.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e8/69/fbeffffc05236398ebfcfb512b6d2511c622871dca1746361006da310399/ipython_genutils-0.2.0.tar.gz", hash = "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8", size = 22208, upload-time = "2017-03-13T22:12:26.393Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/bc/9bd3b5c2b4774d5f33b2d544f1460be9df7df2fe42f352135381c347c69a/ipython_genutils-0.2.0-py2.py3-none-any.whl", hash = "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8", size = 26343, upload-time = "2017-03-13T22:12:25.412Z" }, -] - -[[package]] -name = "ipython-pygments-lexers" -version = "1.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pygments", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, -] - -[[package]] -name = "ipywidgets" -version = "8.1.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "comm", marker = "sys_platform == 'linux'" }, - { name = "ipython", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab-widgets", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, - { name = "widgetsnbextension", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4c/ae/c5ce1edc1afe042eadb445e95b0671b03cee61895264357956e61c0d2ac0/ipywidgets-8.1.8.tar.gz", hash = "sha256:61f969306b95f85fba6b6986b7fe45d73124d1d9e3023a8068710d47a22ea668", size = 116739, upload-time = "2025-11-01T21:18:12.393Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl", hash = "sha256:ecaca67aed704a338f88f67b1181b58f821ab5dc89c1f0f5ef99db43c1c2921e", size = 139808, upload-time = "2025-11-01T21:18:10.956Z" }, -] - -[[package]] -name = "isoduration" -version = "20.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "arrow", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7c/1a/3c8edc664e06e6bd06cce40c6b22da5f1429aa4224d0c590f3be21c91ead/isoduration-20.11.0.tar.gz", hash = "sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9", size = 11649, upload-time = "2020-11-01T11:00:00.312Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl", hash = "sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042", size = 11321, upload-time = "2020-11-01T10:59:58.02Z" }, -] - -[[package]] -name = "jaraco-classes" -version = "3.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "more-itertools", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", size = 6777, upload-time = "2024-03-31T07:27:34.792Z" }, -] - -[[package]] -name = "jaraco-context" -version = "6.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/af/50/4763cd07e722bb6285316d390a164bc7e479db9d90daa769f22578f698b4/jaraco_context-6.1.2.tar.gz", hash = "sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3", size = 16801, upload-time = "2026-03-20T22:13:33.922Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/58/bc8954bda5fcda97bd7c19be11b85f91973d67a706ed4a3aec33e7de22db/jaraco_context-6.1.2-py3-none-any.whl", hash = "sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535", size = 7871, upload-time = "2026-03-20T22:13:32.808Z" }, -] - -[[package]] -name = "jaraco-functools" -version = "4.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "more-itertools", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/36/cf/ea4ef2920830dea3f5ab2ea4da6fb67724e6dca80ee2553788c3607243d0/jaraco_functools-4.5.0.tar.gz", hash = "sha256:3bb5665ea4a020cf78a7040e89154c77edadb3ca74f366479669c5999aa70b03", size = 20272, upload-time = "2026-05-15T21:34:10.025Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/96/9a/982e48afcffcd727a9144506720ffd4224b6b7e355c98641866f38b7c043/jaraco_functools-4.5.0-py3-none-any.whl", hash = "sha256:79ce39246eddbde4b3a03b77ea5f0f7878dc669b166a66cf3fa8e266aa3fa2f4", size = 10594, upload-time = "2026-05-15T21:34:08.595Z" }, -] - -[[package]] -name = "jedi" -version = "0.20.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "parso", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/46/b7/a3635f6a2d7cf5b5dd98064fc1d5fbbafcb25477bcea204a3a92145d158b/jedi-0.20.0.tar.gz", hash = "sha256:c3f4ccbd276696f4b19c54618d4fb18f9fc24b0aef02acf704b23f487daa1011", size = 3119416, upload-time = "2026-05-01T23:38:47.814Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl", hash = "sha256:7bdd9c2634f56713299976f4cbd59cb3fa92165cc5e05ea811fb253480728b67", size = 4884812, upload-time = "2026-05-01T23:38:43.919Z" }, -] - -[[package]] -name = "jeepney" -version = "0.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758, upload-time = "2025-02-27T18:51:01.684Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, -] - -[[package]] -name = "jinja2" -version = "3.1.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, -] - -[[package]] -name = "joblib" -version = "1.5.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, -] - -[[package]] -name = "json5" -version = "0.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e4/7d/05c46a96a78147ae3bf99c2f4169ce144a70220b8d6fcd56f6ec368b8ce9/json5-0.15.0.tar.gz", hash = "sha256:7424d1f1eb1d56da6e3d70643f53619862b4ce81440bdb8ecfd6f875e5ba4a71", size = 53278, upload-time = "2026-06-19T20:08:27.716Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/be/59527c99478aade6bb33a68d72e6e18dd4e6ff6eacfc7d01bdb15bc76912/json5-0.15.0-py3-none-any.whl", hash = "sha256:56636a30c0e8a4665fe2179c0212f32eae3796dea89ea6f649b9436ecdb39618", size = 36570, upload-time = "2026-06-19T20:08:26.748Z" }, -] - -[[package]] -name = "jsonargparse" -version = "4.49.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/87/0b/f012466077d803b546955bf827ab30a320ad1550e3167195432cc2d6bfad/jsonargparse-4.49.0.tar.gz", hash = "sha256:9e691a09d937fb4c6bc1f6d3bf6c3546efa03381ea1e64ee10759095f1b14da7", size = 125237, upload-time = "2026-05-15T06:48:44.807Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/61/64/51cbc9044f5b1ee4fb9894fe791f50dfdb52dc080acfafb5818ba19e8c27/jsonargparse-4.49.0-py3-none-any.whl", hash = "sha256:2025880765fd792f6fb9861aa93f26da5f6c350be5d7656321369b22b0d2538e", size = 134834, upload-time = "2026-05-15T06:48:40.76Z" }, -] - -[package.optional-dependencies] -signatures = [ - { name = "docstring-parser", marker = "sys_platform == 'linux'" }, - { name = "typeshed-client", marker = "sys_platform == 'linux'" }, -] - -[[package]] -name = "jsonpointer" -version = "3.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/18/c7/af399a2e7a67fd18d63c40c5e62d3af4e67b836a2107468b6a5ea24c4304/jsonpointer-3.1.1.tar.gz", hash = "sha256:0b801c7db33a904024f6004d526dcc53bbb8a4a0f4e32bfd10beadf60adf1900", size = 9068, upload-time = "2026-03-23T22:32:32.458Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl", hash = "sha256:8ff8b95779d071ba472cf5bc913028df06031797532f08a7d5b602d8b2a488ca", size = 7659, upload-time = "2026-03-23T22:32:31.568Z" }, -] - -[[package]] -name = "jsonschema" -version = "4.26.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs", marker = "sys_platform == 'linux'" }, - { name = "jsonschema-specifications", marker = "sys_platform == 'linux'" }, - { name = "referencing", marker = "sys_platform == 'linux'" }, - { name = "rpds-py", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, -] - -[package.optional-dependencies] -format-nongpl = [ - { name = "fqdn", marker = "sys_platform == 'linux'" }, - { name = "idna", marker = "sys_platform == 'linux'" }, - { name = "isoduration", marker = "sys_platform == 'linux'" }, - { name = "jsonpointer", marker = "sys_platform == 'linux'" }, - { name = "rfc3339-validator", marker = "sys_platform == 'linux'" }, - { name = "rfc3986-validator", marker = "sys_platform == 'linux'" }, - { name = "rfc3987-syntax", marker = "sys_platform == 'linux'" }, - { name = "uri-template", marker = "sys_platform == 'linux'" }, - { name = "webcolors", marker = "sys_platform == 'linux'" }, -] - -[[package]] -name = "jsonschema-specifications" -version = "2025.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "referencing", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, -] - -[[package]] -name = "jupyter" -version = "1.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ipykernel", marker = "sys_platform == 'linux'" }, - { name = "ipywidgets", marker = "sys_platform == 'linux'" }, - { name = "jupyter-console", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab", marker = "sys_platform == 'linux'" }, - { name = "nbconvert", marker = "sys_platform == 'linux'" }, - { name = "notebook", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/58/f3/af28ea964ab8bc1e472dba2e82627d36d470c51f5cd38c37502eeffaa25e/jupyter-1.1.1.tar.gz", hash = "sha256:d55467bceabdea49d7e3624af7e33d59c37fff53ed3a350e1ac957bed731de7a", size = 5714959, upload-time = "2024-08-30T07:15:48.299Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/64/285f20a31679bf547b75602702f7800e74dbabae36ef324f716c02804753/jupyter-1.1.1-py2.py3-none-any.whl", hash = "sha256:7a59533c22af65439b24bbe60373a4e95af8f16ac65a6c00820ad378e3f7cc83", size = 2657, upload-time = "2024-08-30T07:15:47.045Z" }, -] - -[[package]] -name = "jupyter-builder" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0d/df/db5efc4e28803a1421350445e53d85ec571a4e6928e3524ccc39f4e16584/jupyter_builder-1.1.0.tar.gz", hash = "sha256:b996e8af616900f18724fa34883169d869be2205497940bec78a3a1031eb897d", size = 971142, upload-time = "2026-07-11T07:24:02.4Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/fb/53adbc72931b4429275b160044c3e1bcbc8fbb5ea6b8f318a357834842fb/jupyter_builder-1.1.0-py3-none-any.whl", hash = "sha256:d9d5280e8845f726663545c3a6db55bd205127616eeb8958481091d6cba71b6a", size = 912964, upload-time = "2026-07-11T07:24:00.279Z" }, -] - -[[package]] -name = "jupyter-client" -version = "8.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "python-dateutil", marker = "sys_platform == 'linux'" }, - { name = "pyzmq", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7d/dc/5512503b088997c2250b8bf18258fba9d9ce5ead641183700960d3c9d342/jupyter_client-8.9.1.tar.gz", hash = "sha256:a58f730dd9e728ba16ba1d62ebccf7ffe1ebbdbce4e95cfae941b7321ae1f4fa", size = 359256, upload-time = "2026-06-09T13:15:01.033Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/6f/56d39bf385c5c27988aebaf0c18a2a17e960575740100973511018bd904e/jupyter_client-8.9.1-py3-none-any.whl", hash = "sha256:0b7a295bc46e8751e9adae84781f726c851c1d911bd793edc4a3bde942e3da81", size = 109828, upload-time = "2026-06-09T13:14:58.835Z" }, -] - -[[package]] -name = "jupyter-console" -version = "6.6.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ipykernel", marker = "sys_platform == 'linux'" }, - { name = "ipython", marker = "sys_platform == 'linux'" }, - { name = "jupyter-client", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "prompt-toolkit", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, - { name = "pyzmq", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bd/2d/e2fd31e2fc41c14e2bcb6c976ab732597e907523f6b2420305f9fc7fdbdb/jupyter_console-6.6.3.tar.gz", hash = "sha256:566a4bf31c87adbfadf22cdf846e3069b59a71ed5da71d6ba4d8aaad14a53539", size = 34363, upload-time = "2023-03-06T14:13:31.02Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/77/71d78d58f15c22db16328a476426f7ac4a60d3a5a7ba3b9627ee2f7903d4/jupyter_console-6.6.3-py3-none-any.whl", hash = "sha256:309d33409fcc92ffdad25f0bcdf9a4a9daa61b6f341177570fdac03de5352485", size = 24510, upload-time = "2023-03-06T14:13:28.229Z" }, -] - -[[package]] -name = "jupyter-core" -version = "5.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "platformdirs", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/02/49/9d1284d0dc65e2c757b74c6687b6d319b02f822ad039e5c512df9194d9dd/jupyter_core-5.9.1.tar.gz", hash = "sha256:4d09aaff303b9566c3ce657f580bd089ff5c91f5f89cf7d8846c3cdf465b5508", size = 89814, upload-time = "2025-10-16T19:19:18.444Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl", hash = "sha256:ebf87fdc6073d142e114c72c9e29a9d7ca03fad818c5d300ce2adc1fb0743407", size = 29032, upload-time = "2025-10-16T19:19:16.783Z" }, -] - -[[package]] -name = "jupyter-events" -version = "0.12.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jsonschema", extra = ["format-nongpl"], marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "python-json-logger", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "referencing", marker = "sys_platform == 'linux'" }, - { name = "rfc3339-validator", marker = "sys_platform == 'linux'" }, - { name = "rfc3986-validator", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/f8/475c4241b2b75af0deaae453ed003c6c851766dbc44d332d8baf245dc931/jupyter_events-0.12.1.tar.gz", hash = "sha256:faff25f77218335752f35f23c5fe6e4a392a7bd99a5939ccb9b8fbf594636cf3", size = 62854, upload-time = "2026-04-20T23:17:50.66Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/6c/6fcde0c8f616ed360ffd3587f7db9e225a7e62b583a04494d2f069cf64ea/jupyter_events-0.12.1-py3-none-any.whl", hash = "sha256:c366585253f537a627da52fa7ca7410c5b5301fe893f511e7b077c2d93ec8bcf", size = 19512, upload-time = "2026-04-20T23:17:48.927Z" }, -] - -[[package]] -name = "jupyter-lsp" -version = "2.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jupyter-server", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/36/ff/1e4a61f5170a9a1d978f3ac3872449de6c01fc71eaf89657824c878b1549/jupyter_lsp-2.3.1.tar.gz", hash = "sha256:fdf8a4aa7d85813976d6e29e95e6a2c8f752701f926f2715305249a3829805a6", size = 55677, upload-time = "2026-04-02T08:10:06.749Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/23/e8/9d61dcbd1dce8ef418f06befd4ac084b4720429c26b0b1222bc218685eff/jupyter_lsp-2.3.1-py3-none-any.whl", hash = "sha256:71b954d834e85ff3096400554f2eefaf7fe37053036f9a782b0f7c5e42dadb81", size = 77513, upload-time = "2026-04-02T08:10:01.753Z" }, -] - -[[package]] -name = "jupyter-server" -version = "2.20.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio", marker = "sys_platform == 'linux'" }, - { name = "argon2-cffi", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "jupyter-client", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "jupyter-events", marker = "sys_platform == 'linux'" }, - { name = "jupyter-server-terminals", marker = "sys_platform == 'linux'" }, - { name = "nbconvert", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "prometheus-client", marker = "sys_platform == 'linux'" }, - { name = "pyzmq", marker = "sys_platform == 'linux'" }, - { name = "send2trash", marker = "sys_platform == 'linux'" }, - { name = "terminado", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, - { name = "websocket-client", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6b/dc/db3a582633170186f8c8b31298d7eb26ad0eb031a1f53476c258b64eed05/jupyter_server-2.20.0.tar.gz", hash = "sha256:b5778ba337d8015a3dc2b80803ecdd5ac18d3797fddf61a50ea5fb472b4ebe14", size = 756523, upload-time = "2026-06-17T12:09:09.435Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/71/8c002223e873a870f5c41dc69b0a7c922301123e4a31d5d01ecb700aef77/jupyter_server-2.20.0-py3-none-any.whl", hash = "sha256:c3b67c93c471e947c18b5026f04f21614218adb706df8f48227d3ee8e0a7cdcc", size = 393143, upload-time = "2026-06-17T12:09:07.234Z" }, -] - -[[package]] -name = "jupyter-server-terminals" -version = "0.5.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "terminado", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f4/a7/bcd0a9b0cbba88986fe944aaaf91bfda603e5a50bda8ed15123f381a3b2f/jupyter_server_terminals-0.5.4.tar.gz", hash = "sha256:bbda128ed41d0be9020349f9f1f2a4ab9952a73ed5f5ac9f1419794761fb87f5", size = 31770, upload-time = "2026-01-14T16:53:20.213Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/2d/6674563f71c6320841fc300911a55143925112a72a883e2ca71fba4c618d/jupyter_server_terminals-0.5.4-py3-none-any.whl", hash = "sha256:55be353fc74a80bc7f3b20e6be50a55a61cd525626f578dcb66a5708e2007d14", size = 13704, upload-time = "2026-01-14T16:53:18.738Z" }, -] - -[[package]] -name = "jupyterlab" -version = "4.6.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "async-lru", marker = "sys_platform == 'linux'" }, - { name = "httpx", marker = "sys_platform == 'linux'" }, - { name = "ipykernel", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "jupyter-builder", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "jupyter-lsp", marker = "sys_platform == 'linux'" }, - { name = "jupyter-server", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab-server", marker = "sys_platform == 'linux'" }, - { name = "notebook-shim", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bc/2a/d6af53bfd45a43a5bfe7e40ba47ee7a8921a807daf4bb708e3a295bbb54d/jupyterlab-4.6.1.tar.gz", hash = "sha256:75315982ed28427edaa62bb85eadb5105e4043a757643c910efd787fe6ed0837", size = 28179125, upload-time = "2026-06-29T12:48:45.402Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/81/90ac6cc31d248e83a0d1eab343a5e6e68bc783d3f74fbe61640f42a61da4/jupyterlab-4.6.1-py3-none-any.whl", hash = "sha256:85a58546c831f3dce6cf919468c26874c9065e99c42279fb4abb8e1b552a98bb", size = 17164660, upload-time = "2026-06-29T12:48:41.21Z" }, -] - -[[package]] -name = "jupyterlab-pygments" -version = "0.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/90/51/9187be60d989df97f5f0aba133fa54e7300f17616e065d1ada7d7646b6d6/jupyterlab_pygments-0.3.0.tar.gz", hash = "sha256:721aca4d9029252b11cfa9d185e5b5af4d54772bb8072f9b7036f4170054d35d", size = 512900, upload-time = "2023-11-23T09:26:37.44Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/dd/ead9d8ea85bf202d90cc513b533f9c363121c7792674f78e0d8a854b63b4/jupyterlab_pygments-0.3.0-py3-none-any.whl", hash = "sha256:841a89020971da1d8693f1a99997aefc5dc424bb1b251fd6322462a1b8842780", size = 15884, upload-time = "2023-11-23T09:26:34.325Z" }, -] - -[[package]] -name = "jupyterlab-server" -version = "2.28.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "babel", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "json5", marker = "sys_platform == 'linux'" }, - { name = "jsonschema", marker = "sys_platform == 'linux'" }, - { name = "jupyter-server", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d6/2c/90153f189e421e93c4bb4f9e3f59802a1f01abd2ac5cf40b152d7f735232/jupyterlab_server-2.28.0.tar.gz", hash = "sha256:35baa81898b15f93573e2deca50d11ac0ae407ebb688299d3a5213265033712c", size = 76996, upload-time = "2025-10-22T13:59:18.37Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/07/a000fe835f76b7e1143242ab1122e6362ef1c03f23f83a045c38859c2ae0/jupyterlab_server-2.28.0-py3-none-any.whl", hash = "sha256:e4355b148fdcf34d312bbbc80f22467d6d20460e8b8736bf235577dd18506968", size = 59830, upload-time = "2025-10-22T13:59:16.767Z" }, -] - -[[package]] -name = "jupyterlab-widgets" -version = "3.0.16" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/2d/ef58fed122b268c69c0aa099da20bc67657cdfb2e222688d5731bd5b971d/jupyterlab_widgets-3.0.16.tar.gz", hash = "sha256:423da05071d55cf27a9e602216d35a3a65a3e41cdf9c5d3b643b814ce38c19e0", size = 897423, upload-time = "2025-11-01T21:11:29.724Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/b5/36c712098e6191d1b4e349304ef73a8d06aed77e56ceaac8c0a306c7bda1/jupyterlab_widgets-3.0.16-py3-none-any.whl", hash = "sha256:45fa36d9c6422cf2559198e4db481aa243c7a32d9926b500781c830c80f7ecf8", size = 914926, upload-time = "2025-11-01T21:11:28.008Z" }, -] - -[[package]] -name = "jupytext" -version = "1.19.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, - { name = "mdit-py-plugins", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3b/52/e014296ac8f40ca783aeb73dae52e65edbb0eaae0dcdc1ea41bfaa8aebf7/jupytext-1.19.4.tar.gz", hash = "sha256:739bcd4bc12aa4fe298a38017cdb5ae27b08a6ba3a5470728d2fe9e04b155db1", size = 4581977, upload-time = "2026-06-21T21:48:58.32Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/e9/e2ae007456069dbe01865c69a4203a7ada6f7e337b78fc2f12e51bd3fae7/jupytext-1.19.4-py3-none-any.whl", hash = "sha256:032d4ef4bd2e96addcac780b9b1d6b5a266ca39beceaaca95bfb4f06e0b77029", size = 170889, upload-time = "2026-06-21T21:48:56.352Z" }, -] - -[[package]] -name = "keyring" -version = "25.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jaraco-classes", marker = "sys_platform == 'linux'" }, - { name = "jaraco-context", marker = "sys_platform == 'linux'" }, - { name = "jaraco-functools", marker = "sys_platform == 'linux'" }, - { name = "jeepney", marker = "sys_platform == 'linux'" }, - { name = "secretstorage", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516, upload-time = "2025-11-16T16:26:09.482Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, -] - -[[package]] -name = "kiwisolver" -version = "1.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d0/67/9c61eccb13f0bdca9307614e782fec49ffdde0f7a2314935d489fa93cd9c/kiwisolver-1.5.0.tar.gz", hash = "sha256:d4193f3d9dc3f6f79aaed0e5637f45d98850ebf01f7ca20e69457f3e8946b66a", size = 103482, upload-time = "2026-03-09T13:15:53.382Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/13/680c54afe3e65767bed7ec1a15571e1a2f1257128733851ade24abcefbcc/kiwisolver-1.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb5136fb5352d3f422df33f0c879a1b0c204004324150cc3b5e3c4f310c9049f", size = 1477934, upload-time = "2026-03-09T13:13:27.166Z" }, - { url = "https://files.pythonhosted.org/packages/c8/2f/cebfcdb60fd6a9b0f6b47a9337198bcbad6fbe15e68189b7011fd914911f/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2af221f268f5af85e776a73d62b0845fc8baf8ef0abfae79d29c77d0e776aaf", size = 1278537, upload-time = "2026-03-09T13:13:28.707Z" }, - { url = "https://files.pythonhosted.org/packages/f2/0d/9b782923aada3fafb1d6b84e13121954515c669b18af0c26e7d21f579855/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b0f172dc8ffaccb8522d7c5d899de00133f2f1ca7b0a49b7da98e901de87bf2d", size = 1296685, upload-time = "2026-03-09T13:13:30.528Z" }, - { url = "https://files.pythonhosted.org/packages/27/70/83241b6634b04fe44e892688d5208332bde130f38e610c0418f9ede47ded/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6ab8ba9152203feec73758dad83af9a0bbe05001eb4639e547207c40cfb52083", size = 1346024, upload-time = "2026-03-09T13:13:32.818Z" }, - { url = "https://files.pythonhosted.org/packages/e4/db/30ed226fb271ae1a6431fc0fe0edffb2efe23cadb01e798caeb9f2ceae8f/kiwisolver-1.5.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:cdee07c4d7f6d72008d3f73b9bf027f4e11550224c7c50d8df1ae4a37c1402a6", size = 987241, upload-time = "2026-03-09T13:13:34.435Z" }, - { url = "https://files.pythonhosted.org/packages/ec/bd/c314595208e4c9587652d50959ead9e461995389664e490f4dce7ff0f782/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7c60d3c9b06fb23bd9c6139281ccbdc384297579ae037f08ae90c69f6845c0b1", size = 2227742, upload-time = "2026-03-09T13:13:36.4Z" }, - { url = "https://files.pythonhosted.org/packages/c1/43/0499cec932d935229b5543d073c2b87c9c22846aab48881e9d8d6e742a2d/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e315e5ec90d88e140f57696ff85b484ff68bb311e36f2c414aa4286293e6dee0", size = 2323966, upload-time = "2026-03-09T13:13:38.204Z" }, - { url = "https://files.pythonhosted.org/packages/3d/6f/79b0d760907965acfd9d61826a3d41f8f093c538f55cd2633d3f0db269f6/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:1465387ac63576c3e125e5337a6892b9e99e0627d52317f3ca79e6930d889d15", size = 1977417, upload-time = "2026-03-09T13:13:39.966Z" }, - { url = "https://files.pythonhosted.org/packages/ab/31/01d0537c41cb75a551a438c3c7a80d0c60d60b81f694dac83dd436aec0d0/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:530a3fd64c87cffa844d4b6b9768774763d9caa299e9b75d8eca6a4423b31314", size = 2491238, upload-time = "2026-03-09T13:13:41.698Z" }, - { url = "https://files.pythonhosted.org/packages/e4/34/8aefdd0be9cfd00a44509251ba864f5caf2991e36772e61c408007e7f417/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1d9daea4ea6b9be74fe2f01f7fbade8d6ffab263e781274cffca0dba9be9eec9", size = 2294947, upload-time = "2026-03-09T13:13:43.343Z" }, - { url = "https://files.pythonhosted.org/packages/2b/0a/7b98e1e119878a27ba8618ca1e18b14f992ff1eda40f47bccccf4de44121/kiwisolver-1.5.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:332b4f0145c30b5f5ad9374881133e5aa64320428a57c2c2b61e9d891a51c2f3", size = 1477903, upload-time = "2026-03-09T13:13:52.084Z" }, - { url = "https://files.pythonhosted.org/packages/18/d8/55638d89ffd27799d5cc3d8aa28e12f4ce7a64d67b285114dbedc8ea4136/kiwisolver-1.5.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c50b89ffd3e1a911c69a1dd3de7173c0cd10b130f56222e57898683841e4f96", size = 1278751, upload-time = "2026-03-09T13:13:54.673Z" }, - { url = "https://files.pythonhosted.org/packages/b8/97/b4c8d0d18421ecceba20ad8701358453b88e32414e6f6950b5a4bad54e65/kiwisolver-1.5.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4db576bb8c3ef9365f8b40fe0f671644de6736ae2c27a2c62d7d8a1b4329f099", size = 1296793, upload-time = "2026-03-09T13:13:56.287Z" }, - { url = "https://files.pythonhosted.org/packages/c4/10/f862f94b6389d8957448ec9df59450b81bec4abb318805375c401a1e6892/kiwisolver-1.5.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0b85aad90cea8ac6797a53b5d5f2e967334fa4d1149f031c4537569972596cb8", size = 1346041, upload-time = "2026-03-09T13:13:58.269Z" }, - { url = "https://files.pythonhosted.org/packages/a3/6a/f1650af35821eaf09de398ec0bc2aefc8f211f0cda50204c9f1673741ba9/kiwisolver-1.5.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:d36ca54cb4c6c4686f7cbb7b817f66f5911c12ddb519450bbe86707155028f87", size = 987292, upload-time = "2026-03-09T13:13:59.871Z" }, - { url = "https://files.pythonhosted.org/packages/de/19/d7fb82984b9238115fe629c915007be608ebd23dc8629703d917dbfaffd4/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:38f4a703656f493b0ad185211ccfca7f0386120f022066b018eb5296d8613e23", size = 2227865, upload-time = "2026-03-09T13:14:01.401Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b9/46b7f386589fd222dac9e9de9c956ce5bcefe2ee73b4e79891381dda8654/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3ac2360e93cb41be81121755c6462cff3beaa9967188c866e5fce5cf13170859", size = 2324369, upload-time = "2026-03-09T13:14:02.972Z" }, - { url = "https://files.pythonhosted.org/packages/92/8b/95e237cf3d9c642960153c769ddcbe278f182c8affb20cecc1cc983e7cc5/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c95cab08d1965db3d84a121f1c7ce7479bdd4072c9b3dafd8fecce48a2e6b902", size = 1977989, upload-time = "2026-03-09T13:14:04.503Z" }, - { url = "https://files.pythonhosted.org/packages/1b/95/980c9df53501892784997820136c01f62bc1865e31b82b9560f980c0e649/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fc20894c3d21194d8041a28b65622d5b86db786da6e3cfe73f0c762951a61167", size = 2491645, upload-time = "2026-03-09T13:14:06.106Z" }, - { url = "https://files.pythonhosted.org/packages/cb/32/900647fd0840abebe1561792c6b31e6a7c0e278fc3973d30572a965ca14c/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7a32f72973f0f950c1920475d5c5ea3d971b81b6f0ec53b8d0a956cc965f22e0", size = 2295237, upload-time = "2026-03-09T13:14:08.891Z" }, - { url = "https://files.pythonhosted.org/packages/f0/7f/f943879cda9007c45e1f7dba216d705c3a18d6b35830e488b6c6a4e7cdf0/kiwisolver-1.5.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4432b835675f0ea7414aab3d37d119f7226d24869b7a829caeab49ebda407b0c", size = 1584848, upload-time = "2026-03-09T13:14:19.745Z" }, - { url = "https://files.pythonhosted.org/packages/37/f8/4d4f85cc1870c127c88d950913370dd76138482161cd07eabbc450deff01/kiwisolver-1.5.0-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b0feb50971481a2cc44d94e88bdb02cdd497618252ae226b8eb1201b957e368", size = 1391542, upload-time = "2026-03-09T13:14:21.54Z" }, - { url = "https://files.pythonhosted.org/packages/04/0b/65dd2916c84d252b244bd405303220f729e7c17c9d7d33dca6feeff9ffc4/kiwisolver-1.5.0-cp313-cp313t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56fa888f10d0f367155e76ce849fa1166fc9730d13bd2d65a2aa13b6f5424489", size = 1404447, upload-time = "2026-03-09T13:14:23.205Z" }, - { url = "https://files.pythonhosted.org/packages/39/5c/2606a373247babce9b1d056c03a04b65f3cf5290a8eac5d7bdead0a17e21/kiwisolver-1.5.0-cp313-cp313t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:940dda65d5e764406b9fb92761cbf462e4e63f712ab60ed98f70552e496f3bf1", size = 1455918, upload-time = "2026-03-09T13:14:24.74Z" }, - { url = "https://files.pythonhosted.org/packages/d5/d1/c6078b5756670658e9192a2ef11e939c92918833d2745f85cd14a6004bdf/kiwisolver-1.5.0-cp313-cp313t-manylinux_2_39_riscv64.whl", hash = "sha256:89fc958c702ee9a745e4700378f5d23fddbc46ff89e8fdbf5395c24d5c1452a3", size = 1072856, upload-time = "2026-03-09T13:14:26.597Z" }, - { url = "https://files.pythonhosted.org/packages/cb/c8/7def6ddf16eb2b3741d8b172bdaa9af882b03c78e9b0772975408801fa63/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9027d773c4ff81487181a925945743413f6069634d0b122d0b37684ccf4f1e18", size = 2333580, upload-time = "2026-03-09T13:14:28.237Z" }, - { url = "https://files.pythonhosted.org/packages/9e/87/2ac1fce0eb1e616fcd3c35caa23e665e9b1948bb984f4764790924594128/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:5b233ea3e165e43e35dba1d2b8ecc21cf070b45b65ae17dd2747d2713d942021", size = 2423018, upload-time = "2026-03-09T13:14:30.018Z" }, - { url = "https://files.pythonhosted.org/packages/67/13/c6700ccc6cc218716bfcda4935e4b2997039869b4ad8a94f364c5a3b8e63/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ce9bf03dad3b46408c08649c6fbd6ca28a9fce0eb32fdfffa6775a13103b5310", size = 2062804, upload-time = "2026-03-09T13:14:32.888Z" }, - { url = "https://files.pythonhosted.org/packages/1b/bd/877056304626943ff0f1f44c08f584300c199b887cb3176cd7e34f1515f1/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:fc4d3f1fb9ca0ae9f97b095963bc6326f1dbfd3779d6679a1e016b9baaa153d3", size = 2597482, upload-time = "2026-03-09T13:14:34.971Z" }, - { url = "https://files.pythonhosted.org/packages/75/19/c60626c47bf0f8ac5dcf72c6c98e266d714f2fbbfd50cf6dab5ede3aaa50/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f443b4825c50a51ee68585522ab4a1d1257fac65896f282b4c6763337ac9f5d2", size = 2394328, upload-time = "2026-03-09T13:14:36.816Z" }, - { url = "https://files.pythonhosted.org/packages/e7/f9/b06c934a6aa8bc91f566bd2a214fd04c30506c2d9e2b6b171953216a65b6/kiwisolver-1.5.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:80aa065ffd378ff784822a6d7c3212f2d5f5e9c3589614b5c228b311fd3063ac", size = 1475913, upload-time = "2026-03-09T13:14:46.247Z" }, - { url = "https://files.pythonhosted.org/packages/6b/f0/f768ae564a710135630672981231320bc403cf9152b5596ec5289de0f106/kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e7f886f47ab881692f278ae901039a234e4025a68e6dfab514263a0b1c4ae05", size = 1282782, upload-time = "2026-03-09T13:14:48.458Z" }, - { url = "https://files.pythonhosted.org/packages/e2/9f/1de7aad00697325f05238a5f2eafbd487fb637cc27a558b5367a5f37fb7f/kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5060731cc3ed12ca3a8b57acd4aeca5bbc2f49216dd0bec1650a1acd89486bcd", size = 1300815, upload-time = "2026-03-09T13:14:50.721Z" }, - { url = "https://files.pythonhosted.org/packages/5a/c2/297f25141d2e468e0ce7f7a7b92e0cf8918143a0cbd3422c1ad627e85a06/kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7a4aa69609f40fce3cbc3f87b2061f042eee32f94b8f11db707b66a26461591a", size = 1347925, upload-time = "2026-03-09T13:14:52.304Z" }, - { url = "https://files.pythonhosted.org/packages/b9/d3/f4c73a02eb41520c47610207b21afa8cdd18fdbf64ffd94674ae21c4812d/kiwisolver-1.5.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:d168fda2dbff7b9b5f38e693182d792a938c31db4dac3a80a4888de603c99554", size = 991322, upload-time = "2026-03-09T13:14:54.637Z" }, - { url = "https://files.pythonhosted.org/packages/7b/46/d3f2efef7732fcda98d22bf4ad5d3d71d545167a852ca710a494f4c15343/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:413b820229730d358efd838ecbab79902fe97094565fdc80ddb6b0a18c18a581", size = 2232857, upload-time = "2026-03-09T13:14:56.471Z" }, - { url = "https://files.pythonhosted.org/packages/3f/ec/2d9756bf2b6d26ae4349b8d3662fb3993f16d80c1f971c179ce862b9dbae/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:5124d1ea754509b09e53738ec185584cc609aae4a3b510aaf4ed6aa047ef9303", size = 2329376, upload-time = "2026-03-09T13:14:58.072Z" }, - { url = "https://files.pythonhosted.org/packages/8f/9f/876a0a0f2260f1bde92e002b3019a5fabc35e0939c7d945e0fa66185eb20/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e4415a8db000bf49a6dd1c478bf70062eaacff0f462b92b0ba68791a905861f9", size = 1982549, upload-time = "2026-03-09T13:14:59.668Z" }, - { url = "https://files.pythonhosted.org/packages/6c/4f/ba3624dfac23a64d54ac4179832860cb537c1b0af06024936e82ca4154a0/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d618fd27420381a4f6044faa71f46d8bfd911bd077c555f7138ed88729bfbe79", size = 2494680, upload-time = "2026-03-09T13:15:01.364Z" }, - { url = "https://files.pythonhosted.org/packages/39/b7/97716b190ab98911b20d10bf92eca469121ec483b8ce0edd314f51bc85af/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5092eb5b1172947f57d6ea7d89b2f29650414e4293c47707eb499ec07a0ac796", size = 2297905, upload-time = "2026-03-09T13:15:03.925Z" }, - { url = "https://files.pythonhosted.org/packages/f8/8a/685b297052dd041dcebce8e8787b58923b6e78acc6115a0dc9189011c44b/kiwisolver-1.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7c4c09a490dc4d4a7f8cbee56c606a320f9dc28cf92a7157a39d1ce7676a657", size = 1584858, upload-time = "2026-03-09T13:15:15.103Z" }, - { url = "https://files.pythonhosted.org/packages/9e/80/04865e3d4638ac5bddec28908916df4a3075b8c6cc101786a96803188b96/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a075bd7bd19c70cf67c8badfa36cf7c5d8de3c9ddb8420c51e10d9c50e94920", size = 1392539, upload-time = "2026-03-09T13:15:16.661Z" }, - { url = "https://files.pythonhosted.org/packages/ba/01/77a19cacc0893fa13fafa46d1bba06fb4dc2360b3292baf4b56d8e067b24/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bdd3e53429ff02aa319ba59dfe4ceeec345bf46cf180ec2cf6fd5b942e7975e9", size = 1405310, upload-time = "2026-03-09T13:15:18.229Z" }, - { url = "https://files.pythonhosted.org/packages/53/39/bcaf5d0cca50e604cfa9b4e3ae1d64b50ca1ae5b754122396084599ef903/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cdcb35dc9d807259c981a85531048ede628eabcffb3239adf3d17463518992d", size = 1456244, upload-time = "2026-03-09T13:15:20.444Z" }, - { url = "https://files.pythonhosted.org/packages/d0/7a/72c187abc6975f6978c3e39b7cf67aeb8b3c0a8f9790aa7fd412855e9e1f/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:70d593af6a6ca332d1df73d519fddb5148edb15cd90d5f0155e3746a6d4fcc65", size = 1073154, upload-time = "2026-03-09T13:15:22.039Z" }, - { url = "https://files.pythonhosted.org/packages/c7/ca/cf5b25783ebbd59143b4371ed0c8428a278abe68d6d0104b01865b1bbd0f/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:377815a8616074cabbf3f53354e1d040c35815a134e01d7614b7692e4bf8acfa", size = 2334377, upload-time = "2026-03-09T13:15:23.741Z" }, - { url = "https://files.pythonhosted.org/packages/4a/e5/b1f492adc516796e88751282276745340e2a72dcd0d36cf7173e0daf3210/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0255a027391d52944eae1dbb5d4cc5903f57092f3674e8e544cdd2622826b3f0", size = 2425288, upload-time = "2026-03-09T13:15:25.789Z" }, - { url = "https://files.pythonhosted.org/packages/e6/e5/9b21fbe91a61b8f409d74a26498706e97a48008bfcd1864373d32a6ba31c/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:012b1eb16e28718fa782b5e61dc6f2da1f0792ca73bd05d54de6cb9561665fc9", size = 2063158, upload-time = "2026-03-09T13:15:27.63Z" }, - { url = "https://files.pythonhosted.org/packages/b1/02/83f47986138310f95ea95531f851b2a62227c11cbc3e690ae1374fe49f0f/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:0e3aafb33aed7479377e5e9a82e9d4bf87063741fc99fc7ae48b0f16e32bdd6f", size = 2597260, upload-time = "2026-03-09T13:15:29.421Z" }, - { url = "https://files.pythonhosted.org/packages/07/18/43a5f24608d8c313dd189cf838c8e68d75b115567c6279de7796197cfb6a/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e7a116ae737f0000343218c4edf5bd45893bfeaff0993c0b215d7124c9f77646", size = 2394403, upload-time = "2026-03-09T13:15:31.517Z" }, - { url = "https://files.pythonhosted.org/packages/fa/06/7399a607f434119c6e1fdc8ec89a8d51ccccadf3341dee4ead6bd14caaf5/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31c13da98624f957b0fb1b5bae5383b2333c2c3f6793d9825dd5ce79b525cb7", size = 194295, upload-time = "2026-03-09T13:15:38.22Z" }, -] - -[[package]] -name = "lark" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/34/28fff3ab31ccff1fd4f6c7c7b0ceb2b6968d8ea4950663eadcb5720591a0/lark-1.3.1.tar.gz", hash = "sha256:b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905", size = 382732, upload-time = "2025-10-27T18:25:56.653Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, -] - -[[package]] -name = "latexcodec" -version = "3.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/27/dd/4270b2c5e2ee49316c3859e62293bd2ea8e382339d63ab7bbe9f39c0ec3b/latexcodec-3.0.1.tar.gz", hash = "sha256:e78a6911cd72f9dec35031c6ec23584de6842bfbc4610a9678868d14cdfb0357", size = 31222, upload-time = "2025-06-17T18:47:34.051Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/40/23569737873cc9637fd488606347e9dd92b9fa37ba4fcda1f98ee5219a97/latexcodec-3.0.1-py3-none-any.whl", hash = "sha256:a9eb8200bff693f0437a69581f7579eb6bca25c4193515c09900ce76451e452e", size = 18532, upload-time = "2025-06-17T18:47:30.726Z" }, -] - -[[package]] -name = "lenspack" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/73/de/18cfc64028334a1907d44ba2b40e233a2c7a06d7c1e827acfd1785c0b293/lenspack-1.0.0.tar.gz", hash = "sha256:e528a5409473eb996b39695cccb744db067341eb5fac6f240cd8d0aa669c85ec", size = 32632, upload-time = "2020-09-04T08:33:53.267Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/3d/a93a1332b1412301ddb454f206507b5e0b3679e012471fbbb075ca3b0845/lenspack-1.0.0-py3-none-any.whl", hash = "sha256:f9c632697516bb681e16b3eff66a7210d6355926179b95aa7060c8936c2259cc", size = 37746, upload-time = "2020-09-04T08:33:50.114Z" }, -] - -[[package]] -name = "llvmlite" -version = "0.48.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/a0/acc8ffcd5bdc63df0097e22c719bfcd61b604358343089313a8aebbb24ab/llvmlite-0.48.0.tar.gz", hash = "sha256:543b19f9ef8f3c7c60d1468191e4ee1b1537bf9f8a3d56f64c0ddd98de92edd2", size = 184016, upload-time = "2026-07-02T20:20:05.308Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/80/f2/72409351db66d0a317ec5087e076f31fb7b773a640db8a90ce6b5cac9edd/llvmlite-0.48.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:416fa4c2c66c2c6dc6d0a402648c19206e548efa0aa1eff01ad5cdad0af8217d", size = 59890118, upload-time = "2026-07-01T18:41:44.886Z" }, - { url = "https://files.pythonhosted.org/packages/3a/27/5ae2f3722606360480707adb47f001ad89df8251d06b14ee80336e660b66/llvmlite-0.48.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f5e5a5131045b72345c71062ea1a91910dde913792b6c9b28ebb2c1c0a712e98", size = 58343459, upload-time = "2026-07-01T18:41:40.306Z" }, - { url = "https://files.pythonhosted.org/packages/1b/4a/90715fa12006d681270b08d881195b6fab3ec39572e048764a1f7f59fed7/llvmlite-0.48.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8761b9e522f55207e24424fcd98370289eec2710bf8e915c82d1053f642450dc", size = 59890120, upload-time = "2026-07-01T18:42:00.748Z" }, - { url = "https://files.pythonhosted.org/packages/70/5e/7b3e20d64650ca3c80af0cdb664ec4b575ec83d9d4dd05bea8bd31f9bbb6/llvmlite-0.48.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2fe5cb59b2063bfa039dcb8ca6481c0181bf552f340d10dcf61d7996a665556e", size = 58343457, upload-time = "2026-07-01T18:41:56.41Z" }, - { url = "https://files.pythonhosted.org/packages/d6/e1/05b50692b647cac3c18200ac485b04f342f00ed173c9cc46767274469a15/llvmlite-0.48.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:05f0103c8f2f96a37441337e3643863c01b8e83e530aff38960dcb383c54a065", size = 59890115, upload-time = "2026-07-01T18:42:17.805Z" }, - { url = "https://files.pythonhosted.org/packages/f7/c3/470b8c4ff9ae2db2f9cf5c3e73de76ed908a32788ae9eb5602d43e6a476b/llvmlite-0.48.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37d66fae72802175b0bfe1ea06e624b51e2d7aee6c3c34bbd09739b8f88e8e0b", size = 58343457, upload-time = "2026-07-01T18:42:13.217Z" }, - { url = "https://files.pythonhosted.org/packages/27/98/a29133b4728671a175f7d616fab8b1c6e1d8c269d1523581d3160697bfb1/llvmlite-0.48.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d9f952dff6c350c529997423d4fa43abae9722a884ac7ffafc37a3af676e7db", size = 59890119, upload-time = "2026-07-01T18:42:33.88Z" }, - { url = "https://files.pythonhosted.org/packages/1a/cf/7aac11a1f1c7ec54b60c7f6814e87561fb6b55b2f290455d7941eb113420/llvmlite-0.48.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:054aa7d46595935565f276cf0c1659b4f10929c996dd4a606875fae26fba2a23", size = 58343460, upload-time = "2026-07-01T18:42:29.545Z" }, -] - -[[package]] -name = "lmfit" -version = "1.3.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "asteval", marker = "sys_platform == 'linux'" }, - { name = "dill", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "uncertainties", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/e5/a35942aed2de95e228728c34609b51fe3ec9182398eac50d288eef313aa2/lmfit-1.3.4.tar.gz", hash = "sha256:3c22c28c43f717f6c5b4a3bd81e893a2149739c26a592c046f2e33c23cfbe497", size = 630720, upload-time = "2025-07-19T20:09:01.876Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/7e/7b91c89a4cf0f543a83be978657afb20c86af6d725253e319589dcc4ce52/lmfit-1.3.4-py3-none-any.whl", hash = "sha256:afce1593b42324d37ae2908249b0c55445e2f4c1a0474ff706a8e2f7b5d949fa", size = 97662, upload-time = "2025-07-19T20:09:00.32Z" }, -] - -[[package]] -name = "locket" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2f/83/97b29fe05cb6ae28d2dbd30b81e2e402a3eed5f460c26e9eaa5895ceacf5/locket-1.0.0.tar.gz", hash = "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632", size = 4350, upload-time = "2022-04-20T22:04:44.312Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/db/bc/83e112abc66cd466c6b83f99118035867cecd41802f8d044638aa78a106e/locket-1.0.0-py2.py3-none-any.whl", hash = "sha256:b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3", size = 4398, upload-time = "2022-04-20T22:04:42.23Z" }, -] - -[[package]] -name = "lsstdesc-coord" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "future", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b8/e0/6806df4cfa54927a2d8e425407611b4fc031dd125bc952b4a31deb743305/lsstdesc_coord-1.3.1.tar.gz", hash = "sha256:60f878c29e1f30a9b50bf60dca3c466dc9cfb2cbc71f0a27c575ced969de57ab", size = 41647, upload-time = "2026-02-13T18:37:32.611Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/f8/63121ece13d3dc7521e8989b4224389868f092ce13d3aae4c410b0f5de12/lsstdesc_coord-1.3.1-py3-none-any.whl", hash = "sha256:06afe1621ef5da89f51e1225b1016f9ab640f6eb955bbab1231d60342e536336", size = 27105, upload-time = "2026-02-13T18:37:31.214Z" }, -] - -[[package]] -name = "lxml" -version = "6.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/05/3b/aab6728cae887456f409b4d75e8a01856e4f04bd510de38052a47768b680/lxml-6.1.1.tar.gz", hash = "sha256:ba96ae44888e0185281e937633a743ea90d5a196c6000f82565ebb0580012d40", size = 4197430, upload-time = "2026-05-18T19:19:06.424Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/95/bb63f0fd62e554fe078e1fb3c8fe9083c14ddc7ad7fa178d10e57e071ac7/lxml-6.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c921ba5c51e4e9f63b8b00267d06566e1f63407408a0496da2d1d0bfc819c7fc", size = 4930746, upload-time = "2026-05-18T19:18:29.637Z" }, - { url = "https://files.pythonhosted.org/packages/eb/99/0013e8d9b5960f4f041cf0b73e2f80c23eb5205b1f7bfb20203243651359/lxml-6.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:54a7f95e4de5fb94e2f9f4b9055c6ba33bf3d628fd77a1d647c5923caa2cdcdc", size = 5093723, upload-time = "2026-05-18T19:18:34.168Z" }, - { url = "https://files.pythonhosted.org/packages/29/91/317b332636bfc7bddcff828d41b3307f50043f4b237e40849c333d80fa1a/lxml-6.1.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f2ec43df44b1f76249ee0a615334f9b5b060e1c8bd90e706dad2d14d02f383", size = 5005557, upload-time = "2026-05-18T19:18:39.798Z" }, - { url = "https://files.pythonhosted.org/packages/42/2f/cc9bf06afe70f9c9093ae60855d9759da9db601ec4080f7473319666ffd7/lxml-6.1.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:70ef8a7e102a1508f8121aae5b0867abd663f72c14f0a9c937e6554cb4587b7b", size = 5631036, upload-time = "2026-05-18T19:18:44.858Z" }, - { url = "https://files.pythonhosted.org/packages/08/f6/af32e23e563971ffb0fb86be52bc5be5c2c118858ffc119bf6a9039b173d/lxml-6.1.1-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ebe6af670449830d6d9b752c256a983291c766a1365ba5d5460048f9e33a7818", size = 5240367, upload-time = "2026-05-18T19:18:49.217Z" }, - { url = "https://files.pythonhosted.org/packages/78/83/8555d40948b09ce86f1bd0c68a7ac31d07b1929f92cc1b074006c97ef2d2/lxml-6.1.1-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:27acc820660aaffa4f7c087f29120e12980f7779d56d8492d263170111284740", size = 5350171, upload-time = "2026-05-18T19:18:52.779Z" }, - { url = "https://files.pythonhosted.org/packages/63/75/5d92da93729b7bad783689e6496049fa40927b45bec7bf183c981de3ca70/lxml-6.1.1-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:1db753c9115ec7100d073b744d17e25e88a8f90f5c39b2f5dd878149af59671f", size = 4694874, upload-time = "2026-05-18T19:18:55.139Z" }, - { url = "https://files.pythonhosted.org/packages/c5/b5/3aad415a9a25b822e783f15deeb4dffccf5113030f1afa2222dd929313d9/lxml-6.1.1-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c4f469aebd783bb741c2ecb2a681008fd26bfe5c16a9a72ed5467f834e810df2", size = 5244492, upload-time = "2026-05-18T19:19:01.28Z" }, - { url = "https://files.pythonhosted.org/packages/f1/a1/5fcf7eb9904b80086aa47dcf0027de07b1bb990afad2e6823144c368ae04/lxml-6.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:766b010012d59470072c1816b5b6c69f1d243e5db36ea5968e94accf430a4635", size = 5048232, upload-time = "2026-05-18T19:18:12.67Z" }, - { url = "https://files.pythonhosted.org/packages/77/74/1f601b63c7a69fcdf10fa9b148c81da8442204194f6c55509cc485c786b9/lxml-6.1.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b8d812c6011c08b8111a15e54dd990b8923692d80adf35488bee34026c35accf", size = 4777023, upload-time = "2026-05-18T19:18:15.928Z" }, - { url = "https://files.pythonhosted.org/packages/a2/b9/7a78f51aec95b1bf780d78e12705a9f6533284f8693dc5c0e6724fa53d3f/lxml-6.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:fe0306bd29505a9177aac19f1877174b0e7422c222a59f70b2cd41633448c3dc", size = 5645773, upload-time = "2026-05-18T19:18:23.223Z" }, - { url = "https://files.pythonhosted.org/packages/a5/6e/98a7b7ad54e4e74fa1f20fff776913980619d0ebe5558232d7da6580bdd8/lxml-6.1.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5ba186ad207446c65d3bb3d3e0412b032b1d9f595e59861e2354798c5703d955", size = 5233088, upload-time = "2026-05-18T19:18:31.433Z" }, - { url = "https://files.pythonhosted.org/packages/65/d1/bc0ed2427bf609f2ee10da303a6a226f9c8bce94f945dc29a32ce55de6e4/lxml-6.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aa366a1e55b8ebfe8ca8ddc3cfe75c8ebade181aeb0f661d0cb05986b647f72a", size = 5260995, upload-time = "2026-05-18T19:18:37.091Z" }, - { url = "https://files.pythonhosted.org/packages/6e/ca/ab7bfe2bf4c972af5e7878262845ead3a24a929a9b04bc11c7c1ece6c82a/lxml-6.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb7c9811bfaa8b1ed5ed319f5d370dfbcaa59d52ea64be2a5a85e18195930354", size = 4924139, upload-time = "2026-05-18T19:19:04.873Z" }, - { url = "https://files.pythonhosted.org/packages/6b/55/a0c72851dfee5ecc689f949723a73dea457758912542cb955b108eaf0d8f/lxml-6.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:762ff394d5bd56da0cf034a23dcce4e13923f15321a2adfa2ac00201dc6d3fca", size = 5082329, upload-time = "2026-05-18T19:19:09.728Z" }, - { url = "https://files.pythonhosted.org/packages/f0/b6/0608f7d61a3b96cc67e5648a3d906e31a5082093e10e7be65b3886289938/lxml-6.1.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a088f287f7d8275a33c07f2cac6c50b9319309a0200a39e7e75d80c707723099", size = 4993564, upload-time = "2026-05-18T19:19:13.608Z" }, - { url = "https://files.pythonhosted.org/packages/4c/66/ae227524b066d29d55bf0b453d93d2d793c40218657d643dcbbca13b8faf/lxml-6.1.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e902da4b04e6b52e5893900d4b8ab46068f75f3561f01bf1080957f9fd932ed6", size = 5613467, upload-time = "2026-05-18T19:19:16.228Z" }, - { url = "https://files.pythonhosted.org/packages/a6/76/dbe4a00b50385e40194231dcfe5a12c059de7cf90e89c83407d2b085b719/lxml-6.1.1-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d4962d4c66bf830a7e59ed6cfc17d148149898a3aefa8ec6e59763e6e3ed085", size = 5228304, upload-time = "2026-05-18T19:19:19.354Z" }, - { url = "https://files.pythonhosted.org/packages/1c/01/00b1b8442ed2041793336868ba0b9ea4b13d7da7c085c6404c207a63bf79/lxml-6.1.1-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:581d4c8ae690a6609e64862dd6b7c2489635c2d13907fc2b20f2bc200ff1d21e", size = 5341607, upload-time = "2026-05-18T19:19:22.297Z" }, - { url = "https://files.pythonhosted.org/packages/63/36/1ad29931e9a4638bb707869f01d423a6c815f82152138d1a40dfcfde2b95/lxml-6.1.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:876e1ff5930ed8bf295ec5ef9a8155e9b6b1876bbf1deed8b3a8069311875a8f", size = 4700168, upload-time = "2026-05-18T19:19:25.133Z" }, - { url = "https://files.pythonhosted.org/packages/3c/d1/a9536cecf9be18a0dc72d32bead283a2332d1ffebd2dd3ac70ce444686e5/lxml-6.1.1-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9eb9b5a968f6e0f6d640092a567e14529ff8cea2e29d00da6f78a79fa49f013c", size = 5232487, upload-time = "2026-05-18T19:19:28.603Z" }, - { url = "https://files.pythonhosted.org/packages/0e/77/b4fb1e03bf5d130e879214d3100092e386418807fb74dd0adc4b0a48f351/lxml-6.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:aa49e06d94aba782c6a02eecb7e507969e7e7a41b267f1b359bb35585f295d5b", size = 5044231, upload-time = "2026-05-18T19:18:42.246Z" }, - { url = "https://files.pythonhosted.org/packages/26/4c/d00daeeb0a5530c4028a9232aa1b93db3ef4ed2158c116ea73c79a9765b3/lxml-6.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:70cdfd80589d59e43e18005dd7244e8895e93db8ab6a620b7e23df5445a4e3d2", size = 4769450, upload-time = "2026-05-18T19:18:48.013Z" }, - { url = "https://files.pythonhosted.org/packages/ed/6a/715a3a8d156ce42f29cf014706f5410c2ff3b02267774110fc23266409fe/lxml-6.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:aad9aa39483ed8ec44d6d2e59e5b98a0d80676ef0d92f44bfc374836111f62f5", size = 5635874, upload-time = "2026-05-18T19:18:51.914Z" }, - { url = "https://files.pythonhosted.org/packages/45/37/0544bc21dde2a88f3a17b504e6fc79c0e01d25a33c2f6079724e9e72b9c7/lxml-6.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d49514be2f28d895c38cf9d2b72d7b9a07d00314519f456c0b50b53cfcf4c785", size = 5223987, upload-time = "2026-05-18T19:18:59.715Z" }, - { url = "https://files.pythonhosted.org/packages/4d/f8/f6a5e8185bcb28c2befae3d31f8e3df3b811cb0f47746517a81279fcafe1/lxml-6.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:47402e62c52ff5988c1e8c6c63177f5708bccf48e366dea4e3dcf1e645e04947", size = 5250276, upload-time = "2026-05-18T19:19:03.834Z" }, - { url = "https://files.pythonhosted.org/packages/d2/51/3904907c063451cf8d4a5c9fe0cad95fa1f4ec57f4e3884fa0731bd7a305/lxml-6.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:74a9717fd0d82effef5c2854f0d917231d5324b5a3eb7275c43ac9fa32f97a14", size = 4950022, upload-time = "2026-05-18T19:19:31.958Z" }, - { url = "https://files.pythonhosted.org/packages/94/cd/9c7611a51c37a2830928405817cc5d56a97f64fab83cc3f628748b135749/lxml-6.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:efe0374196335f93b53269acd811b944f2e6bdc88e8894f214bd636455484909", size = 5086695, upload-time = "2026-05-18T19:19:34.764Z" }, - { url = "https://files.pythonhosted.org/packages/da/d6/24e3b5906abb0b674ff2ae195bc3ce59708df2bcd17cf17703b2d7dd643a/lxml-6.1.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac931cdc9442c1763b8a8f6cd62c0c938737eafc5be75eff88df55fc73bc0d00", size = 5031642, upload-time = "2026-05-18T19:19:37.771Z" }, - { url = "https://files.pythonhosted.org/packages/2d/db/6ec54f99019838bff54785c51da07f189eb4676861c5f2730962b0d8d665/lxml-6.1.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aee395f5d0927f947758b4ec119fd5fc8ec71f07a1c5c52077b30b04c0fa6955", size = 5647338, upload-time = "2026-05-18T19:19:40.553Z" }, - { url = "https://files.pythonhosted.org/packages/42/3d/ef4dcfffd22d27a61805d8ed9f7fb888495bc6aa88648fa07c1eaa5586b6/lxml-6.1.1-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9395002973c827b3ed67db77e6ec09f092919a587022174554096a269378fb13", size = 5239528, upload-time = "2026-05-18T19:19:43.657Z" }, - { url = "https://files.pythonhosted.org/packages/62/bb/37fb3f0dff146bdcfa78eec47879273820b2a0bf350ec236ce14bd0b1c26/lxml-6.1.1-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:73bc2086f141224ebddb7fc5c6a36ca58b31b94b561e1dfe8e073e3270fad1e7", size = 5350730, upload-time = "2026-05-18T19:19:46.307Z" }, - { url = "https://files.pythonhosted.org/packages/90/42/43253f168388df4fae1f38c01df36ddb9bee39e2048167b54cdcbae85ea3/lxml-6.1.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3779def59032b81e44a5f70096ef6bf2082f8d901937dca354474ba09782e245", size = 4697530, upload-time = "2026-05-18T19:19:49.889Z" }, - { url = "https://files.pythonhosted.org/packages/eb/a8/c5a8504f81bbdfc8e7094c2c850cdb4ed6777fc4d5ddd9e5ab819f3b0d54/lxml-6.1.1-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:86c89b9d55ebf820ad7c90bc533410f0d098054f293351f10603c0c46ff598f5", size = 5250670, upload-time = "2026-05-18T19:19:53.199Z" }, - { url = "https://files.pythonhosted.org/packages/77/b7/c7e76ab18744d75e21f320ebf9ff9d1ceae2b54dd431ea5a64caf26c9672/lxml-6.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19607c6bbff2a44cf3fe8250abccd20942d3462473e0a721d01d379ed017e462", size = 5084485, upload-time = "2026-05-18T19:19:08.422Z" }, - { url = "https://files.pythonhosted.org/packages/31/31/b35c53f8ef7b7c31cacd23d3638652fff7bcd1deb6eedb709ab43b685908/lxml-6.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c6ed5141a5c7507cf3ee76bd363b0d6f801e3321adc35b5d825a23115faa5465", size = 4737635, upload-time = "2026-05-18T19:19:12.321Z" }, - { url = "https://files.pythonhosted.org/packages/d9/06/31f23c813a7fe8e0cb1b175e915b08c9bf4e86d225b210feadbdbe519667/lxml-6.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:62aeb7e85b5d60320b9d77eef2e773994e2c0ce10121b277e0a19804e1654a5a", size = 5670681, upload-time = "2026-05-18T19:19:15.001Z" }, - { url = "https://files.pythonhosted.org/packages/1a/bc/ce619bccc89b1fd9ad8a8e1330ee3f3beff9f2ff95b712d7bbcdd6e22fc3/lxml-6.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b1b963fd8f5caa68e99dfae060d54de1fe9cba899b8718b44a00cdca53c3e590", size = 5238229, upload-time = "2026-05-18T19:19:18.131Z" }, - { url = "https://files.pythonhosted.org/packages/2f/5d/b329acbbedc0b619ebc2be6cf7ee9ed07e80892c88d4dfd612c33805789a/lxml-6.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:63876be28efefa04a1df615b46770e82042cce445cfdce55160522f57b231ccb", size = 5264191, upload-time = "2026-05-18T19:19:21.118Z" }, - { url = "https://files.pythonhosted.org/packages/16/02/6f7061f4f95f51e545d48e87647c54791d204a4e881be4156e7a26ba5338/lxml-6.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:acd7d70b64c0aae0c7922cca83d288a16f5f6da523637697872253415269baef", size = 4970291, upload-time = "2026-05-18T19:19:56.215Z" }, - { url = "https://files.pythonhosted.org/packages/b0/02/55fc057d8283427dea7d6edb102e7a840239c77a64a983d92f62a304c0e9/lxml-6.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4f0dd2f01f9f8a89f565d000e03abcf0a13d692a346c8d22f628d49af098777a", size = 5102822, upload-time = "2026-05-18T19:19:59.223Z" }, - { url = "https://files.pythonhosted.org/packages/e4/48/8e1cf78d89d66850121d9255a2a24414c98f775da93b90cf976956c24b14/lxml-6.1.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b7e8a14c8634bf6f7a568634cb395305a6d964aeb5b7ee32248094bed3a7e2c", size = 5027923, upload-time = "2026-05-18T19:20:01.549Z" }, - { url = "https://files.pythonhosted.org/packages/ed/00/0632a0647612c8af24d26997b3b961397daa9d5b2581444805933629a4cb/lxml-6.1.1-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:86281fbdd6a8162756f8d603f37e3435bfa38043adb79c6dc6a2dfee065e7525", size = 5595843, upload-time = "2026-05-18T19:20:03.93Z" }, - { url = "https://files.pythonhosted.org/packages/bc/86/ab008a7dc360711b66858d61c80a5979a70a09f2aa2b05d9698df80b803d/lxml-6.1.1-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5d7152ec39ca7c402d8fb9bad86140a15b9503bd0c54484e3f1bbe3dd37ceca", size = 5224515, upload-time = "2026-05-18T19:20:06.381Z" }, - { url = "https://files.pythonhosted.org/packages/75/c6/2702ff375e728e34f56d9a45339a9cf7e4427e917f542225242d63a05afa/lxml-6.1.1-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:88d8cb75b9d82858497a5393e3c63cfbf03035225e4b35a49ed7ccb151e4dc0e", size = 5312511, upload-time = "2026-05-18T19:20:09.308Z" }, - { url = "https://files.pythonhosted.org/packages/b7/57/a5807c98f87a86f10ef9ffab35516df7c0f0c4b6d5d33e9f608ab9c04a31/lxml-6.1.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f64ec5397ea6a41fc1b4af0380d79b44a755b5531dcaccd9940fb260dca93038", size = 4639206, upload-time = "2026-05-18T19:20:11.704Z" }, - { url = "https://files.pythonhosted.org/packages/1f/e1/8a0a2c35734812395f4da4eaf33748a7e5705bfb2a58b128da764339d5ec/lxml-6.1.1-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d34bbf07dbc7ca5970671b1512e928991fb5e9d95365636c9b2d8b4f53af405e", size = 5232404, upload-time = "2026-05-18T19:20:14.064Z" }, - { url = "https://files.pythonhosted.org/packages/c2/e2/0e6a4dd5ad84d01d99aa7bae7cfefd4a760a0e0f8176818241de17d9b6c0/lxml-6.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:17e0e18d4ad8adbd0399291bc44845b69d9dd68439a3cdebdf35ff902ec05072", size = 5083769, upload-time = "2026-05-18T19:19:23.758Z" }, - { url = "https://files.pythonhosted.org/packages/a0/7e/161f33d463f6ffc1c7679104b65086dea120080d49dde4d238f015aaee2f/lxml-6.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:3ab541146f1f6968c462d6c2ac495148e8cdba2f8347700b2141b6ec5a75bf52", size = 4758936, upload-time = "2026-05-18T19:19:27.256Z" }, - { url = "https://files.pythonhosted.org/packages/f1/fb/2369825e3f6ca99305bf9f7b7085fda91c8b0922a89e54d900974aa3ef85/lxml-6.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2a0217714657e023ef4293500f65aa20fce6164c8fd6b08fa5bd4a859fb14b9b", size = 5620296, upload-time = "2026-05-18T19:19:29.993Z" }, - { url = "https://files.pythonhosted.org/packages/30/90/d61e383146f74c5ab683947ea14dc7b82778838ab9b95ea73a23b60d0191/lxml-6.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:05a82eb6e1530a64f26225b55cbd178113bd0b5af1c2b625f25e5296742c26d2", size = 5228598, upload-time = "2026-05-18T19:19:33.523Z" }, - { url = "https://files.pythonhosted.org/packages/76/2d/2dafd8149e94b05bb070690efd5bb2680720681e03ff03fc57d2b70a1105/lxml-6.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9e36f163528fc50cbef305f02a5fd66d404edf7049cdaff211dbc2cba5a7013e", size = 5247845, upload-time = "2026-05-18T19:19:36.649Z" }, -] - -[[package]] -name = "markdown-it-py" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mdurl", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, -] - -[[package]] -name = "markupsafe" -version = "3.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, - { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, - { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, - { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, - { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, - { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, - { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, - { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, - { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, - { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, - { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, - { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, - { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, - { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, - { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, - { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, - { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, - { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, - { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, - { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, - { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, - { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, - { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, - { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, - { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, - { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, - { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, - { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, - { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, - { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, -] - -[[package]] -name = "matplotlib" -version = "3.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "contourpy", marker = "sys_platform == 'linux'" }, - { name = "cycler", marker = "sys_platform == 'linux'" }, - { name = "fonttools", marker = "sys_platform == 'linux'" }, - { name = "kiwisolver", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pillow", marker = "sys_platform == 'linux'" }, - { name = "pyparsing", marker = "sys_platform == 'linux'" }, - { name = "python-dateutil", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1f/24/080c99d223d158d3a8902769269ab6da5b50f7a0e6e072513907e02b7a6c/matplotlib-3.11.0.tar.gz", hash = "sha256:68c0c7be01b30dcca3638934f7f591df73401235cbdbf0d1ab1c71e7db7f8b57", size = 33251176, upload-time = "2026-06-12T02:29:15.508Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/95/7f522393c88313336b20d70fc849555757b2e5febc22b83b3a3f0fd4bce9/matplotlib-3.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be5f93a1d21981bfb802ded0d77a0caa92d4342a47d45754fac77e314a506344", size = 10031353, upload-time = "2026-06-12T02:27:55.215Z" }, - { url = "https://files.pythonhosted.org/packages/87/ce/8f25a0e3186aefd61913e7467d1b999465bcd0d0c03ac695c1b26ca559b7/matplotlib-3.11.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41635d7909d19e52e924a521dde6d8f670b0f53ab1d0e8c331fa831554f681d1", size = 10839232, upload-time = "2026-06-12T02:27:57.746Z" }, - { url = "https://files.pythonhosted.org/packages/85/c2/db15da2bbdf9e3ca66df7db8e2c33a1dfed67be24a24d2c878efaaff01d6/matplotlib-3.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:94f5000f67ca9faa300863ea17f8bce9175cb67b88bec4bc7780502d53dd7c9e", size = 10923899, upload-time = "2026-06-12T02:28:00.223Z" }, - { url = "https://files.pythonhosted.org/packages/a4/c0/1117d53077e3ac3152503a84e9cf7a5c239576805ee71276e80c2aaa7471/matplotlib-3.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be152b7570324dc8d01574cc9474dd2d803237acf528bcbb5b211fa347461a09", size = 10031623, upload-time = "2026-06-12T02:28:11.26Z" }, - { url = "https://files.pythonhosted.org/packages/92/7e/e937138daffad65b71bf831a377809dcbc830fb4f31a31e067dc1faa2575/matplotlib-3.11.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:126f256df600652d7e4b394cf3164ff75210a00038f287c95a012a6f58d0e83f", size = 10839372, upload-time = "2026-06-12T02:28:14.102Z" }, - { url = "https://files.pythonhosted.org/packages/1d/c2/438ecc197ffb8023b6b9922915542f2172f5fd45b76703b0b4fc47322243/matplotlib-3.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:03acfeddf87b0dddb11b081ef7740ad445a3ca8bcb6b8e3011b08f2cf802b75c", size = 10924099, upload-time = "2026-06-12T02:28:16.383Z" }, - { url = "https://files.pythonhosted.org/packages/95/e4/b4b7c33151e74e5c802f3cde1ba807ebfc38401e329b44e215a5888dd76d/matplotlib-3.11.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:565af866fd63e4bd3f987d580afe27c44c2552a3b3305f4ecbb85133601ea6f3", size = 10045491, upload-time = "2026-06-12T02:28:27.141Z" }, - { url = "https://files.pythonhosted.org/packages/71/28/394548efd68354110c1a1be11fe6b6e559e06d1a23da35908a0e316c55a9/matplotlib-3.11.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b3e64dea5062c570f04358e2711859f3531b459f29516274fbad889079e4f3", size = 10857059, upload-time = "2026-06-12T02:28:29.222Z" }, - { url = "https://files.pythonhosted.org/packages/c8/44/e7922e6e2a4d63bdfbc9dc4a53e3850ab438d46cf42e6779bb15ec92c948/matplotlib-3.11.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:942b37c5db1899610bd1543ce8e13e4ecff9a4633e7f63bb6aa9205d2644ebd1", size = 10939576, upload-time = "2026-06-12T02:28:31.66Z" }, - { url = "https://files.pythonhosted.org/packages/b7/2d/4e1240ea82ee197dfb3851e71f71c87eeeb975f1753b56a0588e4e80739a/matplotlib-3.11.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9695457a467ff86d23f35037a43deb6f1134dd6d3e2ac8ce1e2087cff09ffb9", size = 10843005, upload-time = "2026-06-12T02:28:42.39Z" }, - { url = "https://files.pythonhosted.org/packages/29/dc/6377ecfaa5fef79430f74a1a16638b4e2aa30d4692bae2c19f9d76fe3b01/matplotlib-3.11.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19c16c61dea63b3582918503e6b294193961261d9daa806d4ae2151f1ad05430", size = 11127459, upload-time = "2026-06-12T02:28:44.483Z" }, - { url = "https://files.pythonhosted.org/packages/6f/41/795c405aa7560443a3b01309424cde4a1113b85c90b8a63417444a749617/matplotlib-3.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2d72ea8b7924f3cb955e61518d21e43b3df1e6c8a793b480a0c1214f185d30ba", size = 10925160, upload-time = "2026-06-12T02:28:46.564Z" }, - { url = "https://files.pythonhosted.org/packages/4b/cb/22f6bc35711a0b5639a784e74e653e77c86210bd4304449dd399a482f74e/matplotlib-3.11.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfabef0230d0697aa0d717385194dd41162e00207a68bf4abf94c2bf4c27dca0", size = 10854181, upload-time = "2026-06-12T02:28:57.856Z" }, - { url = "https://files.pythonhosted.org/packages/3f/7e/9a9eaca731a2939589da520f0ebe8fd8753d0f51fca98c7d20af6dbe261a/matplotlib-3.11.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1644db30e759199443493ac5e5caec24fdb775a8f6123021f85ba47c4133c3cb", size = 11137715, upload-time = "2026-06-12T02:29:00.555Z" }, - { url = "https://files.pythonhosted.org/packages/ef/f9/9b030b6088354acb0296871bb624b25befc1c42509d3c6cd17420c83a5b8/matplotlib-3.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:15b0d160079cb10699a0e98b5989c70677b2df7cacdc62af67c30f2facec46d9", size = 10939427, upload-time = "2026-06-12T02:29:02.527Z" }, -] - -[[package]] -name = "matplotlib-inline" -version = "0.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "traitlets", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bd/c0/9f7c9a46090390368a4d7bcb76bb87a4a36c421e4c0792cdb53486ffac7a/matplotlib_inline-0.2.2.tar.gz", hash = "sha256:72f3fe8fce36b70d4a5b612f899090cd0401deddc4ea90e1572b9f4bfb058c79", size = 8150, upload-time = "2026-05-08T17:33:33.49Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" }, -] - -[[package]] -name = "mdit-py-plugins" -version = "0.6.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/59/fc/f8d0863f8862f25602c0404d75568e89fb6b4109804645e5cdfb1be5cf56/mdit_py_plugins-0.6.1.tar.gz", hash = "sha256:a2bca0f039f39dbd35fb74ae1b5f998608c437463371f0ff7f49a19a17a114d0", size = 56114, upload-time = "2026-05-13T09:03:38.91Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/69/6da5581c6a7fede7dc261bf4e67d6adca4196f176b43288b55b3db395b6e/mdit_py_plugins-0.6.1-py3-none-any.whl", hash = "sha256:214c82fb2ac524472ab6a5bcab1de80f73b50443e187f401bfd77efbc7c6481d", size = 66663, upload-time = "2026-05-13T09:03:37.76Z" }, -] - -[[package]] -name = "mdurl" -version = "0.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, -] - -[[package]] -name = "mistune" -version = "3.3.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7b/a5/2dab368d6950e6808904dec98f54c7e726ee7be4a0c6afe00e6e011bd52d/mistune-3.3.3.tar.gz", hash = "sha256:c4c6c0c840b8637a2e9b8b6d607eb7c8f00888bf14c754409bcd339e848c2477", size = 115363, upload-time = "2026-07-09T06:18:05.268Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/89/70/b1e4737b84163db5bb1dfde6f216dbfbf32783330a9989c965e121172830/mistune-3.3.3-py3-none-any.whl", hash = "sha256:99de1585e42dcbd826faa9e11a202727a5e202e4e4722a4c69ac1ff615793dd7", size = 63569, upload-time = "2026-07-09T06:18:03.839Z" }, -] - -[[package]] -name = "more-itertools" -version = "11.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/de/1d/f4da6f02cdffe04d6362210b807146a26044c88d839208aec273bb0d9184/more_itertools-11.1.0.tar.gz", hash = "sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d", size = 145772, upload-time = "2026-05-22T14:14:29.909Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192", size = 72226, upload-time = "2026-05-22T14:14:28.824Z" }, -] - -[[package]] -name = "mpi4py" -version = "4.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/75/83/231445bbcf7ef10864746c244ff2d82000011449b79275642c5d4ed8c8f4/mpi4py-4.1.2.tar.gz", hash = "sha256:56860286dc45f20e8821e93cb06669e30462348bf866f685553fa4b712d58d02", size = 501709, upload-time = "2026-05-16T10:35:23.618Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/0a/1da7f403e0d8ce0e26d541f7538302cec00cf5b0a98a7a52b929f938a25c/mpi4py-4.1.2-cp310-abi3-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2ef63b2e3083e6062fd90e4de8c4e3acbf81e0772406e0226eb8dde6a48cab8e", size = 1327130, upload-time = "2026-05-16T10:34:00.269Z" }, - { url = "https://files.pythonhosted.org/packages/e6/f9/65999152ae82bad914c6a083821ee774afefd6d0544e633b940c9a9ebf3f/mpi4py-4.1.2-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6508e654b9c8ff9f611b19548b2a17d1e323b520a15168189f92221e6757b8ff", size = 1182268, upload-time = "2026-05-16T10:34:02.206Z" }, - { url = "https://files.pythonhosted.org/packages/9f/2b/1e48c4c5f9acbdca8dd28beeba9123dde140cd2ca520f8e3a3cf22faeeaa/mpi4py-4.1.2-cp312-cp312-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:00f4cce8999d19f35243c3442ea22debbe3336f69c309cd5d3176df4e51c717a", size = 1358844, upload-time = "2026-05-16T10:34:28.247Z" }, - { url = "https://files.pythonhosted.org/packages/94/46/a37225d47997fcf30adca25d3849d035bbb61d972118b024db900306e528/mpi4py-4.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0a58e164776acb7b52414548b1bf0e5caafce0ee90345deff147873a64b6b2cc", size = 1227206, upload-time = "2026-05-16T10:34:29.866Z" }, - { url = "https://files.pythonhosted.org/packages/0d/5e/493415cdb0da0c1c0b6ec9a1fb65ab57a174e8111b25c87f7507f663d0b4/mpi4py-4.1.2-cp313-cp313-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:085e0cb05427398fe2281856f27a87984b9f234cd6d98a2a384a6fbfe679a56f", size = 1358571, upload-time = "2026-05-16T10:34:37.478Z" }, - { url = "https://files.pythonhosted.org/packages/6f/51/3822e834fc9cafd96811501b85232e02bd9b31596d42965536a269a6c112/mpi4py-4.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:429437883b4511583d56a57bc58ad0d02965fb6c19f84f39e30c0646e6c0cab9", size = 1226562, upload-time = "2026-05-16T10:34:39.652Z" }, - { url = "https://files.pythonhosted.org/packages/ec/5e/d358fadb8672d58abd6dce16c95eda56f497b378dec5642a31cd6ededfc4/mpi4py-4.1.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:85e332feaac3323d8ed1c71f17478fe3f529468f28f6f9bb4c9133ad2d3a3a6d", size = 1419007, upload-time = "2026-05-16T10:34:45.47Z" }, - { url = "https://files.pythonhosted.org/packages/27/eb/5cd53880337009cab9a9d17a007ad5aec731f3a55e211bdcc99dbb98a0e3/mpi4py-4.1.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e05dba75f6ad17ae761bdc01ee5c5271d15cd296e8a508e22c99c91b540dc99", size = 1298482, upload-time = "2026-05-16T10:34:46.858Z" }, - { url = "https://files.pythonhosted.org/packages/26/e4/32575e9b0d5380b08ffa04958c7a13b1bab86e3674f33b6de3827fbbe14c/mpi4py-4.1.2-cp314-cp314-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d53e8f7182bf125b37155f679b301a21cd64e5ae6861ca0ab989c9d2b2073bbd", size = 1368388, upload-time = "2026-05-16T10:34:53.665Z" }, - { url = "https://files.pythonhosted.org/packages/87/a8/ad5e925da9de402704ecfe8715348b068fb0b65059109c65dafb111f95e4/mpi4py-4.1.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cbf982d3425fd06890fbcff79ba19a7435e93bc74e5711ca8ac0b64d0c3ece9", size = 1238211, upload-time = "2026-05-16T10:34:55.094Z" }, - { url = "https://files.pythonhosted.org/packages/39/29/45a41d081342896af52d38a848017728bf6834c4c92814c50aa791dc750b/mpi4py-4.1.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:db189238c37be98933eeb078a3e2b827eef969837e8fe0f3a2e6cdb7f4b1b05f", size = 1423092, upload-time = "2026-05-16T10:35:01.672Z" }, - { url = "https://files.pythonhosted.org/packages/0f/6f/c2127d426d87f2f7cdb4beacbb4febe7b0c93047a3c615d0e56cecb42563/mpi4py-4.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d0fbf0bc100ed966c3d7f44231a19aef0025d523d1b813de468a1b114d0d9436", size = 1303134, upload-time = "2026-05-16T10:35:03.122Z" }, -] - -[[package]] -name = "mpmath" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, -] - -[[package]] -name = "msgpack" -version = "1.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/f9/c0a1c127f9049db9155afc316952ea571720dd01833ff5e4d7e8e6352dbb/msgpack-1.2.1.tar.gz", hash = "sha256:04c721c2c7448767e9e3f2520a475663d8ee0f09c31890f6d2bd70fd636a9647", size = 183960, upload-time = "2026-06-18T16:13:52.594Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/aa/753ad8b007b464e1d8aa0c8e650b9c5f4f725e658fc5ac8a7635c55b7f6e/msgpack-1.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60926b75d00c8e816ef98f3034f484a8bc64242d66839cef4cf7e503142316a0", size = 410634, upload-time = "2026-06-18T16:13:00.383Z" }, - { url = "https://files.pythonhosted.org/packages/6a/fd/6adabd4f6d5e686f97dd02ce7fce3fe4cf672cbac36b8f67ff4040e8ad8b/msgpack-1.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:020e881a764b20d8d7ca1a54fc01b8175519d108e3c3f194fddc200bda95951a", size = 419989, upload-time = "2026-06-18T16:13:01.776Z" }, - { url = "https://files.pythonhosted.org/packages/5a/cc/85039b7b0eb168aaad7383a23c97e291a11f08351cb45a606ce865e4e3f1/msgpack-1.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4202c74688ca06591f78cb18988228bd4cca2cc75d57b60008372892d2f1e6e6", size = 377544, upload-time = "2026-06-18T16:13:03.637Z" }, - { url = "https://files.pythonhosted.org/packages/ed/bf/35963899493b32030c85fc513b723ae66144ac70c11ebc52e889e16e3d99/msgpack-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8b267ce94efb76fbd1b3373511420074ee3187f0f7811bf394531de13294735a", size = 400842, upload-time = "2026-06-18T16:13:05.012Z" }, - { url = "https://files.pythonhosted.org/packages/a6/df/8e2ac970c8f99264cd9997d1c73df5466bc19da3301d7dc5500862a9b089/msgpack-1.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f1d0f8f98ade9634e01fb704a408f9336c0a8f1117b369f5db83dc7551d8b1", size = 374108, upload-time = "2026-06-18T16:13:06.232Z" }, - { url = "https://files.pythonhosted.org/packages/17/dd/fa8bd265110dfa51c20cb529f9e6d240a16fafe7e645004c6af2d01353ba/msgpack-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f02cf17a6ca1abe29b5f980644f7551f94d71f2011509b26d8625ce038f0df64", size = 414939, upload-time = "2026-06-18T16:13:07.478Z" }, - { url = "https://files.pythonhosted.org/packages/e3/10/ddf7b06db879e8792d13934ddda09ff20bd2a583fd84c9b59aae9b0e650b/msgpack-1.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f310233ef7fb9c14e201c93639fe5f5260b005f56f0b29048e999c30935596cc", size = 407518, upload-time = "2026-06-18T16:13:14.233Z" }, - { url = "https://files.pythonhosted.org/packages/79/d3/36a46a8ed992b781acbc05928bd5bee3c810cb0c3563bf81a7b0c04a1a76/msgpack-1.2.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:787c9bebb5833e8f6fc8abca3c0597683d8d87f56a8842b6b89c75a5f3176e2d", size = 416405, upload-time = "2026-06-18T16:13:15.435Z" }, - { url = "https://files.pythonhosted.org/packages/f9/84/e8e9598b557c0ba6ddae901a73780a4c75ac667dddf59414b1e56a42fb34/msgpack-1.2.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dc871b997a9370d855b7394465f2f350e847a5b806dd38dcc9c989e7d87da155", size = 376257, upload-time = "2026-06-18T16:13:17.022Z" }, - { url = "https://files.pythonhosted.org/packages/40/16/738fe6d875ad7e2a9429c165322a4ec088f4f273cdfae63d96a89c467961/msgpack-1.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:85f57e960d877f2977f6430896191b04a21f8901b3b4baf2e4604329f4db5402", size = 397469, upload-time = "2026-06-18T16:13:18.287Z" }, - { url = "https://files.pythonhosted.org/packages/ca/be/6d5952df75a7f24f35833af764c3a6860780364cb3a0030beb8099e1b2b4/msgpack-1.2.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1233ee2dd0cefba127583de50ea654677277047d238303521db35def3d7b2e7c", size = 372802, upload-time = "2026-06-18T16:13:19.685Z" }, - { url = "https://files.pythonhosted.org/packages/e1/39/e2ef7dbf0473bcb8dc7c50bf782a892d67414877b63e47fc88eb189ef5e6/msgpack-1.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e3dc2feb0876209d9c38aa56cb1de169bd6c4348f1aa48271f241226590993e6", size = 411273, upload-time = "2026-06-18T16:13:21.028Z" }, - { url = "https://files.pythonhosted.org/packages/b8/a6/99e58722feaffc5f2fbcc0c8c0d1451ab9f84097f7af87291b46af2390f4/msgpack-1.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca0dacff965c47afdc3749a8469d7302a8f801d6a28758d55120d75e66ce6889", size = 406090, upload-time = "2026-06-18T16:13:28.072Z" }, - { url = "https://files.pythonhosted.org/packages/19/03/8c63e8cf52958534ef688625965ab04c269a6cadd8caef16758b380a821a/msgpack-1.2.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e2bf9280bceb5efca998435904b5d3e9fdbcc11d90dc9df30aec7973252b720", size = 412106, upload-time = "2026-06-18T16:13:29.427Z" }, - { url = "https://files.pythonhosted.org/packages/63/d2/155d9e71b40e41fd934bc0c48b9b2770f22263e1ac20aad8e29fdca7be3f/msgpack-1.2.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6c4be5d1c02a42b066ca6ddb71adf36432868fdcdb6ee87e634e86e0674190", size = 374851, upload-time = "2026-06-18T16:13:30.631Z" }, - { url = "https://files.pythonhosted.org/packages/98/48/deaf2326262a8d5ea3295ce9649912ecd3f551ba7ec8e33c665d2ba583f3/msgpack-1.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec0e675d59150a6269ddc9139087c722292664a37d071a849c05c473350f1f2d", size = 396168, upload-time = "2026-06-18T16:13:31.977Z" }, - { url = "https://files.pythonhosted.org/packages/10/2a/b4410f906c2ec0008f1608d3ab5143afc3ad3f4e6da0fed3ea2231d0bef4/msgpack-1.2.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:dd3bfe82d53edfe4b7fc9a7ec9761e23a7a5b1dac22264505af428253c29ed24", size = 371959, upload-time = "2026-06-18T16:13:33.282Z" }, - { url = "https://files.pythonhosted.org/packages/59/86/1edc67270099a528fa2093ea60fe191233cd238e4bd30cfacf7db79fc959/msgpack-1.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5ad5467fc3f68b5468e06c5f788d712e9f8ffc8b0cd1bcb160c105c1ee92dae7", size = 408457, upload-time = "2026-06-18T16:13:34.567Z" }, - { url = "https://files.pythonhosted.org/packages/a9/b4/b774d7eb95561739907fec675582f83203cf41c597a418c2589b4bfb8e9d/msgpack-1.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20466cca18c49c7292a8984bc15d65857b171e7264bdcb5f96baf8be238791fc", size = 427661, upload-time = "2026-06-18T16:13:41.574Z" }, - { url = "https://files.pythonhosted.org/packages/b2/f9/3243191dc9937e00756c8bc1b0272fed8f23758e43df2a3b46f533e5090f/msgpack-1.2.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:196300e7e5d6e74d50f1607ab9c06c4a1484c383cd22defd727902591f7e8dde", size = 426375, upload-time = "2026-06-18T16:13:42.936Z" }, - { url = "https://files.pythonhosted.org/packages/23/c7/1693111db9944ba4ad4b67a1e788400d78a0b6af7a6523dc7e4e58f8274b/msgpack-1.2.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:575957e79cd51903a4e8495a242442949641e08f1efd5197b43bebd3ea7682b4", size = 380495, upload-time = "2026-06-18T16:13:44.306Z" }, - { url = "https://files.pythonhosted.org/packages/3e/2b/92f86956a0c13e8662f7e2ad630c4eb4db07497b967589bd5245e018b2c1/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8c2ed1e48cc0f460bf3c7780e7137ff21a4e18433451916f2442c1b21036cd7d", size = 410897, upload-time = "2026-06-18T16:13:45.629Z" }, - { url = "https://files.pythonhosted.org/packages/da/ea/1479f72d200313a76fc2f823a79d1e07ed052ab7b8a0280640aa7b95de42/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5f6277e5f783c36786a145e0247fc189a03f35f84b251646e53592d2bc12b355", size = 378519, upload-time = "2026-06-18T16:13:46.998Z" }, - { url = "https://files.pythonhosted.org/packages/f5/4d/fa006060ffa1011d32bfae826fe766fe73e02982183601633b7121058ab3/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f9389552ecf4784886345ead0647e4edc96bee37cbab05b75540f542f766c48c", size = 419815, upload-time = "2026-06-18T16:13:48.205Z" }, -] - -[[package]] -name = "myst-parser" -version = "5.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, - { name = "mdit-py-plugins", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/21/dc/603751677fff302f34396e206b610f556a59d7fe58b9a2145f54e96b48e8/myst_parser-5.1.0.tar.gz", hash = "sha256:ab69322dc6719dcc7f296479dbb70181b66df6ed315064f92dbc85c0e1bf2f02", size = 101182, upload-time = "2026-05-13T09:38:19.361Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/09/dc/f3dfb7488b770f3f67e6545085bf2abea5172e88f57b8ad25ef860ca704c/myst_parser-5.1.0-py3-none-any.whl", hash = "sha256:9c91c52b3cdb4d94a6506e4fab4e2f296c7623a0da0dcbe6de1565c3dad67a8a", size = 85817, upload-time = "2026-05-13T09:38:17.904Z" }, -] - -[[package]] -name = "nbclient" -version = "0.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jupyter-client", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/28/a5/b3bae4b590c0cbcada2c63a34f7580024e834a8ba213e949a2f906705787/nbclient-0.11.0.tar.gz", hash = "sha256:04a134a5b087f2c5887f228aca155db50169b8cd9334dee6942c8e927e56081a", size = 62535, upload-time = "2026-06-05T07:52:41.746Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/36/c9/94d73e5a01c5b926c3fa2496e97d7a8dc28ed5a77c0b2ed712f1a62e6694/nbclient-0.11.0-py3-none-any.whl", hash = "sha256:ef7fa0d59d6e1d41103933d8a445a18d5de860ca6b613b87b8574accdb3c2895", size = 25288, upload-time = "2026-06-05T07:52:40.115Z" }, -] - -[[package]] -name = "nbconvert" -version = "7.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "beautifulsoup4", marker = "sys_platform == 'linux'" }, - { name = "bleach", extra = ["css"], marker = "sys_platform == 'linux'" }, - { name = "defusedxml", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab-pygments", marker = "sys_platform == 'linux'" }, - { name = "markupsafe", marker = "sys_platform == 'linux'" }, - { name = "mistune", marker = "sys_platform == 'linux'" }, - { name = "nbclient", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pandocfilters", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/01/b1/708e53fe2e429c103c6e6e159106bcf0357ac41aa4c28772bd8402339051/nbconvert-7.17.1.tar.gz", hash = "sha256:34d0d0a7e73ce3cbab6c5aae8f4f468797280b01fd8bd2ca746da8569eddd7d2", size = 865311, upload-time = "2026-04-08T00:44:14.914Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl", hash = "sha256:aa85c087b435e7bf1ffd03319f658e285f2b89eccab33bc1ba7025495ab3e7c8", size = 261927, upload-time = "2026-04-08T00:44:12.845Z" }, -] - -[[package]] -name = "nbformat" -version = "5.10.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "fastjsonschema", marker = "sys_platform == 'linux'" }, - { name = "jsonschema", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/fd/91545e604bc3dad7dca9ed03284086039b294c6b3d75c0d2fa45f9e9caf3/nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a", size = 142749, upload-time = "2024-04-04T11:20:37.371Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/82/0340caa499416c78e5d8f5f05947ae4bc3cba53c9f038ab6e9ed964e22f1/nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b", size = 78454, upload-time = "2024-04-04T11:20:34.895Z" }, -] - -[[package]] -name = "ndindex" -version = "1.10.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f5/92/4b9d2f4e0f3eabcfc7b02b48261f6e5ad36a3e2c1bbdcc4e3b7b6c768fa6/ndindex-1.10.1.tar.gz", hash = "sha256:0f6113c1f031248f8818cbee1aa92aa3c9472b7701debcce9fddebcd2f610f11", size = 271395, upload-time = "2025-11-19T20:40:08.899Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/40/139b6b050ba2b2a0bb40e0381a352b1eb6551302dcb8f86fb4c97dd34e92/ndindex-1.10.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:046c1e88d46b2bd2fd3483e06d27b4e85132b55bc693f2fca2db0bb56eea1e78", size = 542901, upload-time = "2025-11-19T20:38:44.43Z" }, - { url = "https://files.pythonhosted.org/packages/27/ae/defd665dbbeb2fffa077491365ed160acaec49274ce8d4b979f55db71f18/ndindex-1.10.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03cf1e6cdac876bd8fc92d3b65bb223496b1581d10eab3ba113f7c195121a959", size = 546875, upload-time = "2025-11-19T20:38:45.938Z" }, - { url = "https://files.pythonhosted.org/packages/59/43/6d54d48e8eaee25cdab70d3e4c4f579ddb0255e4f1660040d5ad55e029c6/ndindex-1.10.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:752e78a5e87911ded117c57a7246596f26c9c6da066de3c2b533b3db694949bb", size = 1510036, upload-time = "2025-11-19T20:38:47.444Z" }, - { url = "https://files.pythonhosted.org/packages/09/61/e28ba3b98eacd18193176526526b34d7d70d2a6f9fd2b4d8309ab5692678/ndindex-1.10.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c9dd58d91220b1c1fe516324bfcf4114566c98e84b1cbbe416abe345c75bd557", size = 1571849, upload-time = "2025-11-19T20:38:48.951Z" }, - { url = "https://files.pythonhosted.org/packages/77/34/a51b7c6f7159718a6a0a694fc1058b94d793c416d9a4fd649f1924cce5f8/ndindex-1.10.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6928e47fb008903f2e41309b7ff1e59b16abbcd59e2e945454571c28b2433c9e", size = 524127, upload-time = "2025-11-19T20:38:59.412Z" }, - { url = "https://files.pythonhosted.org/packages/21/91/d8f19f0b8fc9c5585b50fda44c05415da0bdc5fa9c9c69011015dac27880/ndindex-1.10.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e69a2cb1ac7be955c3c77f1def83f410775a81525c9ce2d4c0a3f2a61589ed47", size = 528213, upload-time = "2025-11-19T20:39:00.882Z" }, - { url = "https://files.pythonhosted.org/packages/2c/a9/77d9d037e871a3faa8579b354ca2dd09cc5bbf3e085d9e3c67f786d55ee3/ndindex-1.10.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cb76e0f3f235d8b1c768b17e771de48775d281713795c3aa045e8114ad61bdda", size = 1492172, upload-time = "2025-11-19T20:39:02.387Z" }, - { url = "https://files.pythonhosted.org/packages/ac/29/ad13676fc9312e0aa1a80a7c04bcb0b502b877ed4956136117ad663eced0/ndindex-1.10.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7da34a78410c14341d5fff73be5ce924bd36500bf7f640fc59b8607d3a0df95e", size = 1552614, upload-time = "2025-11-19T20:39:04.232Z" }, - { url = "https://files.pythonhosted.org/packages/e2/d7/1df02df24880ce3f3c8137b6f3ca5a901a58d9079dcfd8c818419277ff87/ndindex-1.10.1-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d0ebdba2fade3f6916fe21fd49e2a0935af4f58c56100a60f3f2eb26e20baee7", size = 632517, upload-time = "2025-11-19T20:39:13.259Z" }, - { url = "https://files.pythonhosted.org/packages/34/96/b509c2b14e9b10710fe6ab6ba8bda1ee6ce36ab16397ff2f5bbb33bbbba3/ndindex-1.10.1-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:346a4bf09f5771548665c8206e81daadb6b9925d409746e709894bdd98adc701", size = 616179, upload-time = "2025-11-19T20:39:14.757Z" }, - { url = "https://files.pythonhosted.org/packages/38/e3/f89d60cf351c33a484bf1a4546a5dee6f4e7a6a973613ffa12bd316b14ad/ndindex-1.10.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:23d35696f802548143b5cc199bf2f171efb0061aa7934959251dd3bae56d038c", size = 1588373, upload-time = "2025-11-19T20:39:16.62Z" }, - { url = "https://files.pythonhosted.org/packages/ee/19/002fc1e6a4abeef8d92e9aa2e43aea4d462f6b170090f7752ea8887f4897/ndindex-1.10.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a91e1a0398120233d5c3b23ccb2d4b78e970d66136f1a7221fa9a53873c3d5c5", size = 1636436, upload-time = "2025-11-19T20:39:18.266Z" }, - { url = "https://files.pythonhosted.org/packages/09/4a/028ab78a9f29fd2a7e86a90337cde4658eaa77b425c63045d83a1d2e4f26/ndindex-1.10.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf80241b40adffdc3276b2c9fb63a96c6c98b4a9d941892738de8add65083962", size = 528125, upload-time = "2025-11-19T20:39:26.798Z" }, - { url = "https://files.pythonhosted.org/packages/00/a9/bd823b345fb06c83ade6ef1c1933521d4357cd04490e684d4fa30126926c/ndindex-1.10.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf5855881884b8467dfcf45764ccf2e4279075be14b155b89c96994bb08d2e6f", size = 527328, upload-time = "2025-11-19T20:39:28.292Z" }, - { url = "https://files.pythonhosted.org/packages/91/4f/40b9c15588cbf9dde43c4fb88a31dd1f636a913fa29649f18f8e3ebca36a/ndindex-1.10.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e81a9bd36fe054b6c9fcc53d26bc9a28cf15d1ab52a0f5b854f894116f3a54e1", size = 1497508, upload-time = "2025-11-19T20:39:30.735Z" }, - { url = "https://files.pythonhosted.org/packages/24/8f/b8048f7837d2e9dff0af507b398307fa84a2aa9ea3db71b4aa800b21da4a/ndindex-1.10.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:588e8875d836a93b3cd9af482c8074bb02288ae1aff92cf277e1f02d9ae0f992", size = 1552625, upload-time = "2025-11-19T20:39:32.404Z" }, - { url = "https://files.pythonhosted.org/packages/4d/05/a6fda696a2f02a3f8dd2ee9d816cb2edff6423bf0110a4876cc3b1259732/ndindex-1.10.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c65ca639a7abf72d79f22424f4abd18dece1f289a2b7b028a0ca455edd2168d4", size = 630898, upload-time = "2025-11-19T20:39:41.495Z" }, - { url = "https://files.pythonhosted.org/packages/73/78/eb2e5d067d4c054451e33eaece74cbdcb58236dc60516e73d783dae34c7e/ndindex-1.10.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c3634a8df43e7928122225a3d64d850c8957bd1edf2e403907deacb478af27b", size = 614419, upload-time = "2025-11-19T20:39:43.254Z" }, - { url = "https://files.pythonhosted.org/packages/78/51/261bfb49eb7920c2a7314cacba5821930a529911dce48c7c6cd786096a5a/ndindex-1.10.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9d581f931e61f182478f18bdf5edd3955899df5da4892ed0d5de547a4cfd5b6f", size = 1587517, upload-time = "2025-11-19T20:39:44.809Z" }, - { url = "https://files.pythonhosted.org/packages/ec/37/084a332ecdf8b0049151bd78001a7baf2daf7f500d043beb8a1f95d0f4e3/ndindex-1.10.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:78ce45106ebf67aeba99714818c721d8fd5fb9534daebd2565665a2d64b50fc9", size = 1635372, upload-time = "2025-11-19T20:39:47.231Z" }, -] - -[[package]] -name = "nest-asyncio2" -version = "1.7.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b4/73/731debf26e27e0a0323d7bda270dc2f634b398e38f040a09da1f4351d0aa/nest_asyncio2-1.7.2.tar.gz", hash = "sha256:1921d70b92cc4612c374928d081552efb59b83d91b2b789d935c665fa01729a8", size = 14743, upload-time = "2026-02-13T00:34:04.386Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/3c/3179b85b0e1c3659f0369940200cd6d0fa900e6cefcc7ea0bc6dd0e29ffb/nest_asyncio2-1.7.2-py3-none-any.whl", hash = "sha256:f5dfa702f3f81f6a03857e9a19e2ba578c0946a4ad417b4c50a24d7ba641fe01", size = 7843, upload-time = "2026-02-13T00:34:02.691Z" }, -] - -[[package]] -name = "notebook" -version = "5.7.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ipykernel", marker = "sys_platform == 'linux'" }, - { name = "ipython-genutils", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "jupyter-client", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "nbconvert", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "prometheus-client", marker = "sys_platform == 'linux'" }, - { name = "pyzmq", marker = "sys_platform == 'linux'" }, - { name = "send2trash", marker = "sys_platform == 'linux'" }, - { name = "terminado", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6e/22/b5dcce67559d63d0f22e46d806305710808c698a1b91c07eb09e389785e0/notebook-5.7.4.tar.gz", hash = "sha256:d908673a4010787625c8952e91a22adf737db031f2aa0793ad92f6558918a74a", size = 13389469, upload-time = "2018-12-17T10:06:00.805Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/a6/dbdf0954d073ab2cf5421cad20334d1e4c780da37205c04522c045298f67/notebook-5.7.4-py2.py3-none-any.whl", hash = "sha256:3ab2db8bc10e6edbd264c3c4b800bee276c99818386ee0c146d98d7e6bcf0a67", size = 8980706, upload-time = "2018-12-17T10:05:52.01Z" }, -] - -[[package]] -name = "notebook-shim" -version = "0.2.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jupyter-server", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/54/d2/92fa3243712b9a3e8bafaf60aac366da1cada3639ca767ff4b5b3654ec28/notebook_shim-0.2.4.tar.gz", hash = "sha256:b4b2cfa1b65d98307ca24361f5b30fe785b53c3fd07b7a47e89acb5e6ac638cb", size = 13167, upload-time = "2024-02-14T23:35:18.353Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/33/bd5b9137445ea4b680023eb0469b2bb969d61303dedb2aac6560ff3d14a1/notebook_shim-0.2.4-py3-none-any.whl", hash = "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef", size = 13307, upload-time = "2024-02-14T23:35:16.286Z" }, -] - -[[package]] -name = "numba" -version = "0.66.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llvmlite", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ae/a0/570e3dc53e5602b49108f62a13e529f1eec8bfc7ef37d49c825924dcf546/numba-0.66.0.tar.gz", hash = "sha256:b900e63a0e26c05ea9a6d5a3a5a0a177cb64c5011887bf43edb8c3ed2c38d363", size = 2806181, upload-time = "2026-07-01T23:12:46.36Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/55/25c319845e9a4e08f16611ddbda56a192eb7b6ed13e1a2bff2da272ffb97/numba-0.66.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0999e3ee1b18c48e1fb51d11af35ef59852c7f4f50569c9550c25faef0616ad1", size = 3866252, upload-time = "2026-07-01T23:12:20.429Z" }, - { url = "https://files.pythonhosted.org/packages/71/ef/a82d6fd6bf1b0fe461651e924d3647eeec9ac17f8eee4896264bf7480930/numba-0.66.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efe0d2d5099790df945e0cb6e1b3104bd965d7bbfac50d62f1d5d1d6ade0825d", size = 3566974, upload-time = "2026-07-01T23:12:22.116Z" }, - { url = "https://files.pythonhosted.org/packages/44/b5/e930010965568fe7f2c6c962fd2849d458cb9f62c3ab7584af8a19a2b40a/numba-0.66.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:939316d5d8619751207b8972a67852b5a7646665298cb4de693cd6bf135152f4", size = 3873663, upload-time = "2026-07-01T23:12:27.308Z" }, - { url = "https://files.pythonhosted.org/packages/d0/ec/5b51457cbe96e4831141d83e892e65191b23a1b78728456c62909d231ace/numba-0.66.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cdf506775d9f02eb92a87bf5c5b1e0d25506fd18cafd769f4ed914a8feac73e7", size = 3573529, upload-time = "2026-07-01T23:12:28.944Z" }, - { url = "https://files.pythonhosted.org/packages/0f/26/885774c006de6620ed3d10f45d8e20fe0b8e6aad6d573211a2cbc8b3e528/numba-0.66.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e2b101f23b8b63978d334574d2039f27f0dccfe1d891756f33a2e2f3e4c88cf4", size = 3842720, upload-time = "2026-07-01T23:12:33.938Z" }, - { url = "https://files.pythonhosted.org/packages/93/99/edebf7de890b73973d839dd971cf73734adfb81ffa1b4504f84b9059c3e5/numba-0.66.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63b943eb2c9ba371908ce2cd6dfc643db51fc40f7966993376a1701bc922f537", size = 3543537, upload-time = "2026-07-01T23:12:35.566Z" }, - { url = "https://files.pythonhosted.org/packages/39/fd/e9c9680a3813f3d781c20e5d53c1074801b787d4feecca0472fdd7c05ce1/numba-0.66.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:651a2b53298340956db26ecbe7ab043106b50a40c2807e66d617a4917245a4ab", size = 3878695, upload-time = "2026-07-01T23:12:40.302Z" }, - { url = "https://files.pythonhosted.org/packages/61/3a/9b363287b85fcd4537ea3878793822878b2ac1008a78159d2096fea628de/numba-0.66.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8c1144ba1720ea59ad79f4f488ed54d149b2613b357f7e445678b7d0739c70e9", size = 3596323, upload-time = "2026-07-01T23:12:42.805Z" }, -] - -[[package]] -name = "numbaquadpack" -version = "0.1" -source = { git = "https://github.com/Nicholaswogan/NumbaQuadpack.git#9fa67a4a952860042177a3dba61e7102799671b3" } - -[[package]] -name = "numcodecs" -version = "0.16.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/44/bd/8a391e7c356366224734efd24da929cc4796fff468bfb179fe1af6548535/numcodecs-0.16.5.tar.gz", hash = "sha256:0d0fb60852f84c0bd9543cc4d2ab9eefd37fc8efcc410acd4777e62a1d300318", size = 6276387, upload-time = "2025-11-21T02:49:48.986Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/1e/98aaddf272552d9fef1f0296a9939d1487914a239e98678f6b20f8b0a5c8/numcodecs-0.16.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b554ab9ecf69de7ca2b6b5e8bc696bd9747559cb4dd5127bd08d7a28bec59c3a", size = 8534814, upload-time = "2025-11-21T02:49:28.547Z" }, - { url = "https://files.pythonhosted.org/packages/fb/53/78c98ef5c8b2b784453487f3e4d6c017b20747c58b470393e230c78d18e8/numcodecs-0.16.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad1a379a45bd3491deab8ae6548313946744f868c21d5340116977ea3be5b1d6", size = 9173471, upload-time = "2025-11-21T02:49:30.444Z" }, - { url = "https://files.pythonhosted.org/packages/0b/00/787ea5f237b8ea7bc67140c99155f9c00b5baf11c49afc5f3bfefa298f95/numcodecs-0.16.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:015a7c859ecc2a06e2a548f64008c0ec3aaecabc26456c2c62f4278d8fc20597", size = 8483064, upload-time = "2025-11-21T02:49:36.454Z" }, - { url = "https://files.pythonhosted.org/packages/c4/e6/d359fdd37498e74d26a167f7a51e54542e642ea47181eb4e643a69a066c3/numcodecs-0.16.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84230b4b9dad2392f2a84242bd6e3e659ac137b5a1ce3571d6965fca673e0903", size = 9126063, upload-time = "2025-11-21T02:49:38.018Z" }, - { url = "https://files.pythonhosted.org/packages/4e/15/e2e1151b5a8b14a15dfd4bb4abccce7fff7580f39bc34092780088835f3a/numcodecs-0.16.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49f7b7d24f103187f53135bed28bb9f0ed6b2e14c604664726487bb6d7c882e1", size = 8476987, upload-time = "2025-11-21T02:49:43.363Z" }, - { url = "https://files.pythonhosted.org/packages/6d/30/16a57fc4d9fb0ba06c600408bd6634f2f1753c54a7a351c99c5e09b51ee2/numcodecs-0.16.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aec9736d81b70f337d89c4070ee3ffeff113f386fd789492fa152d26a15043e4", size = 9102377, upload-time = "2025-11-21T02:49:45.508Z" }, -] - -[[package]] -name = "numexpr" -version = "2.14.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cb/2f/fdba158c9dbe5caca9c3eca3eaffffb251f2fb8674bf8e2d0aed5f38d319/numexpr-2.14.1.tar.gz", hash = "sha256:4be00b1086c7b7a5c32e31558122b7b80243fe098579b170967da83f3152b48b", size = 119400, upload-time = "2025-10-13T16:17:27.351Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/72/94/cc921e35593b820521e464cbbeaf8212bbdb07f16dc79fe283168df38195/numexpr-2.14.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d686dfb2c1382d9e6e0ee0b7647f943c1886dba3adbf606c625479f35f1956c1", size = 452468, upload-time = "2025-10-13T16:13:29.531Z" }, - { url = "https://files.pythonhosted.org/packages/d9/43/560e9ba23c02c904b5934496486d061bcb14cd3ebba2e3cf0e2dccb6c22b/numexpr-2.14.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee6d4fbbbc368e6cdd0772734d6249128d957b3b8ad47a100789009f4de7083", size = 443631, upload-time = "2025-10-13T16:15:02.473Z" }, - { url = "https://files.pythonhosted.org/packages/7b/6c/78f83b6219f61c2c22d71ab6e6c2d4e5d7381334c6c29b77204e59edb039/numexpr-2.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3a2839efa25f3c8d4133252ea7342d8f81226c7c4dda81f97a57e090b9d87a48", size = 1417670, upload-time = "2025-10-13T16:13:33.464Z" }, - { url = "https://files.pythonhosted.org/packages/0e/bb/1ccc9dcaf46281568ce769888bf16294c40e98a5158e4b16c241de31d0d3/numexpr-2.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9f9137f1351b310436662b5dc6f4082a245efa8950c3b0d9008028df92fefb9b", size = 1466212, upload-time = "2025-10-13T16:15:12.828Z" }, - { url = "https://files.pythonhosted.org/packages/13/65/72b065f9c75baf8f474fd5d2b768350935989d4917db1c6c75b866d4067c/numexpr-2.14.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:06855053de7a3a8425429bd996e8ae3c50b57637ad3e757e0fa0602a7874be30", size = 455860, upload-time = "2025-10-13T16:13:35.811Z" }, - { url = "https://files.pythonhosted.org/packages/fc/f9/c9457652dfe28e2eb898372da2fe786c6db81af9540c0f853ee04a0699cc/numexpr-2.14.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f9366d23a2e991fd5a8b5e61a17558f028ba86158a4552f8f239b005cdf83c", size = 446574, upload-time = "2025-10-13T16:15:17.367Z" }, - { url = "https://files.pythonhosted.org/packages/b6/99/8d3879c4d67d3db5560cf2de65ce1778b80b75f6fa415eb5c3e7bd37ba27/numexpr-2.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c5f1b1605695778896534dfc6e130d54a65cd52be7ed2cd0cfee3981fd676bf5", size = 1417306, upload-time = "2025-10-13T16:13:42.813Z" }, - { url = "https://files.pythonhosted.org/packages/ea/05/6bddac9f18598ba94281e27a6943093f7d0976544b0cb5d92272c64719bd/numexpr-2.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a4ba71db47ea99c659d88ee6233fa77b6dc83392f1d324e0c90ddf617ae3f421", size = 1466145, upload-time = "2025-10-13T16:15:27.464Z" }, - { url = "https://files.pythonhosted.org/packages/84/78/3c8335f713d4aeb99fa758d7c62f0be1482d4947ce5b508e2052bb7aeee9/numexpr-2.14.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33265294376e7e2ae4d264d75b798a915d2acf37b9dd2b9405e8b04f84d05cfc", size = 465972, upload-time = "2025-10-13T16:13:45.061Z" }, - { url = "https://files.pythonhosted.org/packages/35/81/9ee5f69b811e8f18746c12d6f71848617684edd3161927f95eee7a305631/numexpr-2.14.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83647d846d3eeeb9a9255311236135286728b398d0d41d35dedb532dca807fe9", size = 456953, upload-time = "2025-10-13T16:15:31.186Z" }, - { url = "https://files.pythonhosted.org/packages/6d/39/9b8bc6e294d85cbb54a634e47b833e9f3276a8bdf7ce92aa808718a0212d/numexpr-2.14.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6e575fd3ad41ddf3355d0c7ef6bd0168619dc1779a98fe46693cad5e95d25e6e", size = 1426199, upload-time = "2025-10-13T16:13:48.231Z" }, - { url = "https://files.pythonhosted.org/packages/1e/ce/0d4fcd31ab49319740d934fba1734d7dad13aa485532ca754e555ca16c8b/numexpr-2.14.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:67ea4771029ce818573b1998f5ca416bd255156feea017841b86176a938f7d19", size = 1474214, upload-time = "2025-10-13T16:15:38.893Z" }, - { url = "https://files.pythonhosted.org/packages/9a/ed/aabd8678077848dd9a751c5558c2057839f5a09e2a176d8dfcd0850ee00e/numexpr-2.14.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5aedf38d4c0c19d3cecfe0334c3f4099fb496f54c146223d30fa930084bc8574", size = 455918, upload-time = "2025-10-13T16:13:50.338Z" }, - { url = "https://files.pythonhosted.org/packages/88/e1/3db65117f02cdefb0e5e4c440daf1c30beb45051b7f47aded25b7f4f2f34/numexpr-2.14.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439ec4d57b853792ebe5456e3160312281c3a7071ecac5532ded3278ede614de", size = 446512, upload-time = "2025-10-13T16:15:42.313Z" }, - { url = "https://files.pythonhosted.org/packages/9a/fb/7ceb9ee55b5f67e4a3e4d73d5af4c7e37e3c9f37f54bee90361b64b17e3f/numexpr-2.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e23b87f744e04e302d82ac5e2189ae20a533566aec76a46885376e20b0645bf8", size = 1417845, upload-time = "2025-10-13T16:13:53.836Z" }, - { url = "https://files.pythonhosted.org/packages/45/2d/9b5764d0eafbbb2889288f80de773791358acf6fad1a55767538d8b79599/numexpr-2.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:44f84e0e5af219dbb62a081606156420815890e041b87252fbcea5df55214c4c", size = 1466211, upload-time = "2025-10-13T16:15:48.985Z" }, - { url = "https://files.pythonhosted.org/packages/9e/d3/956a13e628d722d649fbf2fded615134a308c082e122a48bad0e90a99ce9/numexpr-2.14.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd72d8c2a165fe45ea7650b16eb8cc1792a94a722022006bb97c86fe51fd2091", size = 466242, upload-time = "2025-10-13T16:13:55.795Z" }, - { url = "https://files.pythonhosted.org/packages/d6/dd/abe848678d82486940892f2cacf39e82eec790e8930d4d713d3f9191063b/numexpr-2.14.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70d80fcb418a54ca208e9a38e58ddc425c07f66485176b261d9a67c7f2864f73", size = 457149, upload-time = "2025-10-13T16:15:52.036Z" }, - { url = "https://files.pythonhosted.org/packages/fd/bb/797b583b5fb9da5700a5708ca6eb4f889c94d81abb28de4d642c0f4b3258/numexpr-2.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:edea2f20c2040df8b54ee8ca8ebda63de9545b2112872466118e9df4d0ae99f3", size = 1426493, upload-time = "2025-10-13T16:13:59.244Z" }, - { url = "https://files.pythonhosted.org/packages/77/c4/0519ab028fdc35e3e7ee700def7f2b4631b175cd9e1202bd7966c1695c33/numexpr-2.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:790447be6879a6c51b9545f79612d24c9ea0a41d537a84e15e6a8ddef0b6268e", size = 1474413, upload-time = "2025-10-13T16:15:59.211Z" }, -] - -[[package]] -name = "numpy" -version = "2.4.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, - { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, - { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, - { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, - { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, - { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, - { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, - { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, - { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, - { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, - { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, - { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, - { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, - { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, - { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, - { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, - { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, - { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, - { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, - { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, -] - -[[package]] -name = "numpydoc" -version = "1.10.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sphinx", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/3c/dfccc9e7dee357fb2aa13c3890d952a370dd0ed071e0f7ed62ed0df567c1/numpydoc-1.10.0.tar.gz", hash = "sha256:3f7970f6eee30912260a6b31ac72bba2432830cd6722569ec17ee8d3ef5ffa01", size = 94027, upload-time = "2025-12-02T16:39:12.937Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/5e/3a6a3e90f35cea3853c45e5d5fb9b7192ce4384616f932cf7591298ab6e1/numpydoc-1.10.0-py3-none-any.whl", hash = "sha256:3149da9874af890bcc2a82ef7aae5484e5aa81cb2778f08e3c307ba6d963721b", size = 69255, upload-time = "2025-12-02T16:39:11.561Z" }, -] - -[[package]] -name = "opencv-python-headless" -version = "5.0.0.93" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1d/99/76b7c80252aa83c1af16393454aafd125a0287101afe8deb0a6821af0e30/opencv_python_headless-5.0.0.93.tar.gz", hash = "sha256:b82f9831daab90b725c7c1ee1b36cb5732c367096ac76d119e64e14eb70d5f3c", size = 81817738, upload-time = "2026-07-02T07:01:06.039Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/78/afca939f40ffe2b2380bfa86f812b2f7d4acc5a27b27dc41b49cad7ce7b4/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10818d91510e05c04568ae12b5cd120779c70c01bf897b001a6221fe430df80f", size = 36521085, upload-time = "2026-07-02T06:55:24.429Z" }, - { url = "https://files.pythonhosted.org/packages/2b/97/8170e9819764c47e436c130d3ff6cfb73b58f923eae9d3a03d8982b04aec/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:09a872a157c1376ab922a69bbf22f9a95bcc7b658a9d8b436a60212b02b2eeb4", size = 56563598, upload-time = "2026-07-02T06:55:47.355Z" }, - { url = "https://files.pythonhosted.org/packages/3a/98/1a28a7101e31801042b3098871a74b76c61581d328ef40774ff4edb53a56/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:840bd717c21e5c11cadadc022a823315ea417f961213d06b4df010e019eb16f4", size = 39648433, upload-time = "2026-07-02T06:56:04.255Z" }, - { url = "https://files.pythonhosted.org/packages/9b/21/f6ef335f6e65724aa78b8d792b48d40a48c381715f1e62f5a5049e09d07e/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ed709fdf9aa0bd1f2ed8549e71d19449b03a675bb581eb292285f6861953be37", size = 61204038, upload-time = "2026-07-02T06:56:41.823Z" }, -] - -[[package]] -name = "packaging" -version = "25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, -] - -[[package]] -name = "pandas" -version = "2.3.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "python-dateutil", marker = "sys_platform == 'linux'" }, - { name = "pytz", marker = "sys_platform == 'linux'" }, - { name = "tzdata", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/57/56/cf2dbe1a3f5271370669475ead12ce77c61726ffd19a35546e31aa8edf4e/pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908", size = 11737212, upload-time = "2025-09-29T23:19:59.765Z" }, - { url = "https://files.pythonhosted.org/packages/e5/63/cd7d615331b328e287d8233ba9fdf191a9c2d11b6af0c7a59cfcec23de68/pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89", size = 12362693, upload-time = "2025-09-29T23:20:14.098Z" }, - { url = "https://files.pythonhosted.org/packages/a6/de/8b1895b107277d52f2b42d3a6806e69cfef0d5cf1d0ba343470b9d8e0a04/pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98", size = 12771002, upload-time = "2025-09-29T23:20:26.76Z" }, - { url = "https://files.pythonhosted.org/packages/87/21/84072af3187a677c5893b170ba2c8fbe450a6ff911234916da889b698220/pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084", size = 13450971, upload-time = "2025-09-29T23:20:41.344Z" }, - { url = "https://files.pythonhosted.org/packages/16/87/9472cf4a487d848476865321de18cc8c920b8cab98453ab79dbbc98db63a/pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d", size = 11709872, upload-time = "2025-09-29T23:21:27.165Z" }, - { url = "https://files.pythonhosted.org/packages/15/07/284f757f63f8a8d69ed4472bfd85122bd086e637bf4ed09de572d575a693/pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac", size = 12306371, upload-time = "2025-09-29T23:21:40.532Z" }, - { url = "https://files.pythonhosted.org/packages/33/81/a3afc88fca4aa925804a27d2676d22dcd2031c2ebe08aabd0ae55b9ff282/pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c", size = 12765333, upload-time = "2025-09-29T23:21:55.77Z" }, - { url = "https://files.pythonhosted.org/packages/8d/0f/b4d4ae743a83742f1153464cf1a8ecfafc3ac59722a0b5c8602310cb7158/pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493", size = 13418120, upload-time = "2025-09-29T23:22:10.109Z" }, - { url = "https://files.pythonhosted.org/packages/46/b1/85331edfc591208c9d1a63a06baa67b21d332e63b7a591a5ba42a10bb507/pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78", size = 11645189, upload-time = "2025-09-29T23:22:51.688Z" }, - { url = "https://files.pythonhosted.org/packages/44/23/78d645adc35d94d1ac4f2a3c4112ab6f5b8999f4898b8cdf01252f8df4a9/pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110", size = 12121912, upload-time = "2025-09-29T23:23:05.042Z" }, - { url = "https://files.pythonhosted.org/packages/53/da/d10013df5e6aaef6b425aa0c32e1fc1f3e431e4bcabd420517dceadce354/pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86", size = 12712160, upload-time = "2025-09-29T23:23:28.57Z" }, - { url = "https://files.pythonhosted.org/packages/bd/17/e756653095a083d8a37cbd816cb87148debcfcd920129b25f99dd8d04271/pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc", size = 13199233, upload-time = "2025-09-29T23:24:24.876Z" }, - { url = "https://files.pythonhosted.org/packages/ca/05/d01ef80a7a3a12b2f8bbf16daba1e17c98a2f039cbc8e2f77a2c5a63d382/pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c", size = 11814049, upload-time = "2025-09-29T23:27:15.384Z" }, - { url = "https://files.pythonhosted.org/packages/15/b2/0e62f78c0c5ba7e3d2c5945a82456f4fac76c480940f805e0b97fcbc2f65/pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b", size = 12332638, upload-time = "2025-09-29T23:27:51.625Z" }, - { url = "https://files.pythonhosted.org/packages/c5/33/dd70400631b62b9b29c3c93d2feee1d0964dc2bae2e5ad7a6c73a7f25325/pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6", size = 12886834, upload-time = "2025-09-29T23:28:21.289Z" }, - { url = "https://files.pythonhosted.org/packages/d3/18/b5d48f55821228d0d2692b34fd5034bb185e854bdb592e9c640f6290e012/pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3", size = 13409925, upload-time = "2025-09-29T23:28:58.261Z" }, - { url = "https://files.pythonhosted.org/packages/f9/88/702bde3ba0a94b8c73a0181e05144b10f13f29ebfc2150c3a79062a8195d/pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450", size = 11634535, upload-time = "2025-09-29T23:30:21.003Z" }, - { url = "https://files.pythonhosted.org/packages/a4/1e/1bac1a839d12e6a82ec6cb40cda2edde64a2013a66963293696bbf31fbbb/pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5", size = 12121582, upload-time = "2025-09-29T23:30:43.391Z" }, - { url = "https://files.pythonhosted.org/packages/44/91/483de934193e12a3b1d6ae7c8645d083ff88dec75f46e827562f1e4b4da6/pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788", size = 12699963, upload-time = "2025-09-29T23:31:10.009Z" }, - { url = "https://files.pythonhosted.org/packages/70/44/5191d2e4026f86a2a109053e194d3ba7a31a2d10a9c2348368c63ed4e85a/pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87", size = 13202175, upload-time = "2025-09-29T23:31:59.173Z" }, -] - -[[package]] -name = "pandocfilters" -version = "1.5.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/70/6f/3dd4940bbe001c06a65f88e36bad298bc7a0de5036115639926b0c5c0458/pandocfilters-1.5.1.tar.gz", hash = "sha256:002b4a555ee4ebc03f8b66307e287fa492e4a77b4ea14d3f934328297bb4939e", size = 8454, upload-time = "2024-01-18T20:08:13.726Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/af/4fbc8cab944db5d21b7e2a5b8e9211a03a79852b1157e2c102fcc61ac440/pandocfilters-1.5.1-py2.py3-none-any.whl", hash = "sha256:93be382804a9cdb0a7267585f157e5d1731bbe5545a85b268d6f5fe6232de2bc", size = 8663, upload-time = "2024-01-18T20:08:11.28Z" }, -] - -[[package]] -name = "parso" -version = "0.8.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/30/4b/90c937815137d43ce71ba043cd3566221e9df6b9c805f24b5d138c9d40a7/parso-0.8.7.tar.gz", hash = "sha256:eaaac4c9fdd5e9e8852dc778d2d7405897ec510f2a298071453e5e3a07914bb1", size = 401824, upload-time = "2026-05-01T23:13:02.138Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, -] - -[[package]] -name = "partd" -version = "1.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "locket", marker = "sys_platform == 'linux'" }, - { name = "toolz", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b2/3a/3f06f34820a31257ddcabdfafc2672c5816be79c7e353b02c1f318daa7d4/partd-1.4.2.tar.gz", hash = "sha256:d022c33afbdc8405c226621b015e8067888173d85f7f5ecebb3cafed9a20f02c", size = 21029, upload-time = "2024-05-06T19:51:41.945Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/71/e7/40fb618334dcdf7c5a316c0e7343c5cd82d3d866edc100d98e29bc945ecd/partd-1.4.2-py3-none-any.whl", hash = "sha256:978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f", size = 18905, upload-time = "2024-05-06T19:51:39.271Z" }, -] - -[[package]] -name = "patsy" -version = "1.0.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/be/44/ed13eccdd0519eff265f44b670d46fbb0ec813e2274932dc1c0e48520f7d/patsy-1.0.2.tar.gz", hash = "sha256:cdc995455f6233e90e22de72c37fcadb344e7586fb83f06696f54d92f8ce74c0", size = 399942, upload-time = "2025-10-20T16:17:37.535Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/70/ba4b949bdc0490ab78d545459acd7702b211dfccf7eb89bbc1060f52818d/patsy-1.0.2-py2.py3-none-any.whl", hash = "sha256:37bfddbc58fcf0362febb5f54f10743f8b21dd2aa73dec7e7ef59d1b02ae668a", size = 233301, upload-time = "2025-10-20T16:17:36.563Z" }, -] - -[[package]] -name = "pexpect" -version = "4.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ptyprocess", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, -] - -[[package]] -name = "pillow" -version = "12.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, - { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, - { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, - { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ac/31fb64e1e7efb5a4b50cd3d92049ba89ac6e4d8d3bb6a74e15048ca3353e/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89", size = 4161684, upload-time = "2026-07-01T11:54:25.934Z" }, - { url = "https://files.pythonhosted.org/packages/87/b4/9805e23d2b4d77842b468513841fda254ee42f0289d25088340e4ff46e2d/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace", size = 4255487, upload-time = "2026-07-01T11:54:27.935Z" }, - { url = "https://files.pythonhosted.org/packages/df/39/ecf519435a200c693fe053a6ee4d835b41cf963a4dfc2551c4e637cb2a71/pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec", size = 3696433, upload-time = "2026-07-01T11:54:29.813Z" }, - { url = "https://files.pythonhosted.org/packages/1f/01/e18aff37cb0b4aac47ac90f016d347a49aca667ef97f190b06ac2aabc928/pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65", size = 6263736, upload-time = "2026-07-01T11:54:36.131Z" }, - { url = "https://files.pythonhosted.org/packages/f7/62/de5bdd77d935331f4f802edc11e4d82950f642caad6cb2f949837b8560e2/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3", size = 6937129, upload-time = "2026-07-01T11:54:38.216Z" }, - { url = "https://files.pythonhosted.org/packages/70/4d/105627a13300c5e0df1d174230b32fd1273062c96f7745fd552b945d1e1d/pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a", size = 6339562, upload-time = "2026-07-01T11:54:40.354Z" }, - { url = "https://files.pythonhosted.org/packages/6b/1d/f13de01a553988ab895ba1c722e06cf3144d4f57656fd5b81b6d881f1179/pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e", size = 7049439, upload-time = "2026-07-01T11:54:42.489Z" }, - { url = "https://files.pythonhosted.org/packages/dc/01/001f65b68192f0228cc1dbbc8d2530ab5d58b61037ba0587f946fea607cd/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330", size = 4161736, upload-time = "2026-07-01T11:54:51.156Z" }, - { url = "https://files.pythonhosted.org/packages/1a/d2/0219746d0fd16fc8a84498e79452375be3797d3ce4044596ce565164b84f/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217", size = 4255435, upload-time = "2026-07-01T11:54:53.414Z" }, - { url = "https://files.pythonhosted.org/packages/c8/02/8d0bc62ef0302318c46ff2a512822d2610e81c7aa46c9b3abe6cbaca5ad0/pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930", size = 3696262, upload-time = "2026-07-01T11:54:55.739Z" }, - { url = "https://files.pythonhosted.org/packages/b1/9d/8b2c807dbef61a5197c047afe99823787eb66f63daf9fb2432f91d6f0462/pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321", size = 6263757, upload-time = "2026-07-01T11:55:01.778Z" }, - { url = "https://files.pythonhosted.org/packages/5c/44/c85361f65dbe00eea8576ee467c768d25129989efb76e94f205e9ca9bb46/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b", size = 6936962, upload-time = "2026-07-01T11:55:03.93Z" }, - { url = "https://files.pythonhosted.org/packages/18/7e/e483414b35800b86b6f08dbbc7803fb5cd52c4d6f897f47d53ea2c7e6f65/pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198", size = 6339171, upload-time = "2026-07-01T11:55:05.989Z" }, - { url = "https://files.pythonhosted.org/packages/f0/f4/68c491844841ede6bed70189546b3ee9731cf9f2cbad396faff5e1ccba45/pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130", size = 7048116, upload-time = "2026-07-01T11:55:08.131Z" }, - { url = "https://files.pythonhosted.org/packages/d5/37/25c6692f06927ee973ff18c8d9ee98ad0b4d84ee67a09610c2dd1447958e/pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385", size = 6322855, upload-time = "2026-07-01T11:55:21.613Z" }, - { url = "https://files.pythonhosted.org/packages/cc/91/420637fcb8f1bc11029e403b4538e6694744428d8246118e45719f944556/pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c", size = 6989642, upload-time = "2026-07-01T11:55:24.006Z" }, - { url = "https://files.pythonhosted.org/packages/10/08/b94d7811281ccf0d143a1cf768d1c49e1e54af63e7b708ab2ee3eb87face/pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d", size = 6391281, upload-time = "2026-07-01T11:55:26.252Z" }, - { url = "https://files.pythonhosted.org/packages/d2/87/24233f785f55474dc02ce3e739c5528a77e3a862e9333d1dd7a25cc31f70/pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931", size = 7096716, upload-time = "2026-07-01T11:55:28.318Z" }, - { url = "https://files.pythonhosted.org/packages/5d/dc/8fdce34ec725a33c81c6ba122b904d6b9024e50ea9ac7bede62fab54506c/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139", size = 4162063, upload-time = "2026-07-01T11:55:37.941Z" }, - { url = "https://files.pythonhosted.org/packages/76/66/2044b9a63d3b84ff048228dfcb7cd9bf0df983e8470971bf7d4c57b693de/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402", size = 4255549, upload-time = "2026-07-01T11:55:40.022Z" }, - { url = "https://files.pythonhosted.org/packages/52/7e/1f67e6f4ece6b582ee4b539decbcc9f848dc245a93ed8cd7338bafef72f1/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c", size = 3696331, upload-time = "2026-07-01T11:55:41.98Z" }, - { url = "https://files.pythonhosted.org/packages/0c/08/93fa2e70e30a2d81547e481b6ee2bb9522117221fb1e0ce4b5df70967677/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace", size = 6273659, upload-time = "2026-07-01T11:55:48.264Z" }, - { url = "https://files.pythonhosted.org/packages/f8/6d/043e96ff814fc31a33077e4cba86082167db520c93632afdf2042febbb0c/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4", size = 6947439, upload-time = "2026-07-01T11:55:50.503Z" }, - { url = "https://files.pythonhosted.org/packages/af/92/ba71d2ee2ac0edf3fa33bd9d5ee9ee080da70b1766f3ca3934f9938ddac9/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39", size = 6353577, upload-time = "2026-07-01T11:55:52.697Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ce/e63064e2122923ff687c8ad792d0d736a7b3920a56a46982e81a7fdd25d6/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71", size = 7060394, upload-time = "2026-07-01T11:55:55.149Z" }, - { url = "https://files.pythonhosted.org/packages/13/4f/9e049dfa21af7c22427275720e2490267ba8138120add5c4c574deb69782/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e", size = 6329237, upload-time = "2026-07-01T11:56:08.868Z" }, - { url = "https://files.pythonhosted.org/packages/36/16/cf6eeaae8d0fce8dd390a33437cf68c5d5bd73834a2bc6e2f14efda0ab45/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777", size = 6997047, upload-time = "2026-07-01T11:56:11.379Z" }, - { url = "https://files.pythonhosted.org/packages/1e/69/dbf769bdd55f48bf5733cac28edc6364ffaa072ec9ba336266e4fe66be55/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1", size = 6400440, upload-time = "2026-07-01T11:56:13.908Z" }, - { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, -] - -[[package]] -name = "pims" -version = "0.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "imageio", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "slicerator", marker = "sys_platform == 'linux'" }, - { name = "tifffile", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b8/02/5bf3639f5b77e9b183011c08541c5039ba3d04f5316c70312b48a8e003a9/pims-0.7.tar.gz", hash = "sha256:55907a4c301256086d2aa4e34a5361b9109f24e375c2071e1117b9491e82946b", size = 87779, upload-time = "2024-06-10T19:20:42.842Z" } - -[[package]] -name = "platformdirs" -version = "4.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/47/e4501f49c178ae1d9f4a75073fda4204f52647993f075a9db4d14930e0c5/platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7", size = 31224, upload-time = "2026-05-28T03:32:53.587Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, -] - -[[package]] -name = "pluggy" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, -] - -[[package]] -name = "prometheus-client" -version = "0.25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/fb/d9aa83ffe43ce1f19e557c0971d04b90561b0cfd50762aafb01968285553/prometheus_client-0.25.0.tar.gz", hash = "sha256:5e373b75c31afb3c86f1a52fa1ad470c9aace18082d39ec0d2f918d11cc9ba28", size = 86035, upload-time = "2026-04-09T19:53:42.359Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/9b/d4b1e644385499c8346fa9b622a3f030dce14cd6ef8a1871c221a17a67e7/prometheus_client-0.25.0-py3-none-any.whl", hash = "sha256:d5aec89e349a6ec230805d0df882f3807f74fd6c1a2fa86864e3c2279059fed1", size = 64154, upload-time = "2026-04-09T19:53:41.324Z" }, -] - -[[package]] -name = "prompt-toolkit" -version = "3.0.52" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wcwidth", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, -] - -[[package]] -name = "psutil" -version = "7.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, - { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, - { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, - { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, - { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, - { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, - { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, - { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, -] - -[[package]] -name = "ptyprocess" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, -] - -[[package]] -name = "pulp" -version = "3.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4e/70/69be07a67621ad804d6cf347965eb4e0d7786a97330d99c31d735aaa6c5a/pulp-3.3.2.tar.gz", hash = "sha256:d0904700c207ac11e25e3b1213b70eae1d6fb25faa719d75f3f15054901258c0", size = 16305346, upload-time = "2026-05-25T09:41:26.207Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/6e/d674f1dde91c71e2ac19e5e5cf1ee6d5845e3aefecd9c39ed9c4b0c9a696/pulp-3.3.2-py3-none-any.whl", hash = "sha256:631b166f72086971a9597f7a0233ababa99bb8d50a01cd543f7758be5a9f86c0", size = 16391742, upload-time = "2026-05-25T09:41:22.2Z" }, -] - -[[package]] -name = "pure-eval" -version = "0.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cd/05/0a34433a064256a578f1783a10da6df098ceaa4a57bbeaa96a6c0352786b/pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42", size = 19752, upload-time = "2024-07-21T12:58:21.801Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, -] - -[[package]] -name = "py-cpuinfo" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, -] - -[[package]] -name = "pyarrow" -version = "25.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/27/f3/95428098d1fa7d04432fb750eed06b41304c2f6a5d3319985e64db2d9d41/pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4", size = 1199181, upload-time = "2026-07-10T08:29:50.116Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/1a/22bfb6597dcdc861fa83c39c06e1457cb56f698940eff42fbb25de30e8e5/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f", size = 46841966, upload-time = "2026-07-10T08:27:07.685Z" }, - { url = "https://files.pythonhosted.org/packages/55/0e/cd705c042bc4fe7022478db577fcab4abdcfabb9bc37ab7a75556b3fcb2b/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778", size = 50088993, upload-time = "2026-07-10T08:27:14.268Z" }, - { url = "https://files.pythonhosted.org/packages/98/ee/d822e1ee31fe31ec5d057210e0605c950b975dcd8d9a332976cc859a9df8/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537", size = 49941005, upload-time = "2026-07-10T08:27:21.274Z" }, - { url = "https://files.pythonhosted.org/packages/33/1b/207a90cc64619a095eb75a263ae069735f2810056d43c667befd573ec083/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b", size = 53112355, upload-time = "2026-07-10T08:27:27.911Z" }, - { url = "https://files.pythonhosted.org/packages/53/55/6d1d5f5aff317ec5de9421594679ed51ed828fe7e2ce209327f819d801e4/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be", size = 46841701, upload-time = "2026-07-10T08:27:49.741Z" }, - { url = "https://files.pythonhosted.org/packages/b5/5d/f790fb6965ab54c9da0dda7856abc75fd0d7648d865f8d603c111d203a64/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9", size = 50090118, upload-time = "2026-07-10T08:27:56.051Z" }, - { url = "https://files.pythonhosted.org/packages/0c/8c/faf025357ebf31bc96777f234277aa31e2aeca6dd4ecaa391f29085473c2/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d", size = 49945559, upload-time = "2026-07-10T08:28:01.927Z" }, - { url = "https://files.pythonhosted.org/packages/07/a1/bd051871708ea99a5e0fc711926c26c6f2c6d0130c7aaac8093e34998af6/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1", size = 53114238, upload-time = "2026-07-10T08:28:08.594Z" }, - { url = "https://files.pythonhosted.org/packages/af/9f/2d81ba89d1e4198d0cb25fe7529de936830fdaec0db926bb52a1ef7080d4/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec", size = 46905617, upload-time = "2026-07-10T08:28:29.376Z" }, - { url = "https://files.pythonhosted.org/packages/6a/29/0ed312ec800fb536f93783215126cee4b8977dcfeccba6f0f44df0cc87d7/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887", size = 50119765, upload-time = "2026-07-10T08:28:35.826Z" }, - { url = "https://files.pythonhosted.org/packages/ca/88/cab5063ba0c4d46a9f6b4b7eb1c9029dc0302d65cd5ab3510c949a386568/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62", size = 50027563, upload-time = "2026-07-10T08:28:43.848Z" }, - { url = "https://files.pythonhosted.org/packages/7b/fb/4d24f1b7fe2e042dc4ef315ef75e4e702d8e46fe10c37e63caff00502b03/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f", size = 53162437, upload-time = "2026-07-10T08:28:52.819Z" }, - { url = "https://files.pythonhosted.org/packages/8a/18/b37fc31a69cff4bdfb8842683def5612f551b93fff6f44375e4a4a6a5535/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50", size = 46912304, upload-time = "2026-07-10T08:29:14.656Z" }, - { url = "https://files.pythonhosted.org/packages/32/35/5cae19ba72493e5598022468b56f6a5571f399f485bf412f157356476caa/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe", size = 50073652, upload-time = "2026-07-10T08:29:22.467Z" }, - { url = "https://files.pythonhosted.org/packages/2e/a5/ddd508424bdfd5e6945765e9e2ffc687e2f6115972badc8ecf423076c407/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc", size = 50058654, upload-time = "2026-07-10T08:29:29.689Z" }, - { url = "https://files.pythonhosted.org/packages/5f/a4/324d0db203ff5eebe8694ec2d6ec5a23f9aaa5d02e5b8c692914c518c33c/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0", size = 53140153, upload-time = "2026-07-10T08:29:36.041Z" }, -] - -[[package]] -name = "pyavm" -version = "0.9.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/87/ac/a925d36dd37fc37f89afccc1f62ffa03b7344c8e4ff7850be7879b4497e6/pyavm-0.9.9.tar.gz", hash = "sha256:bc0f605d957c1fd6d7765523fcba8b9a72377ac6c51461c2a838fe44600bcd9a", size = 220572, upload-time = "2026-03-12T09:54:50.969Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/ab/ba8d2b40aee05cd986807d58ee324369eb16248b717a08b51eee977f8d33/pyavm-0.9.9-py3-none-any.whl", hash = "sha256:8bba0ee9645a8a9f215af9ceea67b494a6ee1fe380cebdc00ba99d781539ad76", size = 379786, upload-time = "2026-03-12T09:54:49.455Z" }, -] - -[[package]] -name = "pybind11" -version = "3.0.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/f0/35145a3c3baffeef55d4b8324caa33abaa8fa56ab345ecd4b2211d09163e/pybind11-3.0.4.tar.gz", hash = "sha256:3286b59c8a774b9ee650169302dd5a4eedc30a8617905a0560dd8ee44775130c", size = 589533, upload-time = "2026-04-19T03:08:15.925Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/06/c3a23c9a0263b136c519f033a58d4641e73065fefc7754e9667ec206d992/pybind11-3.0.4-py3-none-any.whl", hash = "sha256:961720ee652da51d531b7b2451a6bd2bc042b0106e6d9baa48ecb7d58034ce63", size = 314166, upload-time = "2026-04-19T03:08:14.091Z" }, -] - -[[package]] -name = "pybtex" -version = "0.26.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "latexcodec", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4d/f5/f30da9c93f0fa6d619332b2f69597219b625f35780473a05164a9981fd9a/pybtex-0.26.1.tar.gz", hash = "sha256:2e5543bea424e60e9e42eef70bff597be48649d8f68ba061a7a092b2477d5464", size = 692991, upload-time = "2026-04-03T13:05:39.014Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/f6/775eb92e865b28cdb4ad1f2bed7a5446197516f76b58a950faa3be3fd08d/pybtex-0.26.1-py3-none-any.whl", hash = "sha256:e26c0412cc54f5f21b2a6d9d175762a2d2af9ccf3a8f651cdb89ec035db77aa1", size = 126134, upload-time = "2026-04-03T13:05:40.623Z" }, -] - -[[package]] -name = "pybtex-docutils" -version = "1.0.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "pybtex", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7e/84/796ea94d26188a853660f81bded39f8de4cfe595130aef0dea1088705a11/pybtex-docutils-1.0.3.tar.gz", hash = "sha256:3a7ebdf92b593e00e8c1c538aa9a20bca5d92d84231124715acc964d51d93c6b", size = 18348, upload-time = "2023-08-22T18:47:54.833Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/11/b1/ce1f4596211efb5410e178a803f08e59b20bedb66837dcf41e21c54f9ec1/pybtex_docutils-1.0.3-py3-none-any.whl", hash = "sha256:8fd290d2ae48e32fcb54d86b0efb8d573198653c7e2447d5bec5847095f430b9", size = 6385, upload-time = "2023-08-22T06:43:20.513Z" }, -] - -[[package]] -name = "pyccl" -version = "3.3.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/57/40/ff357d61a20c21420f1fe7417ca6d889be2e07a67b3564775aeaf6c3f366/pyccl-3.3.4.tar.gz", hash = "sha256:a9d22557354fd161df909e8f4f199e4f1c434c080fd6cddb2246609ddbd70e7c", size = 16783719, upload-time = "2026-05-14T16:26:27.327Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/80/1db2504f21eec9b481e1fe298b05deb96e108d06ddb1828494d16d32bf1c/pyccl-3.3.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0229c863acea5c3fc6748dc7c40611dd529709c9d26e3679ebdbc467511ef993", size = 3707759, upload-time = "2026-05-14T16:26:16.779Z" }, - { url = "https://files.pythonhosted.org/packages/1a/91/3000c5fc1ee8379636dc894946a3633baae74a3f1cce3c2d402e8991a0b5/pyccl-3.3.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:685d528a096251a2eb5d569e611b5a2e68cc3a4a0134734b16ac4e73a07e7f27", size = 3706950, upload-time = "2026-05-14T16:26:20.394Z" }, - { url = "https://files.pythonhosted.org/packages/e5/66/803e8196149d8d8a75319b4f20268210b1e1b31b3158a1143ce99cacf953/pyccl-3.3.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:601b687b50b506f7f5775475d79fa55d221616a827a96ad5d6d68bf7a15de9b9", size = 3706955, upload-time = "2026-05-14T16:26:25.302Z" }, -] - -[[package]] -name = "pycparser" -version = "3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, -] - -[[package]] -name = "pydantic" -version = "2.13.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-types", marker = "sys_platform == 'linux'" }, - { name = "pydantic-core", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, - { name = "typing-inspection", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, -] - -[[package]] -name = "pydantic-core" -version = "2.46.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, - { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, - { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, - { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, - { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, - { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, - { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, - { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, - { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, - { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, - { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, - { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, - { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, - { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, - { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, - { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, - { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, - { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, - { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, - { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, - { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, - { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, - { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, - { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, - { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, - { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, - { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, - { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, - { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, - { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, - { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, - { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, - { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, - { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, - { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, - { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, - { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, - { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, - { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, - { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, - { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, -] - -[[package]] -name = "pyerfa" -version = "2.0.1.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/71/39/63cc8291b0cf324ae710df41527faf7d331bce573899199d926b3e492260/pyerfa-2.0.1.5.tar.gz", hash = "sha256:17d6b24fe4846c65d5e7d8c362dcb08199dc63b30a236aedd73875cc83e1f6c0", size = 818430, upload-time = "2024-11-11T15:22:30.852Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/96/b6210fc624123c8ae13e1eecb68fb75e3f3adff216d95eee1c7b05843e3e/pyerfa-2.0.1.5-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0603e8e1b839327d586c8a627cdc634b795e18b007d84f0cda5500a0908254e", size = 692794, upload-time = "2024-11-11T15:22:19.429Z" }, - { url = "https://files.pythonhosted.org/packages/e5/e0/050018d855d26d3c0b4a7d1b2ed692be758ce276d8289e2a2b44ba1014a5/pyerfa-2.0.1.5-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e43c7194e3242083f2350b46c09fd4bf8ba1bcc0ebd1460b98fc47fe2389906", size = 738711, upload-time = "2024-11-11T15:22:20.661Z" }, - { url = "https://files.pythonhosted.org/packages/b9/f5/ff91ee77308793ae32fa1e1de95e9edd4551456dd888b4e87c5938657ca5/pyerfa-2.0.1.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:07b80cd70701f5d066b1ac8cce406682cfcd667a1186ec7d7ade597239a6021d", size = 722966, upload-time = "2024-11-11T15:22:21.905Z" }, -] - -[[package]] -name = "pygments" -version = "2.20.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, -] - -[[package]] -name = "pymaster" -version = "2.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ducc0", marker = "sys_platform == 'linux'" }, - { name = "healpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/95/45/470858f2a6a3f5133955bb87aed978a5e921d4052aea1c19718e6b72f0ae/pymaster-2.6.tar.gz", hash = "sha256:056a09bc8631cf338e93ef14d8c70272ad9554ef10498d919e920994531c57a2", size = 33707181, upload-time = "2025-12-16T09:18:28.905Z" } - -[[package]] -name = "pyopenssl" -version = "26.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/74/b7/da07bae88f5a9506b4def6f2f4903cf4c3b8831e560dba8fa18ca08f758f/pyopenssl-26.3.0.tar.gz", hash = "sha256:589de7fae1c9ea670d18422ed00fc04da787bbde8e1454aea872aa57b49ad341", size = 182024, upload-time = "2026-06-12T20:28:07.458Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/18/1dd71c9b43192ab83f1d531ad6002dc81108ac36c475f79fb7a295abe2f4/pyopenssl-26.3.0-py3-none-any.whl", hash = "sha256:46367f8f66b92271e6d218da9c87607e1ef5a0bc5c8dea5bb3db82f395c385a3", size = 56008, upload-time = "2026-06-12T20:28:05.999Z" }, -] - -[[package]] -name = "pyparsing" -version = "3.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, -] - -[[package]] -name = "pytest" -version = "9.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "iniconfig", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pluggy", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, -] - -[[package]] -name = "pytest-cov" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "coverage", marker = "sys_platform == 'linux'" }, - { name = "pluggy", marker = "sys_platform == 'linux'" }, - { name = "pytest", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, -] - -[[package]] -name = "python-json-logger" -version = "4.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f7/ff/3cc9165fd44106973cd7ac9facb674a65ed853494592541d339bdc9a30eb/python_json_logger-4.1.0.tar.gz", hash = "sha256:b396b9e3ed782b09ff9d6e4f1683d46c83ad0d35d2e407c09a9ebbf038f88195", size = 17573, upload-time = "2026-03-29T04:39:56.805Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/be/0631a861af4d1c875f096c07d34e9a63639560a717130e7a87cbc82b7e3f/python_json_logger-4.1.0-py3-none-any.whl", hash = "sha256:132994765cf75bf44554be9aa49b06ef2345d23661a96720262716438141b6b2", size = 15021, upload-time = "2026-03-29T04:39:55.266Z" }, -] - -[[package]] -name = "pytz" -version = "2026.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ff/46/dd499ec9038423421951e4fad73051febaa13d2df82b4064f87af8b8c0c3/pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a", size = 320861, upload-time = "2026-05-04T01:35:29.667Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141, upload-time = "2026-05-04T01:35:27.408Z" }, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, - { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, - { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, - { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, - { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, - { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, - { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, - { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, - { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, - { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, - { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, - { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, - { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, - { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, - { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, - { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, - { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, - { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, -] - -[[package]] -name = "pyzmq" -version = "27.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "implementation_name == 'pypy' and sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/04/0b/3c9baedbdf613ecaa7aa07027780b8867f57b6293b6ee50de316c9f3222b/pyzmq-27.1.0.tar.gz", hash = "sha256:ac0765e3d44455adb6ddbf4417dcce460fc40a05978c08efdf2948072f6db540", size = 281750, upload-time = "2025-09-08T23:10:18.157Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/5e/c3c49fdd0f535ef45eefcc16934648e9e59dace4a37ee88fc53f6cd8e641/pyzmq-27.1.0-cp312-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1c179799b118e554b66da67d88ed66cd37a169f1f23b5d9f0a231b4e8d44a113", size = 895645, upload-time = "2025-09-08T23:08:05.301Z" }, - { url = "https://files.pythonhosted.org/packages/f8/e5/b0b2504cb4e903a74dcf1ebae157f9e20ebb6ea76095f6cfffea28c42ecd/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3837439b7f99e60312f0c926a6ad437b067356dc2bc2ec96eb395fd0fe804233", size = 652574, upload-time = "2025-09-08T23:08:06.828Z" }, - { url = "https://files.pythonhosted.org/packages/f8/9b/c108cdb55560eaf253f0cbdb61b29971e9fb34d9c3499b0e96e4e60ed8a5/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43ad9a73e3da1fab5b0e7e13402f0b2fb934ae1c876c51d0afff0e7c052eca31", size = 840995, upload-time = "2025-09-08T23:08:08.396Z" }, - { url = "https://files.pythonhosted.org/packages/c2/bb/b79798ca177b9eb0825b4c9998c6af8cd2a7f15a6a1a4272c1d1a21d382f/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0de3028d69d4cdc475bfe47a6128eb38d8bc0e8f4d69646adfbcd840facbac28", size = 1642070, upload-time = "2025-09-08T23:08:09.989Z" }, - { url = "https://files.pythonhosted.org/packages/9c/80/2df2e7977c4ede24c79ae39dcef3899bfc5f34d1ca7a5b24f182c9b7a9ca/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:cf44a7763aea9298c0aa7dbf859f87ed7012de8bda0f3977b6fb1d96745df856", size = 2021121, upload-time = "2025-09-08T23:08:11.907Z" }, - { url = "https://files.pythonhosted.org/packages/46/bd/2d45ad24f5f5ae7e8d01525eb76786fa7557136555cac7d929880519e33a/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f30f395a9e6fbca195400ce833c731e7b64c3919aa481af4d88c3759e0cb7496", size = 1878550, upload-time = "2025-09-08T23:08:13.513Z" }, - { url = "https://files.pythonhosted.org/packages/14/1d/d343f3ce13db53a54cb8946594e567410b2125394dafcc0268d8dda027e0/pyzmq-27.1.0-cp313-cp313t-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:08363b2011dec81c354d694bdecaef4770e0ae96b9afea70b3f47b973655cc05", size = 897275, upload-time = "2025-09-08T23:08:26.063Z" }, - { url = "https://files.pythonhosted.org/packages/69/2d/d83dd6d7ca929a2fc67d2c3005415cdf322af7751d773524809f9e585129/pyzmq-27.1.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d54530c8c8b5b8ddb3318f481297441af102517602b569146185fa10b63f4fa9", size = 660469, upload-time = "2025-09-08T23:08:27.623Z" }, - { url = "https://files.pythonhosted.org/packages/3e/cd/9822a7af117f4bc0f1952dbe9ef8358eb50a24928efd5edf54210b850259/pyzmq-27.1.0-cp313-cp313t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f3afa12c392f0a44a2414056d730eebc33ec0926aae92b5ad5cf26ebb6cc128", size = 847961, upload-time = "2025-09-08T23:08:29.672Z" }, - { url = "https://files.pythonhosted.org/packages/9a/12/f003e824a19ed73be15542f172fd0ec4ad0b60cf37436652c93b9df7c585/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c65047adafe573ff023b3187bb93faa583151627bc9c51fc4fb2c561ed689d39", size = 1650282, upload-time = "2025-09-08T23:08:31.349Z" }, - { url = "https://files.pythonhosted.org/packages/d5/4a/e82d788ed58e9a23995cee70dbc20c9aded3d13a92d30d57ec2291f1e8a3/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:90e6e9441c946a8b0a667356f7078d96411391a3b8f80980315455574177ec97", size = 2024468, upload-time = "2025-09-08T23:08:33.543Z" }, - { url = "https://files.pythonhosted.org/packages/d9/94/2da0a60841f757481e402b34bf4c8bf57fa54a5466b965de791b1e6f747d/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:add071b2d25f84e8189aaf0882d39a285b42fa3853016ebab234a5e78c7a43db", size = 1885394, upload-time = "2025-09-08T23:08:35.51Z" }, - { url = "https://files.pythonhosted.org/packages/48/43/d72ccdbf0d73d1343936296665826350cb1e825f92f2db9db3e61c2162a2/pyzmq-27.1.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1779be8c549e54a1c38f805e56d2a2e5c009d26de10921d7d51cfd1c8d4632ea", size = 897175, upload-time = "2025-09-08T23:08:46.601Z" }, - { url = "https://files.pythonhosted.org/packages/2f/2e/a483f73a10b65a9ef0161e817321d39a770b2acf8bcf3004a28d90d14a94/pyzmq-27.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7200bb0f03345515df50d99d3db206a0a6bee1955fbb8c453c76f5bf0e08fb96", size = 660427, upload-time = "2025-09-08T23:08:48.187Z" }, - { url = "https://files.pythonhosted.org/packages/f5/d2/5f36552c2d3e5685abe60dfa56f91169f7a2d99bbaf67c5271022ab40863/pyzmq-27.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01c0e07d558b06a60773744ea6251f769cd79a41a97d11b8bf4ab8f034b0424d", size = 847929, upload-time = "2025-09-08T23:08:49.76Z" }, - { url = "https://files.pythonhosted.org/packages/c4/2a/404b331f2b7bf3198e9945f75c4c521f0c6a3a23b51f7a4a401b94a13833/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:80d834abee71f65253c91540445d37c4c561e293ba6e741b992f20a105d69146", size = 1650193, upload-time = "2025-09-08T23:08:51.7Z" }, - { url = "https://files.pythonhosted.org/packages/1c/0b/f4107e33f62a5acf60e3ded67ed33d79b4ce18de432625ce2fc5093d6388/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:544b4e3b7198dde4a62b8ff6685e9802a9a1ebf47e77478a5eb88eca2a82f2fd", size = 2024388, upload-time = "2025-09-08T23:08:53.393Z" }, - { url = "https://files.pythonhosted.org/packages/0d/01/add31fe76512642fd6e40e3a3bd21f4b47e242c8ba33efb6809e37076d9b/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cedc4c68178e59a4046f97eca31b148ddcf51e88677de1ef4e78cf06c5376c9a", size = 1885316, upload-time = "2025-09-08T23:08:55.702Z" }, -] - -[[package]] -name = "qp-prob" -version = "1.1.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecated", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "tables-io", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b0/65/c54639ac63667800ee00d74c8851bd104ccaefcae95138073d6e77832d17/qp_prob-1.1.4.tar.gz", hash = "sha256:a68a3a3785d3ca57a2246c0524398338b7b7dffef0aef0722c231c64dde30107", size = 2460490, upload-time = "2026-06-25T20:08:34.63Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1c/9f/6df94f6a8fc6af157b2479c7ea0f1ef86d06e8ce7f6cba09bfbb004d5de7/qp_prob-1.1.4-py3-none-any.whl", hash = "sha256:e68d193ba0a4eb0a92f9b4861b9defc59194712865fa90e06db57196f8227e30", size = 112941, upload-time = "2026-06-25T20:08:32.68Z" }, -] - -[[package]] -name = "referencing" -version = "0.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs", marker = "sys_platform == 'linux'" }, - { name = "rpds-py", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, -] - -[[package]] -name = "regions" -version = "0.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d3/86/44ff464efa2777247d6addc74416059f1872ab05c88588bbe1aadca27a7c/regions-0.12.tar.gz", hash = "sha256:1c9460770f250ef299e90a9d5c0b35941f7d05bbf879f6ffaa0538250c018ef9", size = 273609, upload-time = "2026-06-24T20:10:17.78Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/a2/fe550b219404e98f2afc742ebb1d1bcd27888a6ad48ce60baf5fa96bde97/regions-0.12-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf5a6d8f1060607eab07f131cf12709ef7c4e9b1918d4aa269572e6c49e68b12", size = 2410524, upload-time = "2026-06-24T20:10:11.56Z" }, - { url = "https://files.pythonhosted.org/packages/92/4c/21bed50e1f86aa9fc5148318057f23d68d81f9ce0c1b31038e6c5c0e9547/regions-0.12-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e3b74d83fc4042430e8f11a0f1a3939ec6f95a7244a7103caeb6ad115d845acf", size = 2426898, upload-time = "2026-06-24T20:10:13.042Z" }, -] - -[[package]] -name = "reproject" -version = "0.21.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "astropy-healpix", marker = "sys_platform == 'linux'" }, - { name = "dask", extra = ["array"], marker = "sys_platform == 'linux'" }, - { name = "dask-image", marker = "sys_platform == 'linux'" }, - { name = "fsspec", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pillow", marker = "sys_platform == 'linux'" }, - { name = "pyavm", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "zarr", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/44/6fd820ba336484277a91a2f4808b60d6ec0b9f033f588c237e778f20fe89/reproject-0.21.0.tar.gz", hash = "sha256:01ede715a1993c29431f52ff74189ef30f5e7b2e8b4dc88c1b002145a971dc1c", size = 1622661, upload-time = "2026-06-25T15:11:34.886Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/5b/8d9b51c754ab014194d374cb4873d0729b397f67997e67721538b643e682/reproject-0.21.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2132fc5d2fa3fbbd57337099f9d47582136e653ac82402783c7ceffa26804816", size = 1776590, upload-time = "2026-06-25T15:11:30.358Z" }, - { url = "https://files.pythonhosted.org/packages/da/2d/f9d76e8e308813978227e1e43a1f25f64cd0f4b030cecaeff8baeec9eeac/reproject-0.21.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8525baaca84e949a69532c02ff491c23993e3bc22a59f694af576e473f2a9d2f", size = 1791889, upload-time = "2026-06-25T15:11:31.741Z" }, -] - -[[package]] -name = "requests" -version = "2.34.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi", marker = "sys_platform == 'linux'" }, - { name = "charset-normalizer", marker = "sys_platform == 'linux'" }, - { name = "idna", marker = "sys_platform == 'linux'" }, - { name = "urllib3", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, -] - -[[package]] -name = "rfc3339-validator" -version = "0.1.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/28/ea/a9387748e2d111c3c2b275ba970b735e04e15cdb1eb30693b6b5708c4dbd/rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b", size = 5513, upload-time = "2021-05-12T16:37:54.178Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa", size = 3490, upload-time = "2021-05-12T16:37:52.536Z" }, -] - -[[package]] -name = "rfc3986-validator" -version = "0.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/88/f270de456dd7d11dcc808abfa291ecdd3f45ff44e3b549ffa01b126464d0/rfc3986_validator-0.1.1.tar.gz", hash = "sha256:3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055", size = 6760, upload-time = "2019-10-28T16:00:19.144Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl", hash = "sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9", size = 4242, upload-time = "2019-10-28T16:00:13.976Z" }, -] - -[[package]] -name = "rfc3987-syntax" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lark", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2c/06/37c1a5557acf449e8e406a830a05bf885ac47d33270aec454ef78675008d/rfc3987_syntax-1.1.0.tar.gz", hash = "sha256:717a62cbf33cffdd16dfa3a497d81ce48a660ea691b1ddd7be710c22f00b4a0d", size = 14239, upload-time = "2025-07-18T01:05:05.015Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/71/44ce230e1b7fadd372515a97e32a83011f906ddded8d03e3c6aafbdedbb7/rfc3987_syntax-1.1.0-py3-none-any.whl", hash = "sha256:6c3d97604e4c5ce9f714898e05401a0445a641cfa276432b0a648c80856f6a3f", size = 8046, upload-time = "2025-07-18T01:05:03.843Z" }, -] - -[[package]] -name = "rich" -version = "15.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, -] - -[[package]] -name = "rocket-fft" -version = "0.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numba", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/44/b0/09cbec3177ecf56e5ceb789a69fbcdcbabe758cbb682e5d36b14aa8c05fb/rocket_fft-0.3.1.tar.gz", hash = "sha256:ddc902a361099aff9fc02fe9ea3b0d036c2f67f93df356c2b754f7627d7fa4f4", size = 74276, upload-time = "2025-12-26T00:30:54.083Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/18/12d001c33cd7c4e012da59d25086879aec303e0e157d9ec737ecab8e8f58/rocket_fft-0.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46970c0dcc70bd9c365d6a9f4a235edec2e8eaad628097c0e21d920669e25abe", size = 2151682, upload-time = "2025-12-26T00:30:30.315Z" }, - { url = "https://files.pythonhosted.org/packages/c8/92/98bd01cad9e5ceeb383722d744597d066e37cdd0e54742906341aa369c28/rocket_fft-0.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d64c9fc81206f2c77f74f68761f54ac6f1ebe4c8840f0ef9795c28ab229463f", size = 2160197, upload-time = "2025-12-26T00:30:31.945Z" }, - { url = "https://files.pythonhosted.org/packages/48/2c/5bbe8d6b0c4e009f7bc6f9f4eee802a9b556ec2c3b02b9d5e5ccfa8edff1/rocket_fft-0.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5906c6517fc87295a1d142c7e6ebd1c55e057f6ee4c605549bbcecacab83774f", size = 2151627, upload-time = "2025-12-26T00:30:37.177Z" }, - { url = "https://files.pythonhosted.org/packages/55/0d/ac67fb0b2fac382ab5585719bbd1f4927c19ba0ee3bbc28dc8a1ef83e81d/rocket_fft-0.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d830aacc34d65b28daab715aa4a90d98489e89a48c2836039982f18ba1e2db3", size = 2160355, upload-time = "2025-12-26T00:30:38.81Z" }, - { url = "https://files.pythonhosted.org/packages/d5/c6/3ff33ec86e61867c0e36436eb2c9ba258eceba94419594d73ff473460e27/rocket_fft-0.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65a7577d7f3b4e3672de179f91fee8339211b3f5ae13eef1b30cb6a318f0308c", size = 2151746, upload-time = "2025-12-26T00:30:43.66Z" }, - { url = "https://files.pythonhosted.org/packages/bf/d9/def65716cd225954d0cd5f1f6326c72c20601db5dc65557ef4b66157ccea/rocket_fft-0.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:97614dbebb2686a644bbf6c6f8a6198f063052f4c88901d0ca2808536c2e83cc", size = 2160407, upload-time = "2025-12-26T00:30:44.959Z" }, -] - -[[package]] -name = "roman-numerals" -version = "4.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/41dc953bbeb056c17d5f7a519f50fdf010bd0553be2d630bc69d1e022703/roman_numerals-4.1.0.tar.gz", hash = "sha256:1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2", size = 9077, upload-time = "2025-12-17T18:25:34.381Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7", size = 7676, upload-time = "2025-12-17T18:25:33.098Z" }, -] - -[[package]] -name = "rpds-py" -version = "2026.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, - { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, - { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, - { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, - { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, - { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, - { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, - { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, - { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, - { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, - { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, - { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, - { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, - { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, - { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, - { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, - { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, - { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, - { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, - { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, - { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, - { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, - { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, - { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, - { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, - { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, - { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, - { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, - { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, - { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, - { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, - { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, - { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, - { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, - { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, - { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, - { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, - { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, - { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, - { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, - { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, - { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, - { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, - { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, - { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, - { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, - { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, - { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, - { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, - { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, - { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, - { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, - { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, - { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, - { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, - { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, - { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, - { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, - { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, - { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, -] - -[[package]] -name = "ruff" -version = "0.15.21" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/36/6f65aa9989acdec45d417192d8f4e7921931d8a6cf87ac74bce3eed98a8e/ruff-0.15.21.tar.gz", hash = "sha256:d0cfc841c572283c36548f82664a54ce6565567f1b0d5b4cf2caac693d8b7500", size = 4769401, upload-time = "2026-07-09T20:01:34.005Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/c6/ede15cac6839f3dbce52565c8f5164a8210e669c7bc4decb03e5bdf47d0d/ruff-0.15.21-py3-none-linux_armv6l.whl", hash = "sha256:63ea0e965e5d73c90e95b2434beeafc70820536717f561b32ab6e777cb9bdf5d", size = 10854342, upload-time = "2026-07-09T20:00:53.998Z" }, - { url = "https://files.pythonhosted.org/packages/9a/6f/b4523cc90ba239ede441447a19d0c968846a3012e5a0b0c5b62831a3d5e3/ruff-0.15.21-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01d65b4831c6b2a4ba8ee6faa84049d44d982b7a706e622c4094c509e51673be", size = 10990053, upload-time = "2026-07-09T20:01:02.187Z" }, - { url = "https://files.pythonhosted.org/packages/92/cc/c6a9872a5375f0628875481cf2f66b13d7d865bf3ca2e57f91c7e762d976/ruff-0.15.21-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2c5a913a589120ce67933d5d05fd6ddbcc2481c6a054980ee767f7414c72b4fd", size = 10666096, upload-time = "2026-07-09T20:01:04.299Z" }, - { url = "https://files.pythonhosted.org/packages/ab/97/c621f7a17e097f1790fa3af6374138823b330b2d03fc38337945daca212c/ruff-0.15.21-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef04b681d02ad4dc9620f00f83ac5c22f652d0e9a9cfe431d219b16ad5ccc41", size = 11537011, upload-time = "2026-07-09T20:01:06.771Z" }, - { url = "https://files.pythonhosted.org/packages/ea/51/d928727e476e25ccc57c6f449ffd80241a651a973ad949d39cfb2a771d28/ruff-0.15.21-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16d090c0740916594157e75b80d666eab8e78083b39b3b0e1d698f4670a17b86", size = 12347101, upload-time = "2026-07-09T20:01:08.859Z" }, - { url = "https://files.pythonhosted.org/packages/1e/88/8cd62026802b16018ad06931d87997cf795ba2a6239ab659606c87d96bf0/ruff-0.15.21-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a10e74757dd65004d779b73e2f3c5210156d9980b41224d50d2ebcf1db51e67", size = 11572001, upload-time = "2026-07-09T20:01:11.092Z" }, - { url = "https://files.pythonhosted.org/packages/b2/97/f63084cf55444fc110e8cb985ebfcc592af47f597d44453d778cb81bc156/ruff-0.15.21-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bab0905d2f29e0d9fbc3c373ed23db0095edaa3f71f1f4f519ec15134d9e85c8", size = 11549239, upload-time = "2026-07-09T20:01:13.27Z" }, - { url = "https://files.pythonhosted.org/packages/9d/77/f107da4a2874b7715914b03f09ba9c54424de3ff8a1cc5d015d3ee2ce0ac/ruff-0.15.21-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:00eca240af5789fec6fe7df74c088cc1f9644ed83027113468efba7c92b94075", size = 11535340, upload-time = "2026-07-09T20:01:15.206Z" }, - { url = "https://files.pythonhosted.org/packages/d5/e9/601deb322d3303a7bf212b0100ead6f2ee3f6a044d89c30f2f92bf83c731/ruff-0.15.21-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:262ab31557a75141325e32d3357f3597645a7f084e732b6b054dde428ecd9341", size = 10964048, upload-time = "2026-07-09T20:01:17.723Z" }, - { url = "https://files.pythonhosted.org/packages/ea/2e/0f2176d1e99c15192caea19c8c3a0a955246b4cb4de795042eeb616345cd/ruff-0.15.21-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:659c4e7a4212f83306045ec7c5e5a356d16d9a6ef4ae0c7a4d872914fc655d9d", size = 10667055, upload-time = "2026-07-09T20:01:19.73Z" }, - { url = "https://files.pythonhosted.org/packages/48/60/abd74a02e0c4214f12a68becfd30af7165cfdcb0e661ecdc60bbb949c09a/ruff-0.15.21-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9e866eab611a5f959d36df2d10e446973a3610bc42b0c15b31dc27977d59c233", size = 11242043, upload-time = "2026-07-09T20:01:21.947Z" }, - { url = "https://files.pythonhosted.org/packages/b2/c6/583075d8ccabb4b229345edcaf1545eb3d8d6be90f686a479d7e94088bbf/ruff-0.15.21-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e89bc93c0d3803ba870b55c29671bad9dc6d94bb1eb181b056b52eb05b52854f", size = 11648064, upload-time = "2026-07-09T20:01:24.023Z" }, -] - -[[package]] -name = "sacc" -version = "2.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b6/82/5b603d960279b8eb7fac677ba3550c0903a8390f1bff2eee6fbfa59765da/sacc-2.4.tar.gz", hash = "sha256:a98fb4e59b3c742ddb492a15a10e1f8a4279ba36d56c0d3c35e1cd13cca1167b", size = 424905, upload-time = "2026-07-02T11:29:27.458Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/9f/56288a8aabafa24679d32d01e7db732e8e89e4074cc91aedc0bb7b7c8e46/sacc-2.4-py3-none-any.whl", hash = "sha256:da6b648998738c7c307cbdff644429633726844f8c048642725be5efc1859fa3", size = 50931, upload-time = "2026-07-02T11:29:26.286Z" }, -] - -[[package]] -name = "scipy" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/34/16f10e3042d2f1d6b66e0428308ab52224b6a23049cb2f5c1756f713815f/scipy-1.17.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21", size = 32927842, upload-time = "2026-02-23T00:18:35.367Z" }, - { url = "https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458", size = 35235890, upload-time = "2026-02-23T00:18:49.188Z" }, - { url = "https://files.pythonhosted.org/packages/c5/5c/9d7f4c88bea6e0d5a4f1bc0506a53a00e9fcb198de372bfe4d3652cef482/scipy-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb", size = 35003557, upload-time = "2026-02-23T00:18:54.74Z" }, - { url = "https://files.pythonhosted.org/packages/65/94/7698add8f276dbab7a9de9fb6b0e02fc13ee61d51c7c3f85ac28b65e1239/scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea", size = 37625856, upload-time = "2026-02-23T00:19:00.307Z" }, - { url = "https://files.pythonhosted.org/packages/b4/e0/e58fbde4a1a594c8be8114eb4aac1a55bcd6587047efc18a61eb1f5c0d30/scipy-1.17.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6", size = 32896429, upload-time = "2026-02-23T00:19:35.536Z" }, - { url = "https://files.pythonhosted.org/packages/f5/5f/f17563f28ff03c7b6799c50d01d5d856a1d55f2676f537ca8d28c7f627cd/scipy-1.17.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464", size = 35203952, upload-time = "2026-02-23T00:19:42.259Z" }, - { url = "https://files.pythonhosted.org/packages/8d/a5/9afd17de24f657fdfe4df9a3f1ea049b39aef7c06000c13db1530d81ccca/scipy-1.17.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950", size = 34979063, upload-time = "2026-02-23T00:19:47.547Z" }, - { url = "https://files.pythonhosted.org/packages/8b/13/88b1d2384b424bf7c924f2038c1c409f8d88bb2a8d49d097861dd64a57b2/scipy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369", size = 37598449, upload-time = "2026-02-23T00:19:53.238Z" }, - { url = "https://files.pythonhosted.org/packages/6d/a0/3cb6f4d2fb3e17428ad2880333cac878909ad1a89f678527b5328b93c1d4/scipy-1.17.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e", size = 33019667, upload-time = "2026-02-23T00:20:17.208Z" }, - { url = "https://files.pythonhosted.org/packages/f3/c3/2d834a5ac7bf3a0c806ad1508efc02dda3c8c61472a56132d7894c312dea/scipy-1.17.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475", size = 35264159, upload-time = "2026-02-23T00:20:23.087Z" }, - { url = "https://files.pythonhosted.org/packages/4d/77/d3ed4becfdbd217c52062fafe35a72388d1bd82c2d0ba5ca19d6fcc93e11/scipy-1.17.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50", size = 35102771, upload-time = "2026-02-23T00:20:28.636Z" }, - { url = "https://files.pythonhosted.org/packages/bd/12/d19da97efde68ca1ee5538bb261d5d2c062f0c055575128f11a2730e3ac1/scipy-1.17.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca", size = 37665910, upload-time = "2026-02-23T00:20:34.743Z" }, - { url = "https://files.pythonhosted.org/packages/ef/f2/7cdb8eb308a1a6ae1e19f945913c82c23c0c442a462a46480ce487fdc0ac/scipy-1.17.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350", size = 32957007, upload-time = "2026-02-23T00:21:19.663Z" }, - { url = "https://files.pythonhosted.org/packages/0b/2e/7eea398450457ecb54e18e9d10110993fa65561c4f3add5e8eccd2b9cd41/scipy-1.17.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118", size = 35221333, upload-time = "2026-02-23T00:21:25.278Z" }, - { url = "https://files.pythonhosted.org/packages/d9/77/5b8509d03b77f093a0d52e606d3c4f79e8b06d1d38c441dacb1e26cacf46/scipy-1.17.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068", size = 35042066, upload-time = "2026-02-23T00:21:31.358Z" }, - { url = "https://files.pythonhosted.org/packages/f9/df/18f80fb99df40b4070328d5ae5c596f2f00fffb50167e31439e932f29e7d/scipy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118", size = 37612763, upload-time = "2026-02-23T00:21:37.247Z" }, - { url = "https://files.pythonhosted.org/packages/86/f1/3383beb9b5d0dbddd030335bf8a8b32d4317185efe495374f134d8be6cce/scipy-1.17.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad", size = 33030397, upload-time = "2026-02-23T00:22:01.404Z" }, - { url = "https://files.pythonhosted.org/packages/41/68/8f21e8a65a5a03f25a79165ec9d2b28c00e66dc80546cf5eb803aeeff35b/scipy-1.17.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a", size = 35281163, upload-time = "2026-02-23T00:22:07.024Z" }, - { url = "https://files.pythonhosted.org/packages/84/8d/c8a5e19479554007a5632ed7529e665c315ae7492b4f946b0deb39870e39/scipy-1.17.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4", size = 35116291, upload-time = "2026-02-23T00:22:12.585Z" }, - { url = "https://files.pythonhosted.org/packages/52/52/e57eceff0e342a1f50e274264ed47497b59e6a4e3118808ee58ddda7b74a/scipy-1.17.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2", size = 37682317, upload-time = "2026-02-23T00:22:18.513Z" }, -] - -[[package]] -name = "seaborn" -version = "0.13.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pandas", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/86/59/a451d7420a77ab0b98f7affa3a1d78a313d2f7281a57afb1a34bae8ab412/seaborn-0.13.2.tar.gz", hash = "sha256:93e60a40988f4d65e9f4885df477e2fdaff6b73a9ded434c1ab356dd57eefff7", size = 1457696, upload-time = "2024-01-25T13:21:52.551Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/11/00d3c3dfc25ad54e731d91449895a79e4bf2384dc3ac01809010ba88f6d5/seaborn-0.13.2-py3-none-any.whl", hash = "sha256:636f8336facf092165e27924f223d3c62ca560b1f2bb5dff7ab7fad265361987", size = 294914, upload-time = "2024-01-25T13:21:49.598Z" }, -] - -[[package]] -name = "secretstorage" -version = "3.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography", marker = "sys_platform == 'linux'" }, - { name = "jeepney", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" }, -] - -[[package]] -name = "send2trash" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c5/f0/184b4b5f8d00f2a92cf96eec8967a3d550b52cf94362dad1100df9e48d57/send2trash-2.1.0.tar.gz", hash = "sha256:1c72b39f09457db3c05ce1d19158c2cbef4c32b8bedd02c155e49282b7ea7459", size = 17255, upload-time = "2026-01-14T06:27:36.056Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1c/78/504fdd027da3b84ff1aecd9f6957e65f35134534ccc6da8628eb71e76d3f/send2trash-2.1.0-py3-none-any.whl", hash = "sha256:0da2f112e6d6bb22de6aa6daa7e144831a4febf2a87261451c4ad849fe9a873c", size = 17610, upload-time = "2026-01-14T06:27:35.218Z" }, -] - -[[package]] -name = "setuptools" -version = "83.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, -] - -[[package]] -name = "shear-psf-leakage" -version = "0.2.1" -source = { git = "https://github.com/CosmoStat/shear_psf_leakage.git?rev=develop#0b3c17523b77760c9259be15ba5bbba99d575d0a" } -dependencies = [ - { name = "camb", marker = "sys_platform == 'linux'" }, - { name = "cs-util", marker = "sys_platform == 'linux'" }, - { name = "emcee", marker = "sys_platform == 'linux'" }, - { name = "getdist", marker = "sys_platform == 'linux'" }, - { name = "gsl", marker = "sys_platform == 'linux'" }, - { name = "jupyter", marker = "sys_platform == 'linux'" }, - { name = "jupyter-server", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab", marker = "sys_platform == 'linux'" }, - { name = "jupytext", marker = "sys_platform == 'linux'" }, - { name = "lenspack", marker = "sys_platform == 'linux'" }, - { name = "lmfit", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "notebook", marker = "sys_platform == 'linux'" }, - { name = "pandas", marker = "sys_platform == 'linux'" }, - { name = "pyccl", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "stats", marker = "sys_platform == 'linux'" }, - { name = "swig", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "tqdm", marker = "sys_platform == 'linux'" }, - { name = "treecorr", marker = "sys_platform == 'linux'" }, - { name = "uncertainties", marker = "sys_platform == 'linux'" }, -] - -[[package]] -name = "six" -version = "1.17.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, -] - -[[package]] -name = "skyproj" -version = "2.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "healsparse", marker = "sys_platform == 'linux'" }, - { name = "hpgeom", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/90/cf/5158151ae6fc60459f430451b612a4bf2da07ebff2a9f52a37e73715a5ee/skyproj-2.5.0.tar.gz", hash = "sha256:cb8d5115927ca43cacdb6d92f00bb4fcc8563ee71028f755203ce1752e67c140", size = 8521587, upload-time = "2026-06-26T19:51:15.378Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/b1/1d86ee4ae4ae7a47e31cc770dc83057388ef16311498283ca90046b3bd04/skyproj-2.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cbfcbd4d3d600557ab19bb5cf79b750ec552da054e3be781bce88e996b9955d", size = 4146686, upload-time = "2026-06-26T19:51:04.011Z" }, - { url = "https://files.pythonhosted.org/packages/6e/e9/f2dd235dcb9f6c3b2fec36ef04d7dd36aef4daf9afc4ab4390d6230c96dc/skyproj-2.5.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78618084c9362b62ddbfd336ca1864c94d99e2a1f26d2fbf3466fd19dc702c58", size = 4146630, upload-time = "2026-06-26T19:51:10.552Z" }, -] - -[[package]] -name = "slicerator" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0c/52/f38586b82b2935f8b59a09b0a79c545a22ed062e728c9418bafeb51f61e0/slicerator-1.1.0.tar.gz", hash = "sha256:44010a7f5cd87680c07213b5cabe81d1fb71252962943e5373ee7d14605d6046", size = 38283, upload-time = "2022-04-07T18:54:08.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/ae/fa6cd331b364ad2bbc31652d025f5747d89cbb75576733dfdf8efe3e4d62/slicerator-1.1.0-py3-none-any.whl", hash = "sha256:167668d48c6d3a5ba0bd3d54b2688e81ee267dc20aef299e547d711e6f3c441a", size = 10274, upload-time = "2022-04-07T18:54:07.029Z" }, -] - -[[package]] -name = "smart-open" -version = "7.7.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wrapt", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/db/c6/22e7a2acd5d27941e85e0d7ede398da5abe2e4677d2265c924157247c32e/smart_open-7.7.1.tar.gz", hash = "sha256:9414ba5733e28309f29b28a303b0f1054ad23fe0275f1a1b600c80a724f4bd1a", size = 54952, upload-time = "2026-06-26T07:56:35.309Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/c5/14732c140fa0af0e59345949eaf9016c75ad3f076586f2413e3d3566a8da/smart_open-7.7.1-py3-none-any.whl", hash = "sha256:cb62dc45f519bf39b612564d326d2a17556f3a6056e3c3a86a07215d670d45bc", size = 65061, upload-time = "2026-06-26T07:56:33.685Z" }, -] - -[[package]] -name = "smmap" -version = "5.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1f/ea/49c993d6dfdd7338c9b1000a0f36817ed7ec84577ae2e52f890d1a4ff909/smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c", size = 22506, upload-time = "2026-03-09T03:43:26.1Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, -] - -[[package]] -name = "smokescreen" -version = "1.5.6" -source = { git = "https://github.com/UNIONS-WL/Smokescreen?rev=588a6b9b26560bd5ba3dd5ba342f3c40152644f9#588a6b9b26560bd5ba3dd5ba342f3c40152644f9" } -dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "cryptography", marker = "sys_platform == 'linux'" }, - { name = "jsonargparse", extra = ["signatures"], marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pyccl", marker = "sys_platform == 'linux'" }, - { name = "sacc", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, -] - -[[package]] -name = "snakemake" -version = "9.23.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "conda-inject", marker = "sys_platform == 'linux'" }, - { name = "configargparse", marker = "sys_platform == 'linux'" }, - { name = "connection-pool", marker = "sys_platform == 'linux'" }, - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "dpath", marker = "sys_platform == 'linux'" }, - { name = "gitpython", marker = "sys_platform == 'linux'" }, - { name = "humanfriendly", marker = "sys_platform == 'linux'" }, - { name = "immutables", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "jsonschema", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "platformdirs", marker = "sys_platform == 'linux'" }, - { name = "psutil", marker = "sys_platform == 'linux'" }, - { name = "pulp", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "referencing", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, - { name = "smart-open", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-executor-plugins", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-logger-plugins", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-report-plugins", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-scheduler-plugins", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-storage-plugins", marker = "sys_platform == 'linux'" }, - { name = "sqlmodel", marker = "sys_platform == 'linux'" }, - { name = "tabulate", marker = "sys_platform == 'linux'" }, - { name = "tenacity", marker = "sys_platform == 'linux'" }, - { name = "throttler", marker = "sys_platform == 'linux'" }, - { name = "wrapt", marker = "sys_platform == 'linux'" }, - { name = "yte", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9e/c2/45aa858e55edbb7dbc243ed60859f4f3d92d63ecdd67a80632aa3659fe6b/snakemake-9.23.1.tar.gz", hash = "sha256:ef8d698bfce66a6669cc29df7e344b0b367fd90956c725e5ae0a79a556e8e93f", size = 6802587, upload-time = "2026-06-18T09:42:57.653Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/65/2cb1c99117746bea8aff34dc434703f8f23d3876860e2ff3ebeccb754b0f/snakemake-9.23.1-py3-none-any.whl", hash = "sha256:9fdbfff25f1b442cdf7906ca06b465fc24b9bf72e2d51514fb93d83e3f8aeb35", size = 1162133, upload-time = "2026-06-18T09:42:55.541Z" }, -] - -[[package]] -name = "snakemake-interface-common" -version = "1.23.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argparse-dataclass", marker = "sys_platform == 'linux'" }, - { name = "configargparse", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/89/c3/592f832f6e5d2d31f749392e48e8401b7625dec668d3d365d8d28f2b6c30/snakemake_interface_common-1.23.0.tar.gz", hash = "sha256:6ed14531a461417659364a0dd0acc51b786af4e26fc15cc5e00ff3d9fcaffacc", size = 13960, upload-time = "2026-03-08T21:54:29.251Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/28/90/a4a3652e832667917383f8d34112bdb37b78a6d14efc049e22a2c8192524/snakemake_interface_common-1.23.0-py3-none-any.whl", hash = "sha256:43bb10102d07149f27787dfc65a7342b86ff772973e7f9ca4940c89cb412a11e", size = 17162, upload-time = "2026-03-08T21:54:28.074Z" }, -] - -[[package]] -name = "snakemake-interface-executor-plugins" -version = "9.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argparse-dataclass", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, - { name = "throttler", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/54/50/de06b284c45a8e94fb8e4a12d5235065e78b49b8f84329dc10fe39f4b7dd/snakemake_interface_executor_plugins-9.4.0.tar.gz", hash = "sha256:9d4138897beacbaadaedad94b63f948eaeb604b7fc78f9cf65ac57f090f2c066", size = 16549, upload-time = "2026-03-08T17:04:02.644Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/2c/4422eb14884bb4955ce65819799ff6452dcd3aa5f8e158540a287877bbcf/snakemake_interface_executor_plugins-9.4.0-py3-none-any.whl", hash = "sha256:d335c093185897e4e3654a39bab3a6f1c57dc1a54f998c92bd2d88640ab4a05e", size = 22567, upload-time = "2026-03-08T17:04:01.296Z" }, -] - -[[package]] -name = "snakemake-interface-logger-plugins" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a7/0c/3fa5d592663c65669a867526604aadc6fbc235fb9284e94b49c0ef59aa41/snakemake_interface_logger_plugins-2.1.0.tar.gz", hash = "sha256:c89a00d2a398490cecd91b6dc6db8049cba93712d82e1d8f3000f3040bf3791c", size = 15917, upload-time = "2026-05-20T15:12:35.259Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/43/cfdaf562303012024c8f48efebdfe258eef3d895665bd7e0000fdeb2411d/snakemake_interface_logger_plugins-2.1.0-py3-none-any.whl", hash = "sha256:601e3e0770dd1b460c1dcb7bd33f4fc2eb8af5016634b80d985cb993f674ef46", size = 14313, upload-time = "2026-05-20T15:12:34.186Z" }, -] - -[[package]] -name = "snakemake-interface-report-plugins" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/d6/6160ed98de665d6871dd356597dbf726688cc786e88668359ca37b7d9f54/snakemake_interface_report_plugins-1.3.0.tar.gz", hash = "sha256:fc9495298bec4e69721ab8afe6d6d88a86966fda2eeb003db56b9a88b86d5934", size = 4283, upload-time = "2025-10-31T10:52:36.55Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/f0/df73f6abc9b5910e43612ae28c7b6f666af80c4edd46a216ef47599ab6cb/snakemake_interface_report_plugins-1.3.0-py3-none-any.whl", hash = "sha256:78da3931f70e79eef51e5645a40b172929e555fe4d86ff45d6b856e521a379db", size = 7251, upload-time = "2025-10-31T10:52:35.474Z" }, -] - -[[package]] -name = "snakemake-interface-scheduler-plugins" -version = "2.0.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/88/d9/d480807d2cfc2d132bc760d877d45ec8fbe620a24200ec4d2697c4a26031/snakemake_interface_scheduler_plugins-2.0.2.tar.gz", hash = "sha256:2797e8fa9019d983132c2b403f14d6fcd3c5ad4c8d8a66b984b4740a71cacc46", size = 8642, upload-time = "2025-10-20T13:58:12.988Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/d0/f4e9894c8aaf37efe3bf1afe15ee3cf0546d82b2713a589e266ee47bf2ef/snakemake_interface_scheduler_plugins-2.0.2-py3-none-any.whl", hash = "sha256:b9ddfa508bd480711de1770dfb24f3b813cfa3cd0f862f0127ef721ae5346915", size = 10766, upload-time = "2025-10-20T13:58:11.898Z" }, -] - -[[package]] -name = "snakemake-interface-storage-plugins" -version = "4.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "humanfriendly", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, - { name = "tenacity", marker = "sys_platform == 'linux'" }, - { name = "throttler", marker = "sys_platform == 'linux'" }, - { name = "wrapt", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/93/6e/f3c5b2d621fd6a6b78d8cfc01fef6b926fe2c277f5ed77c5e4deeacb94eb/snakemake_interface_storage_plugins-4.4.1.tar.gz", hash = "sha256:b2b5bf05318af36955ebf2ce76c921c0fb06904ca98fb30e1657d88b7b7b6945", size = 14924, upload-time = "2026-03-16T11:16:01.075Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/49/1f72d8c05d8588f06a61f1af3deb125a4ff8169d2a41f1b839d62c824d1c/snakemake_interface_storage_plugins-4.4.1-py3-none-any.whl", hash = "sha256:6e075f448543cc3d0cb1ee5e6e5a83af9761e7085f3ce0922011685d098f231e", size = 18256, upload-time = "2026-03-16T11:15:59.582Z" }, -] - -[[package]] -name = "snowballstemmer" -version = "3.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/f8/0a71edf031f03c40db17503cb8ca78a69a171254e568e7db241b0ab57ea1/snowballstemmer-3.1.1.tar.gz", hash = "sha256:e07bbc54a0d798fe6010a12398422e62a8bfbba95c394fd0956ef58cb4d3e260", size = 123314, upload-time = "2026-06-03T00:56:40.194Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl", hash = "sha256:7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752", size = 104164, upload-time = "2026-06-03T00:56:38.614Z" }, -] - -[[package]] -name = "soupsieve" -version = "2.8.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/2c/0a5f6f8ee0d5589e48c7640213ed5175d52cf540a06725b628cc1a45d6ce/soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e", size = 121110, upload-time = "2026-05-24T13:55:57.154Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" }, -] - -[[package]] -name = "sp-validation" -version = "0.6.0" -source = { virtual = "." } -dependencies = [ - { name = "adjusttext", marker = "sys_platform == 'linux'" }, - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "camb", marker = "sys_platform == 'linux'" }, - { name = "clmm", marker = "sys_platform == 'linux'" }, - { name = "colorama", marker = "sys_platform == 'linux'" }, - { name = "cosmo-numba", marker = "sys_platform == 'linux'" }, - { name = "cryptography", marker = "sys_platform == 'linux'" }, - { name = "cs-util", marker = "sys_platform == 'linux'" }, - { name = "emcee", marker = "sys_platform == 'linux'" }, - { name = "getdist", marker = "sys_platform == 'linux'" }, - { name = "h5py", marker = "sys_platform == 'linux'" }, - { name = "healpy", marker = "sys_platform == 'linux'" }, - { name = "healsparse", marker = "sys_platform == 'linux'" }, - { name = "importlib-metadata", marker = "sys_platform == 'linux'" }, - { name = "joblib", marker = "sys_platform == 'linux'" }, - { name = "jupyter", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab", marker = "sys_platform == 'linux'" }, - { name = "jupytext", marker = "sys_platform == 'linux'" }, - { name = "lenspack", marker = "sys_platform == 'linux'" }, - { name = "lmfit", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numba", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "opencv-python-headless", marker = "sys_platform == 'linux'" }, - { name = "pandas", marker = "sys_platform == 'linux'" }, - { name = "pyarrow", marker = "sys_platform == 'linux'" }, - { name = "pyccl", marker = "sys_platform == 'linux'" }, - { name = "pymaster", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "regions", marker = "sys_platform == 'linux'" }, - { name = "reproject", marker = "sys_platform == 'linux'" }, - { name = "sacc", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "seaborn", marker = "sys_platform == 'linux'" }, - { name = "shear-psf-leakage", marker = "sys_platform == 'linux'" }, - { name = "skyproj", marker = "sys_platform == 'linux'" }, - { name = "smokescreen", marker = "sys_platform == 'linux'" }, - { name = "statsmodels", marker = "sys_platform == 'linux'" }, - { name = "tqdm", marker = "sys_platform == 'linux'" }, - { name = "treecorr", marker = "sys_platform == 'linux'" }, - { name = "uncertainties", marker = "sys_platform == 'linux'" }, -] - -[package.optional-dependencies] -develop = [ - { name = "myst-parser", marker = "sys_platform == 'linux'" }, - { name = "numpydoc", marker = "sys_platform == 'linux'" }, - { name = "pytest", marker = "sys_platform == 'linux'" }, - { name = "pytest-cov", marker = "sys_platform == 'linux'" }, - { name = "ruff", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, - { name = "sphinxawesome-theme", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-bibtex", marker = "sys_platform == 'linux'" }, -] -docs = [ - { name = "myst-parser", marker = "sys_platform == 'linux'" }, - { name = "numpydoc", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, - { name = "sphinxawesome-theme", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-bibtex", marker = "sys_platform == 'linux'" }, -] -glass = [ - { name = "cosmology", marker = "sys_platform == 'linux'" }, - { name = "fitsio", marker = "sys_platform == 'linux'" }, - { name = "glass", marker = "sys_platform == 'linux'" }, - { name = "glass-ext-camb", marker = "sys_platform == 'linux'" }, -] -test = [ - { name = "pytest", marker = "sys_platform == 'linux'" }, - { name = "pytest-cov", marker = "sys_platform == 'linux'" }, - { name = "ruff", marker = "sys_platform == 'linux'" }, -] -workflow = [ - { name = "mpi4py", marker = "sys_platform == 'linux'" }, - { name = "snakemake", marker = "sys_platform == 'linux'" }, -] - -[package.metadata] -requires-dist = [ - { name = "adjusttext" }, - { name = "astropy", specifier = ">=5.0" }, - { name = "camb", specifier = ">=1.6" }, - { name = "clmm" }, - { name = "colorama" }, - { name = "cosmo-numba", git = "https://github.com/aguinot/cosmo-numba.git?rev=main" }, - { name = "cosmology", marker = "extra == 'glass'", specifier = "==2022.10.9" }, - { name = "cryptography" }, - { name = "cs-util", git = "https://github.com/CosmoStat/cs_util.git?rev=develop" }, - { name = "emcee" }, - { name = "fitsio", marker = "extra == 'glass'" }, - { name = "getdist", git = "https://github.com/benabed/getdist.git?rev=113cd22a9a0d013b6f72fe734be81f260f3d3be5" }, - { name = "glass", marker = "extra == 'glass'", specifier = "==2025.1" }, - { name = "glass-ext-camb", marker = "extra == 'glass'", specifier = "==2023.6" }, - { name = "h5py" }, - { name = "healpy" }, - { name = "healsparse" }, - { name = "importlib-metadata" }, - { name = "joblib", specifier = ">=0.13" }, - { name = "jupyter" }, - { name = "jupyterlab" }, - { name = "jupytext", specifier = ">=1.15" }, - { name = "lenspack" }, - { name = "lmfit" }, - { name = "matplotlib" }, - { name = "mpi4py", marker = "extra == 'workflow'" }, - { name = "myst-parser", marker = "extra == 'docs'", specifier = ">=4.0" }, - { name = "numba" }, - { name = "numpy", specifier = ">=2.0" }, - { name = "numpydoc", marker = "extra == 'docs'", specifier = ">=1.8" }, - { name = "opencv-python-headless" }, - { name = "pandas" }, - { name = "pyarrow" }, - { name = "pyccl" }, - { name = "pymaster" }, - { name = "pytest", marker = "extra == 'test'" }, - { name = "pytest-cov", marker = "extra == 'test'" }, - { name = "pyyaml" }, - { name = "regions" }, - { name = "reproject" }, - { name = "ruff", marker = "extra == 'test'" }, - { name = "sacc", specifier = ">=0.12" }, - { name = "scipy", specifier = "<1.18" }, - { name = "seaborn" }, - { name = "shear-psf-leakage", git = "https://github.com/CosmoStat/shear_psf_leakage.git?rev=develop" }, - { name = "skyproj" }, - { name = "smokescreen", git = "https://github.com/UNIONS-WL/Smokescreen?rev=588a6b9b26560bd5ba3dd5ba342f3c40152644f9" }, - { name = "snakemake", marker = "extra == 'workflow'" }, - { name = "sp-validation", extras = ["test", "docs"], marker = "extra == 'develop'" }, - { name = "sphinx", marker = "extra == 'docs'", specifier = ">=8.0" }, - { name = "sphinxawesome-theme", marker = "extra == 'docs'", specifier = ">=5.3,!=6.0.3" }, - { name = "sphinxcontrib-bibtex", marker = "extra == 'docs'", specifier = ">=2.6" }, - { name = "statsmodels" }, - { name = "tqdm" }, - { name = "treecorr", specifier = ">=5.0" }, - { name = "uncertainties" }, -] -provides-extras = ["test", "docs", "glass", "workflow", "develop"] - -[[package]] -name = "sphinx" -version = "9.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "alabaster", marker = "sys_platform == 'linux'" }, - { name = "babel", marker = "sys_platform == 'linux'" }, - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "imagesize", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, - { name = "roman-numerals", marker = "sys_platform == 'linux'" }, - { name = "snowballstemmer", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-applehelp", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-devhelp", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-htmlhelp", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-jsmath", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-qthelp", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-serializinghtml", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/73/f7/b1884cb3188ab181fc81fa00c266699dab600f927a964df02ec3d5d1916a/sphinx-9.1.0-py3-none-any.whl", hash = "sha256:c84fdd4e782504495fe4f2c0b3413d6c2bf388589bb352d439b2a3bb99991978", size = 3921742, upload-time = "2025-12-31T15:09:25.561Z" }, -] - -[[package]] -name = "sphinxawesome-theme" -version = "6.0.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "beautifulsoup4", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9a/3f/ba7f8dc2837c0ac362127c6cd491578c23ac4d4e8cc6a37726531be29256/sphinxawesome_theme-6.0.5.tar.gz", hash = "sha256:ed3d82b7f0e30e3d6f38f5245d89775e9472a80a45ffd73d6a0a4b9d05ac5736", size = 343775, upload-time = "2026-06-15T15:09:36.094Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/5c/31521b6a35410cdced12af394b715387537a262bdd1bcf08b2a34a9a4df9/sphinxawesome_theme-6.0.5-py3-none-any.whl", hash = "sha256:c18e60b06f164f181e3ef7540a4a237f1cf4e82d8909a964e5c65e6e8293f51c", size = 376755, upload-time = "2026-06-15T15:09:34.523Z" }, -] - -[[package]] -name = "sphinxcontrib-applehelp" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1", size = 20053, upload-time = "2024-07-29T01:09:00.465Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5", size = 119300, upload-time = "2024-07-29T01:08:58.99Z" }, -] - -[[package]] -name = "sphinxcontrib-bibtex" -version = "2.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "pybtex", marker = "sys_platform == 'linux'" }, - { name = "pybtex-docutils", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/15/6a/8e0b2c2420286389e7fed78ff361ec30e2f1d58c8560af8d64df5e7b61e0/sphinxcontrib_bibtex-2.7.0.tar.gz", hash = "sha256:fee700f7aae29bb8f654c62913f00d34ac44fc0b8ca0fa67ac922ff4453addee", size = 120669, upload-time = "2026-05-06T09:29:24.935Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/c0/d28e62407f4733bbe0169287bc012f0ac3b4a2021066b285570654119c8b/sphinxcontrib_bibtex-2.7.0-py3-none-any.whl", hash = "sha256:28cf0ec7a957d1c7548d5749317ed472ce877e1b629f430f88e3789aa51f87b1", size = 40287, upload-time = "2026-05-06T09:29:23.253Z" }, -] - -[[package]] -name = "sphinxcontrib-devhelp" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad", size = 12967, upload-time = "2024-07-29T01:09:23.417Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2", size = 82530, upload-time = "2024-07-29T01:09:21.945Z" }, -] - -[[package]] -name = "sphinxcontrib-htmlhelp" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9", size = 22617, upload-time = "2024-07-29T01:09:37.889Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705, upload-time = "2024-07-29T01:09:36.407Z" }, -] - -[[package]] -name = "sphinxcontrib-jsmath" -version = "1.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8", size = 5787, upload-time = "2019-01-21T16:10:16.347Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071, upload-time = "2019-01-21T16:10:14.333Z" }, -] - -[[package]] -name = "sphinxcontrib-qthelp" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab", size = 17165, upload-time = "2024-07-29T01:09:56.435Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb", size = 88743, upload-time = "2024-07-29T01:09:54.885Z" }, -] - -[[package]] -name = "sphinxcontrib-serializinghtml" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d", size = 16080, upload-time = "2024-07-29T01:10:09.332Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, -] - -[[package]] -name = "sqlalchemy" -version = "2.0.51" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "greenlet", marker = "(platform_machine == 'AMD64' and sys_platform == 'linux') or (platform_machine == 'WIN32' and sys_platform == 'linux') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'amd64' and sys_platform == 'linux') or (platform_machine == 'ppc64le' and sys_platform == 'linux') or (platform_machine == 'win32' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/02/f1/a7a892f18d4d224e6b26f706531eafccc41e37594d37d304786969ee13cb/sqlalchemy-2.0.51.tar.gz", hash = "sha256:804dccd8a4a6242c4e30ad961e540e18a588f6527202f2d6791b01845d59fdc9", size = 9912201, upload-time = "2026-06-15T15:41:20.012Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/1c/71ee0f8a6b9d7316a1ccd30430b4c62b6c2e36adc96017a4e3a72dce49d6/sqlalchemy-2.0.51-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581921d849d6e6f994d560389192955e80e2950e18fcdfe2ccea863e01158e6e", size = 3343835, upload-time = "2026-06-15T16:19:42.613Z" }, - { url = "https://files.pythonhosted.org/packages/2b/7c/7ab9f9aadc5944fdd06612484ed7918fe376ad871a5f50404dc1536e0194/sqlalchemy-2.0.51-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d21ce524ab86c23046e992a5b81cb54c21079c6df6e78b8fc77d77cac70a6b9", size = 3358470, upload-time = "2026-06-15T16:26:38.011Z" }, - { url = "https://files.pythonhosted.org/packages/d0/7d/ff77169fee6186de145a7f2b87006c39638391130abbab2b1f63ac6ea583/sqlalchemy-2.0.51-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c5d98a2709840027f5a347c3af0a7c3d5f6c1ff93af2ca1c54494e23cba8f389", size = 3289874, upload-time = "2026-06-15T16:19:45.212Z" }, - { url = "https://files.pythonhosted.org/packages/6f/3b/6c505903710d781b55bc3141ee34a062bf9745a6b5bc7333305b9ed63b33/sqlalchemy-2.0.51-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1181256e0f16479691b5616d36375dc2620ad8332b25978763c3d206ad3f3f1d", size = 3321692, upload-time = "2026-06-15T16:26:39.747Z" }, - { url = "https://files.pythonhosted.org/packages/17/6b/2dce8369b199cb855110e056032f94a9f66dacc2237d3d39c115a86eac56/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:483b11bd46bf35fc14c52faf338b04300c9e6ce554bce9b11be85bfec3bc3195", size = 3284735, upload-time = "2026-06-15T16:19:46.934Z" }, - { url = "https://files.pythonhosted.org/packages/53/ff/dbc495b8a14da840faffb353857a72d4190113cac33727906fb997047f0f/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1bed1ee8b01da6088210aa9412023326fb98a599ba502e6118308601dcbef77f", size = 3302756, upload-time = "2026-06-15T16:26:41.336Z" }, - { url = "https://files.pythonhosted.org/packages/cf/d5/fde8f4dddcf518ee15ab35a7c6a28acc32c8ba548d1d2aa451f96e6dbb0b/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:72ca54c952107ba5cd58854b67a5a6268631289d21651a1235396f3b98b47400", size = 3232055, upload-time = "2026-06-15T16:19:49.286Z" }, - { url = "https://files.pythonhosted.org/packages/67/d1/43d3a0ac955a58601c24fa23038b1c55ee3a1ec02c0f96ebb1eae2bcf614/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b3e693d15533a45cd5906f0589f9c35090bef6ef45bf1e8195c424aa0ae06a8d", size = 3269850, upload-time = "2026-06-15T16:26:43.017Z" }, - { url = "https://files.pythonhosted.org/packages/23/6b/2e0e38cf75c8780eca78d9b2e78164f8bcfd70125e5caa588ff5cbb9c9f4/sqlalchemy-2.0.51-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c45a496d6bc05dec41dcd4c3a2b183723f47473255c159cd80b503c8f246424d", size = 3282539, upload-time = "2026-06-15T16:19:51.065Z" }, - { url = "https://files.pythonhosted.org/packages/dd/a1/e77854cb5336fd37dc3c6ae3b71de242c98caac5725120be0b526b31cbd0/sqlalchemy-2.0.51-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4004ada0aafe8ae1991b2cd1d99c6d9146126e123bd6f883c260d974aa012e54", size = 3287545, upload-time = "2026-06-15T16:26:44.735Z" }, - { url = "https://files.pythonhosted.org/packages/f6/ab/9e17272fd4dac8df3b83c4fbe52b998a1c9d89a843c8c35ff29b74ff7364/sqlalchemy-2.0.51-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0f6bcad487aee1c638d707235682fc96f741de00663619881ab235400d03289e", size = 3230929, upload-time = "2026-06-15T16:19:52.625Z" }, - { url = "https://files.pythonhosted.org/packages/02/3c/52f408ea701781caee975606beccc48845f2aee8711ac29843d612c0306c/sqlalchemy-2.0.51-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:39a76529db6305693d8d4affa58ad5b5e2e18edd62daea628b29b97930b3513d", size = 3252888, upload-time = "2026-06-15T16:26:46.454Z" }, - { url = "https://files.pythonhosted.org/packages/bf/90/a1bb5c7cbba76b7bc1fbd586d0a5479a7bc9c27b4a8298f22ec9423b2bb3/sqlalchemy-2.0.51-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c6b36ed71f41942bdcd2ad2522be46bfce09d5705be5640ecf19bbc7660e4b7", size = 3566534, upload-time = "2026-06-15T15:58:35.024Z" }, - { url = "https://files.pythonhosted.org/packages/15/4b/481f1fed30e0e9e8dd24aecbb49f29eb57fe7657ece5cf06ee9b84bb97d8/sqlalchemy-2.0.51-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c2c62877097e1a0db401fba5cb4debee33265e5b2a55c4ccb489c02c53b4f72", size = 3535844, upload-time = "2026-06-15T16:02:43.973Z" }, - { url = "https://files.pythonhosted.org/packages/02/71/0aa64aeda645510af0a43f7d9ee70932f0d1dc4263aed34c50ee891d9df3/sqlalchemy-2.0.51-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0378d055e9e8cd6ce4d8dff683bdd3d7d413533c4ee51d67a2b1e0f9eacc0f23", size = 3475355, upload-time = "2026-06-15T15:58:36.592Z" }, - { url = "https://files.pythonhosted.org/packages/05/db/6061db32316446135a3abae5f308d144ab988a34234726042da3e58b1c63/sqlalchemy-2.0.51-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6e46fc36029eff666391e0531e5387b62ce6c4f1d8e50b3fb3099eaca1b42522", size = 3486591, upload-time = "2026-06-15T16:02:45.346Z" }, - { url = "https://files.pythonhosted.org/packages/e2/22/dbf013a12ec759e54a34a119e9e217435b3f71b2dd5c61a7ade0a25dae87/sqlalchemy-2.0.51-py3-none-any.whl", hash = "sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5", size = 1944334, upload-time = "2026-06-15T16:09:22.418Z" }, -] - -[[package]] -name = "sqlmodel" -version = "0.0.37" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic", marker = "sys_platform == 'linux'" }, - { name = "sqlalchemy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fb/26/1d2faa0fd5a765267f49751de533adac6b9ff9366c7c6e7692df4f32230f/sqlmodel-0.0.37.tar.gz", hash = "sha256:d2c19327175794faf50b1ee31cc966764f55b1dedefc046450bc5741a3d68352", size = 85527, upload-time = "2026-02-21T16:39:47.038Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/e1/7c8d18e737433f3b5bbe27b56a9072a9fcb36342b48f1bef34b6da1d61f2/sqlmodel-0.0.37-py3-none-any.whl", hash = "sha256:2137a4045ef3fd66a917a7717ada959a1ceb3630d95e1f6aaab39dd2c0aef278", size = 27224, upload-time = "2026-02-21T16:39:47.781Z" }, -] - -[[package]] -name = "stack-data" -version = "0.6.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "asttokens", marker = "sys_platform == 'linux'" }, - { name = "executing", marker = "sys_platform == 'linux'" }, - { name = "pure-eval", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, -] - -[[package]] -name = "stats" -version = "0.1.2a0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/87/43/f2d8d8e2660740ecca9c0fab85a07bc02da27d1fcc5071ee481082888720/stats-0.1.2a.tar.gz", hash = "sha256:854e59fa99f92ce8319c16149ef8e9939dba3e2237bbb0fb6a6094c60155bd4c", size = 127625, upload-time = "2010-12-31T09:02:15.131Z" } - -[[package]] -name = "statsmodels" -version = "0.14.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pandas", marker = "sys_platform == 'linux'" }, - { name = "patsy", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0d/81/e8d74b34f85285f7335d30c5e3c2d7c0346997af9f3debf9a0a9a63de184/statsmodels-0.14.6.tar.gz", hash = "sha256:4d17873d3e607d398b85126cd4ed7aad89e4e9d89fc744cdab1af3189a996c2a", size = 20689085, upload-time = "2025-12-05T23:08:39.522Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/48/f5/3a73b51e6450c31652c53a8e12e24eac64e3824be816c0c2316e7dbdcb7d/statsmodels-0.14.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8c00a42863e4f4733ac9d078bbfad816249c01451740e6f5053ecc7db6d6368", size = 10058649, upload-time = "2025-12-05T23:10:12.775Z" }, - { url = "https://files.pythonhosted.org/packages/81/68/dddd76117df2ef14c943c6bbb6618be5c9401280046f4ddfc9fb4596a1b8/statsmodels-0.14.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19b58cf7474aa9e7e3b0771a66537148b2df9b5884fbf156096c0e6c1ff0469d", size = 10339446, upload-time = "2025-12-05T23:10:28.503Z" }, - { url = "https://files.pythonhosted.org/packages/56/4a/dce451c74c4050535fac1ec0c14b80706d8fc134c9da22db3c8a0ec62c33/statsmodels-0.14.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:81e7dcc5e9587f2567e52deaff5220b175bf2f648951549eae5fc9383b62bc37", size = 10368705, upload-time = "2025-12-05T23:10:44.339Z" }, - { url = "https://files.pythonhosted.org/packages/ee/77/0ec96803eba444efd75dba32f2ef88765ae3e8f567d276805391ec2c98c6/statsmodels-0.14.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89ee7d595f5939cc20bf946faedcb5137d975f03ae080f300ebb4398f16a5bd4", size = 10060269, upload-time = "2025-12-05T23:11:46.338Z" }, - { url = "https://files.pythonhosted.org/packages/10/b9/fd41f1f6af13a1a1212a06bb377b17762feaa6d656947bf666f76300fc05/statsmodels-0.14.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:730f3297b26749b216a06e4327fe0be59b8d05f7d594fb6caff4287b69654589", size = 10324155, upload-time = "2025-12-05T23:12:01.805Z" }, - { url = "https://files.pythonhosted.org/packages/ee/0f/a6900e220abd2c69cd0a07e3ad26c71984be6061415a60e0f17b152ecf08/statsmodels-0.14.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f1c08befa85e93acc992b72a390ddb7bd876190f1360e61d10cf43833463bc9c", size = 10349765, upload-time = "2025-12-05T23:12:18.018Z" }, - { url = "https://files.pythonhosted.org/packages/c1/98/b0dfb4f542b2033a3341aa5f1bdd97024230a4ad3670c5b0839d54e3dcab/statsmodels-0.14.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e443e7077a6e2d3faeea72f5a92c9f12c63722686eb80bb40a0f04e4a7e267ad", size = 10090802, upload-time = "2025-12-05T23:13:20.653Z" }, - { url = "https://files.pythonhosted.org/packages/34/0e/2408735aca9e764643196212f9069912100151414dd617d39ffc72d77eee/statsmodels-0.14.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3414e40c073d725007a6603a18247ab7af3467e1af4a5e5a24e4c27bc26673b4", size = 10337587, upload-time = "2025-12-05T23:13:37.597Z" }, - { url = "https://files.pythonhosted.org/packages/0f/36/4d44f7035ab3c0b2b6a4c4ebb98dedf36246ccbc1b3e2f51ebcd7ac83abb/statsmodels-0.14.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a518d3f9889ef920116f9fa56d0338069e110f823926356946dae83bc9e33e19", size = 10363350, upload-time = "2025-12-05T23:13:53.08Z" }, -] - -[[package]] -name = "swig" -version = "4.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9c/99/0556919f96b1c291a7ff83793adb328e2a1a5e6cfb379d5acc637cdec959/swig-4.4.1.tar.gz", hash = "sha256:db9a625653f6454530a6c6553d17cf576d7aa2d253c76dc8cdd5979f0cb15012", size = 25956, upload-time = "2025-12-27T03:25:49.609Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c0/4d/860e5475fe38b9c7dc36f61d0c370a62b6cc725d2bd11ada1d22a60ff5f7/swig-4.4.1-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:520fde8805b4775ef3814576929b8c44d666220b06d8da625f292192f9e74fe8", size = 1917511, upload-time = "2025-12-27T03:25:18.064Z" }, - { url = "https://files.pythonhosted.org/packages/84/e1/5cd4de23ff2b1d225279f7c7826191b1dbdb77fc858fe12ee7be2562ddb7/swig-4.4.1-py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:d8cca45a0e8d1e045a1490ea2d05bc1c49e2c266dbbe5954048e2a93d125ee07", size = 2066284, upload-time = "2025-12-27T03:25:19.624Z" }, - { url = "https://files.pythonhosted.org/packages/96/42/d49992f3c39c90d3d46c431a4afea60fc16bae542cf07f394b298a1b21b5/swig-4.4.1-py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:bfe6ce6189fc8b983f65f5ed3db8e1d8be03accb0ec3869f986a8343f014e4d9", size = 1912492, upload-time = "2025-12-27T03:25:21.843Z" }, - { url = "https://files.pythonhosted.org/packages/10/2e/4baa50fcf932f9d58bc8394b211c7daf3b7ec1e53c08d560142f1bc99cb9/swig-4.4.1-py3-none-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9c98d7b882b6debe3cbb87e6503858e104162d7ce5ab4cfc23425da2e26e51b5", size = 1979872, upload-time = "2025-12-27T03:25:23.48Z" }, - { url = "https://files.pythonhosted.org/packages/fa/7b/e3a14d053fa18b0d2e14efcc21883816964ddbe52a0c43018e195a99aba2/swig-4.4.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ae3da2bf679a4c942a2c100789395d4d167e7da8286018124e4665f5eff43e31", size = 1895896, upload-time = "2025-12-27T03:25:24.98Z" }, - { url = "https://files.pythonhosted.org/packages/f0/a6/c961a911385a4cc4ae10b4dcbcf5dede0407530e59870ce6e4f7bacce47d/swig-4.4.1-py3-none-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7945f153f7fc93f1ab280f44a3740d6bbfeec3829f954784ba83990fc61a11a8", size = 1912119, upload-time = "2025-12-27T03:25:27.661Z" }, - { url = "https://files.pythonhosted.org/packages/86/4b/a08a2673a2fd3185932db110de67a9f6fdfcece6bf9220e18502bf745f7e/swig-4.4.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:5a9c74b87151d8247d802bda33f78cab9f27c759f0605b760a8f9c9893b5606b", size = 1885810, upload-time = "2025-12-27T03:25:29.562Z" }, - { url = "https://files.pythonhosted.org/packages/7e/38/efa343b506221a9d43bb1132f08b29fcb07d4113165f598d8a1ea733a1e4/swig-4.4.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a95ae66aebb8ccce4c50febd496e6927b888d98aba272dd5642fe9312bf61a3b", size = 2950985, upload-time = "2025-12-27T03:25:31.586Z" }, - { url = "https://files.pythonhosted.org/packages/de/0b/5b46e61798982f3cb0ed54fc30a30bad5fa6e3bdd1d9177e439976f7a7e5/swig-4.4.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a8f06227e6472832dad3ae34d5bd88710b7bbb109f2927e14a21e6725146e944", size = 2781782, upload-time = "2025-12-27T03:25:33.534Z" }, - { url = "https://files.pythonhosted.org/packages/17/e6/a15efd9cbf39b1289ee7c1acb8d18824a0c2532b85cdbd93acfdf15ee000/swig-4.4.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:98d61b45215982dd3a71745dfc31599176ed79d286b58bc712c10ed031432fdf", size = 3272993, upload-time = "2025-12-27T03:25:35.862Z" }, - { url = "https://files.pythonhosted.org/packages/d6/fa/516b95f9966c57acf1da9f2eb22e4e63104452d6e28686d57b216171ee7c/swig-4.4.1-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:b6909a8bd68037ea0df061a52c664431b062dbdfb1eb90768719518c120139d1", size = 3189093, upload-time = "2025-12-27T03:25:38.158Z" }, - { url = "https://files.pythonhosted.org/packages/0c/03/88ca92d3d8e9dd765a66985202db23745bd26a368570a8d8db9a218bfe90/swig-4.4.1-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:6a6a6331a5c7759ba5a910aa83aa5d33022b8e691b6cb466635c7fc2238facd2", size = 2967789, upload-time = "2025-12-27T03:25:40.24Z" }, - { url = "https://files.pythonhosted.org/packages/bf/97/5ca3d750f294048c983edfad1f2ad093a83480494fccb2f2e2496304f965/swig-4.4.1-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:2d076358cded172d2f44b3c617923c593dbae664b3fa45051641f028f7f74010", size = 3203253, upload-time = "2025-12-27T03:25:41.953Z" }, - { url = "https://files.pythonhosted.org/packages/97/44/96586955c0f08b0b9b6c8b2b68ecb43a815da94d5362cfb6691ede3b9fdf/swig-4.4.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:43a0a876b4660f6addb6ae72d14a72324dc1388e1c168ed38bb00fd123048749", size = 3060721, upload-time = "2025-12-27T03:25:44.322Z" }, -] - -[[package]] -name = "sympy" -version = "1.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mpmath", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, -] - -[[package]] -name = "tables" -version = "3.11.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "blosc2", marker = "sys_platform == 'linux'" }, - { name = "numexpr", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "py-cpuinfo", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/a3/d213ebe7376d48055bd55a29cd9f99061afa0dcece608f94a5025d797b0a/tables-3.11.1.tar.gz", hash = "sha256:78abcf413091bc7c1e4e8c10fbbb438d1ac0b5a87436c5b972c3e8253871b6fb", size = 4790533, upload-time = "2026-03-01T11:43:36.036Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/a3/ec228901fca4c996306b17f5c60a4105144df0bbd07b3a4a816f91f37b4a/tables-3.11.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56bf6fb9132ead989b7e76695d7613d6d08f071a8019038d6565ba90c66b9f3e", size = 6903733, upload-time = "2026-03-01T11:42:58.349Z" }, - { url = "https://files.pythonhosted.org/packages/99/29/c2dc674ea70fa9a4819417289a9c0d3e4780835beeed573eb66964cfb763/tables-3.11.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e78fe190fdeb4afe430b79651bae2a4f341904eb85aa8dbafe5f1caee1c7f67", size = 7241357, upload-time = "2026-03-01T11:43:03.938Z" }, - { url = "https://files.pythonhosted.org/packages/59/3c/f4a694aa744d2b14d536e172c28dd70c84445f4787083a82d6d44a39e39f/tables-3.11.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a9c35f87fcb6a48c79fbc4e3ab15ca8f6053c4ce13063d6ca2ec36cbb58f40f", size = 7014135, upload-time = "2026-03-01T11:43:22.359Z" }, - { url = "https://files.pythonhosted.org/packages/45/82/94d4320d6c0fe5bd55230eec90cd142d58cda37b7cce00a318ac2a6abd93/tables-3.11.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4cf3218b76ba78d156d6ee75c19fb757d50682f6c7b4905370441afbfc9d77f3", size = 7349293, upload-time = "2026-03-01T11:43:27.569Z" }, -] - -[[package]] -name = "tables-io" -version = "1.1.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click", marker = "sys_platform == 'linux'" }, - { name = "deprecated", marker = "sys_platform == 'linux'" }, - { name = "h5py", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pandas", marker = "sys_platform == 'linux'" }, - { name = "tables", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/12/1b/61bed3c55ce4602be5379362ce939b71a2e707e856ee497c1d1e72f8ff3d/tables_io-1.1.2.tar.gz", hash = "sha256:ea27eac8b22ff4e3500afd92140794bd41161f768a2c84520b91b3b58e945182", size = 91394, upload-time = "2026-06-18T16:21:20.896Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/68/025307ebd3c54c65e540d2a4ced11d0e5aa370c8ddb6d0e83066e5a39054/tables_io-1.1.2-py3-none-any.whl", hash = "sha256:4361e04dfcc10471bf9a3940d9ee6a1f903ac76ece51d32ef7fec7ea572b20c0", size = 39206, upload-time = "2026-06-18T16:21:19.577Z" }, -] - -[[package]] -name = "tabulate" -version = "0.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, -] - -[[package]] -name = "tenacity" -version = "9.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, -] - -[[package]] -name = "terminado" -version = "0.18.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ptyprocess", marker = "os_name != 'nt' and sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8a/11/965c6fd8e5cc254f1fe142d547387da17a8ebfd75a3455f637c663fb38a0/terminado-0.18.1.tar.gz", hash = "sha256:de09f2c4b85de4765f7714688fff57d3e75bad1f909b589fde880460c753fd2e", size = 32701, upload-time = "2024-03-12T14:34:39.026Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl", hash = "sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0", size = 14154, upload-time = "2024-03-12T14:34:36.569Z" }, -] - -[[package]] -name = "threadpoolctl" -version = "3.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, -] - -[[package]] -name = "throttler" -version = "1.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ce/3f/47baf510c31e0e52ac0d80d9071e5e166ca069167fee4a6c13841f9d5f5f/throttler-1.2.3.tar.gz", hash = "sha256:d2f5b0b499d62f1fc984dcac8043450b606549b0097753a9c8a707f7427c27e1", size = 11792, upload-time = "2026-01-27T00:48:11.552Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/70/42d8796acc57c8bcd9ae395b1a6a0bbc833f738492a8ed192a44ccd58035/throttler-1.2.3-py3-none-any.whl", hash = "sha256:241ea3e97438dec4dc2f31ddc56dbd96262787a9b1d0598adfcc0bada1134b66", size = 9704, upload-time = "2026-01-27T00:48:09.544Z" }, -] - -[[package]] -name = "tifffile" -version = "2026.6.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b7/38/5e2ecef5af2f4fd4a89bb8d6240de9458bab4d51a4cbd97aeb3a0cd618e2/tifffile-2026.6.1.tar.gz", hash = "sha256:626c892c0e899d959b9438e7c0e1491dc154a7fead1f1f37a991724a50eceba9", size = 429694, upload-time = "2026-05-31T23:57:12.165Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/59/208f71d70ddc6184f79b8c6d87d46eb7d7b12c19186a817dec9c9c3f3693/tifffile-2026.6.1-py3-none-any.whl", hash = "sha256:0d7382d2769b855b81ce358528e2b40c16d48aa39031746efa81215205332a8d", size = 267108, upload-time = "2026-05-31T23:57:10.597Z" }, -] - -[[package]] -name = "tinycss2" -version = "1.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "webencodings", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a3/ae/2ca4913e5c0f09781d75482874c3a95db9105462a92ddd303c7d285d3df2/tinycss2-1.5.1.tar.gz", hash = "sha256:d339d2b616ba90ccce58da8495a78f46e55d4d25f9fd71dfd526f07e7d53f957", size = 88195, upload-time = "2025-11-23T10:29:10.082Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/60/45/c7b5c3168458db837e8ceab06dc77824e18202679d0463f0e8f002143a97/tinycss2-1.5.1-py3-none-any.whl", hash = "sha256:3415ba0f5839c062696996998176c4a3751d18b7edaaeeb658c9ce21ec150661", size = 28404, upload-time = "2025-11-23T10:29:08.676Z" }, -] - -[[package]] -name = "toolz" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/11/d6/114b492226588d6ff54579d95847662fc69196bdeec318eb45393b24c192/toolz-1.1.0.tar.gz", hash = "sha256:27a5c770d068c110d9ed9323f24f1543e83b2f300a687b7891c1a6d56b697b5b", size = 52613, upload-time = "2025-10-17T04:03:21.661Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/12/5911ae3eeec47800503a238d971e51722ccea5feb8569b735184d5fcdbc0/toolz-1.1.0-py3-none-any.whl", hash = "sha256:15ccc861ac51c53696de0a5d6d4607f99c210739caf987b5d2054f3efed429d8", size = 58093, upload-time = "2025-10-17T04:03:20.435Z" }, -] - -[[package]] -name = "tornado" -version = "6.5.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/64/24/95ec527ad67b76d59299e5465b3935d05e4294b7e0290a3924b7487df30b/tornado-6.5.7.tar.gz", hash = "sha256:66c513a76cda70d53907bc27cf1447557699c2e95aa48ba27a442ff61c3ddfc2", size = 519252, upload-time = "2026-06-08T17:34:51.232Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/d8/ef374952fd5da67d4463122c2b8e5a96536ec10b4b339254c6dcde81d01c/tornado-6.5.7-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8d759e71906ee783f8867b93bf26a265743da4c1e2f4a018464c1ba019862972", size = 449774, upload-time = "2026-06-08T17:34:41.204Z" }, - { url = "https://files.pythonhosted.org/packages/35/37/d434c73f4c6e014b745b9b37085f34f40c022f007efff3d7fe65991899f3/tornado-6.5.7-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a46347a18f23fb92b396beebe0fb78f61dda0cc302445202c16203d8a18848b", size = 450745, upload-time = "2026-06-08T17:34:42.531Z" }, - { url = "https://files.pythonhosted.org/packages/b6/2b/56b9aff361d7f1ab728a805ec7d7ea835f8807afa9f5cc690ea0e630efb9/tornado-6.5.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7778b30bef919231265e91c69963ce0f49a1e9c07ac900bbe75b19ce2575ba92", size = 450578, upload-time = "2026-06-08T17:34:43.787Z" }, - { url = "https://files.pythonhosted.org/packages/02/30/a7444fb23aa76860a14198fab96ac79f1866b0a6e19e26c4381b0938e50f/tornado-6.5.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e726f0c75da7726eec023aa62751ff8878bd2737e34fbdd33b1ae5897d2200f5", size = 449985, upload-time = "2026-06-08T17:34:45.326Z" }, -] - -[[package]] -name = "tqdm" -version = "4.68.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ae/5f/57ff8b434839e70dab45601284ea413e947a63799891b7553e5960a793a8/tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520", size = 792418, upload-time = "2026-07-07T09:58:18.369Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2", size = 676612, upload-time = "2026-07-07T09:58:16.256Z" }, -] - -[[package]] -name = "traitlets" -version = "5.15.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/57/a9/a2584b8313b89f94869ddb3c4074617a691de1812a614d2d50e32ca5a7a6/traitlets-5.15.1.tar.gz", hash = "sha256:7b1c07854fe25acb39e009bae49f11b79ff6cbb2f27999104e9110e7a6b53722", size = 163344, upload-time = "2026-06-03T12:26:06.181Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/96/8d/1080ee4c231f361b6ce4470d556c8c435b67c7e0753aaa641497ee92f88b/traitlets-5.15.1-py3-none-any.whl", hash = "sha256:770a53705f84b81ac107e83a1b3328ff2dae16094d8fc3cfc004e4b22dfd8e92", size = 85858, upload-time = "2026-06-03T12:26:04.395Z" }, -] - -[[package]] -name = "transformcl" -version = "2026.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "array-api-compat", marker = "sys_platform == 'linux'" }, - { name = "flt", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9d/ce/ae8373b67259c3ea6033425161f7d195387b1e68b17d4d00eee1b68e5edd/transformcl-2026.1.tar.gz", hash = "sha256:14551c33d164f3db24b9141f1f77553182a4b9c789f883476eb33d87ea4d8f13", size = 4850, upload-time = "2026-03-02T19:18:21.221Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/98/de7a05005238e03bdc57cd2885c27ad9140539bbaddf2b6ad2b2a0aa9863/transformcl-2026.1-py3-none-any.whl", hash = "sha256:fe9c380074ef783a4f854e0d326eadac677d6a35bc683e3b429d5b49a8e57223", size = 3597, upload-time = "2026-03-02T19:18:19.832Z" }, -] - -[[package]] -name = "treecorr" -version = "5.1.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lsstdesc-coord", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pybind11", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "setuptools", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5d/95/ff399cb37b9e3d7c758f8cf0d5757f06a38093f223f2a65d7cc5b50313d5/treecorr-5.1.3.tar.gz", hash = "sha256:d921ccace064325a8ebb93f371c624f13fcc9559373e3b37cc216cb94ae81ed2", size = 1146147, upload-time = "2026-03-18T18:38:03.626Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/74/1cc590e4fe1afb69ec2f10a67ee8e0a92ecafd36512052755362ff302d9c/treecorr-5.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:596b6f5eea3115a9e211dc83ec6465887983f41b24e484e08fc47170c6cf4e26", size = 20941767, upload-time = "2026-03-18T18:37:37.874Z" }, - { url = "https://files.pythonhosted.org/packages/6f/bb/5ba5aa1021fce0cf459fa264cb0d298dfd095ddb48d63fb6589b90fdefac/treecorr-5.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:660dacd47a9af18790de279bcb6e7f375369096ad77e7cc64bf21a9b78f6c430", size = 19785388, upload-time = "2026-03-18T18:37:40.573Z" }, - { url = "https://files.pythonhosted.org/packages/be/92/be01211263a4fa488e768c9288c6f9b0f4402870423a88de4d60031b3384/treecorr-5.1.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73c938c6c31e1bad4d5338ad4519d1d78eedf3b7ef579b1e3a19d983e220ed93", size = 20940568, upload-time = "2026-03-18T18:37:44.966Z" }, - { url = "https://files.pythonhosted.org/packages/dc/a3/69e5a16aa33600ba22d28a47c6d8de3b3118292e79f9ae77dc69a3df7c77/treecorr-5.1.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc17b1f623bae6769148f57cac8c3a295e26e5d6215698117bd97e14c8bb0d6b", size = 19781991, upload-time = "2026-03-18T18:37:47.468Z" }, -] - -[[package]] -name = "typeshed-client" -version = "2.12.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "importlib-resources", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/54/5d/97d6fa8c204b0d42be931e7a568b306cda43f93ff45f8b47537ee61390de/typeshed_client-2.12.0.tar.gz", hash = "sha256:54dcfa25fcff82cedcb1b51c03c1b3c51a614097aab8fe49e4316aff6f79d9c7", size = 529457, upload-time = "2026-06-02T04:00:51.51Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/85/98/3aef1587ec9b3a5cd062b8d8910545a0ee3715c31fd65b1f2386b27b94af/typeshed_client-2.12.0-py3-none-any.whl", hash = "sha256:da3ef54a0e60c0f45f59006b73a3cb20b1acbf7a784e982476d8193e8828ddf5", size = 796929, upload-time = "2026-06-02T04:00:49.643Z" }, -] - -[[package]] -name = "typing-extensions" -version = "4.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, -] - -[[package]] -name = "typing-inspection" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, -] - -[[package]] -name = "tzdata" -version = "2026.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, -] - -[[package]] -name = "uncertainties" -version = "3.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fe/0c/cb09f33b26955399c675ab378e4063ed7e48422d3d49f96219ab0be5eba9/uncertainties-3.2.3.tar.gz", hash = "sha256:76a5653e686f617a42922d546a239e9efce72e6b35411b7750a1d12dcba03031", size = 160492, upload-time = "2025-04-21T19:58:28.63Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/5e/f1e1dd319e35e962a4e00b33150a8868b6329cc1d19fd533436ba5488f09/uncertainties-3.2.3-py3-none-any.whl", hash = "sha256:313353900d8f88b283c9bad81e7d2b2d3d4bcc330cbace35403faaed7e78890a", size = 60118, upload-time = "2025-04-21T19:58:26.864Z" }, -] - -[[package]] -name = "uri-template" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/c7/0336f2bd0bcbada6ccef7aaa25e443c118a704f828a0620c6fa0207c1b64/uri-template-1.3.0.tar.gz", hash = "sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7", size = 21678, upload-time = "2023-06-21T01:49:05.374Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl", hash = "sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363", size = 11140, upload-time = "2023-06-21T01:49:03.467Z" }, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, -] - -[[package]] -name = "vos" -version = "3.6.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aenum", marker = "sys_platform == 'linux'" }, - { name = "cadcutils", marker = "sys_platform == 'linux'" }, - { name = "html2text", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3e/6c/445bbedf2f90e71c09ff890aa8b409af5d074b1578eef383117cbcdfd09a/vos-3.6.4.tar.gz", hash = "sha256:67e4e24689ee881124b6670d7d2fc6527cc404687fc527fffc20cf5b1bd4c0bd", size = 103445, upload-time = "2025-12-12T00:20:56.398Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/21/e2d351883aa11b9175223a78d3a77b0fedf86461596cdbfb019893ca9260/vos-3.6.4-py3-none-any.whl", hash = "sha256:4a7b2248bad0f50da465a47ab1c819dff8378c189c184c2d8e2c17a714c62f17", size = 153404, upload-time = "2025-12-12T00:20:53.232Z" }, -] - -[[package]] -name = "wcwidth" -version = "0.8.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/34/74/c6428f875774288bec1396f5bfcbc2d925700a4dad61727fd5f2b12f249d/wcwidth-0.8.2.tar.gz", hash = "sha256:91fbef97204b96a3d4d421609b80340b760cf33e26da123ff243d76b1fda8dda", size = 1466253, upload-time = "2026-06-29T18:11:11.601Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl", hash = "sha256:d63947694a0539a1d51e01eda7caf800c291020e6cdd7e28ad7b14dd33ad4f85", size = 323166, upload-time = "2026-06-29T18:11:09.888Z" }, -] - -[[package]] -name = "webcolors" -version = "25.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/7a/eb316761ec35664ea5174709a68bbd3389de60d4a1ebab8808bfc264ed67/webcolors-25.10.0.tar.gz", hash = "sha256:62abae86504f66d0f6364c2a8520de4a0c47b80c03fc3a5f1815fedbef7c19bf", size = 53491, upload-time = "2025-10-31T07:51:03.977Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl", hash = "sha256:032c727334856fc0b968f63daa252a1ac93d33db2f5267756623c210e57a4f1d", size = 14905, upload-time = "2025-10-31T07:51:01.778Z" }, -] - -[[package]] -name = "webencodings" -version = "0.5.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47/webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923", size = 9721, upload-time = "2017-04-05T20:21:34.189Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", size = 11774, upload-time = "2017-04-05T20:21:32.581Z" }, -] - -[[package]] -name = "websocket-client" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, -] - -[[package]] -name = "widgetsnbextension" -version = "4.0.15" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/f4/c67440c7fb409a71b7404b7aefcd7569a9c0d6bd071299bf4198ae7a5d95/widgetsnbextension-4.0.15.tar.gz", hash = "sha256:de8610639996f1567952d763a5a41af8af37f2575a41f9852a38f947eb82a3b9", size = 1097402, upload-time = "2025-11-01T21:15:55.178Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl", hash = "sha256:8156704e4346a571d9ce73b84bee86a29906c9abfd7223b7228a28899ccf3366", size = 2196503, upload-time = "2025-11-01T21:15:53.565Z" }, -] - -[[package]] -name = "wrapt" -version = "2.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fe/a4/282c8e64300a59fc834518a54bf0afabb4ff9218b5fa76958b450459a844/wrapt-2.2.2.tar.gz", hash = "sha256:0788e321027c999bf221b667bd4a54aaefd1a36283749a860ac3eb77daed0302", size = 129068, upload-time = "2026-06-20T23:49:44.49Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/b3/7e6e9fcf4fe7e1b69a49fe6cc5a44e8224bab6283c5233c97e132f14908e/wrapt-2.2.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e096c9d39a59b35b63c9aacfbbbec2088ff51ff1fc31051acc60a07f42f273a", size = 169828, upload-time = "2026-06-20T23:48:02.719Z" }, - { url = "https://files.pythonhosted.org/packages/0b/43/894f132d857ed5a9904d937baf368badcbe5ea9e436e2f1930fe21c9f1f0/wrapt-2.2.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d1a6050405bf334be33bf66296f113563622972a34900ae6fa60fd283a1a900", size = 171544, upload-time = "2026-06-20T23:48:04.266Z" }, - { url = "https://files.pythonhosted.org/packages/29/de/3c833e03725b477e9ea34028224dd21a48781830101e4e036f77e8b6b102/wrapt-2.2.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10adb01371408c6de504a6658b9886480f1a4919a83752748a387a504a21df79", size = 160663, upload-time = "2026-06-20T23:48:05.708Z" }, - { url = "https://files.pythonhosted.org/packages/33/be/27edce350b24e3054d9d047f65f16d4c4d4c1f3f31c4278a1f8a95c723c8/wrapt-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3442eee2a5798f9b451f1b2cd7518ce8b7e28a2a364696c414460a0e295c012a", size = 169387, upload-time = "2026-06-20T23:48:07.243Z" }, - { url = "https://files.pythonhosted.org/packages/e2/c4/9fd9679af8bf38e146652c7f47b6b352c3e5795b4ad1c0b7f94e15ac2aa7/wrapt-2.2.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6c99012a22f735a85eed7c4b86a3e99c30fdd57d9e115b2b45f796264b58d0bf", size = 158849, upload-time = "2026-06-20T23:48:08.91Z" }, - { url = "https://files.pythonhosted.org/packages/bc/c2/aa6c0c2206803068c6859dabe01f8c84c43744da93d4c67b8946d21655ee/wrapt-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b686cfc008776a3952d6213cb296ed7f45d782a8453936406faa89eac0835ab", size = 168147, upload-time = "2026-06-20T23:48:10.374Z" }, - { url = "https://files.pythonhosted.org/packages/15/ff/63ad96f98eb58a742b1a20d80f21da88924405910149950b912368150468/wrapt-2.2.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb18fc51e813df0d9c98049e3bf2298a5495a648602040e21fa3c7329371159e", size = 167882, upload-time = "2026-06-20T23:48:18.764Z" }, - { url = "https://files.pythonhosted.org/packages/20/1f/8bb62d8933df7acf3247194e6e9fc68edf9d2fa203252c89c94b319dd472/wrapt-2.2.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94b00b00f806eb3ef2abe9049ed45994a81ee9284884d96e6b8314927c6cea3d", size = 167411, upload-time = "2026-06-20T23:48:20.315Z" }, - { url = "https://files.pythonhosted.org/packages/17/09/8789dcb09ee1de715727db7521aabbb68ffa68dfade3a49468440cfced49/wrapt-2.2.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:62415fd095bc590b842b6d092f2b5d9ccbaeb7e0b28535c03dcea2718b48636b", size = 158607, upload-time = "2026-06-20T23:48:21.728Z" }, - { url = "https://files.pythonhosted.org/packages/9c/20/66e02562d53ee67d841f175e38e3c993c2d78a3e104c576cad61c028b43c/wrapt-2.2.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a41e758d80dc0ab8c210f641ac892009d356cf1f955d97db544c8dd317b4d14c", size = 166367, upload-time = "2026-06-20T23:48:23.177Z" }, - { url = "https://files.pythonhosted.org/packages/bd/a3/832ac4e41222fb263b3042d42c2f08d305db7d0f0c9b1d3a271a9eede8f6/wrapt-2.2.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b84cd4058001c9727b0e9980b7a9e66325b5ca748b1b578e822cade1bc6b304f", size = 157176, upload-time = "2026-06-20T23:48:24.711Z" }, - { url = "https://files.pythonhosted.org/packages/b7/01/1bd5e4d2df9c0178989ac8da9186543465388588ee2ef153e2591accebef/wrapt-2.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26fc73a1b15e0946d2942b9a4426d162b51676338327dc067ccd8d2d76385f94", size = 167025, upload-time = "2026-06-20T23:48:26.118Z" }, - { url = "https://files.pythonhosted.org/packages/18/1e/f060df47755e87b57684cee7bfc1362b204df55fac96ffebc0631b697b79/wrapt-2.2.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0065a3b657cec06813b4241d2462ccec287f6863103d7445b725fb3a889736f9", size = 203399, upload-time = "2026-06-20T23:48:34.97Z" }, - { url = "https://files.pythonhosted.org/packages/c4/de/2316a757a1abb6453700b79d83e532146dcef2611348282d4d8889792161/wrapt-2.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30f7424af5c5c345b7f26490e097f74a2ef45b3d08b664dc33571aee3bd3b56c", size = 210461, upload-time = "2026-06-20T23:48:36.569Z" }, - { url = "https://files.pythonhosted.org/packages/ed/29/d1160785ae18ca2495a6d82a21154103d74f656c9fd457fb35f6b11b965a/wrapt-2.2.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:07fdcb012821859168641acf68afad61ef9783cf37100af85f152550e9677194", size = 195313, upload-time = "2026-06-20T23:48:38.175Z" }, - { url = "https://files.pythonhosted.org/packages/f5/2d/7caa9598ae61a9cf0989cc501739cbeeb7d650ab3193cca1407b9af0c6ab/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f90038ab58fafb584801ca62d72384d7d5225d93c76f7b773c22fae545bd8066", size = 206116, upload-time = "2026-06-20T23:48:39.804Z" }, - { url = "https://files.pythonhosted.org/packages/ac/02/281ea1088b8650d865f311b35cf86fd21df89128e2909714f1161e01c9d0/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:c5d7825491bfa2d08b97e9557768987952c7b9ae687d06c3320b40a37ccb7f20", size = 192668, upload-time = "2026-06-20T23:48:41.346Z" }, - { url = "https://files.pythonhosted.org/packages/be/7d/976e2d5b4b5c5babda40974edd54d0a5585cb60132ed86b46f4b80239b16/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ad520e6daa9bbf136f14de735474dbec7dcc0891f718e1d274ce8dc92e645af", size = 198891, upload-time = "2026-06-20T23:48:43.056Z" }, - { url = "https://files.pythonhosted.org/packages/74/eb/df7b7f0b631dbbc750f39be27d8b55f65777d8ac86da80e12be41a644c4b/wrapt-2.2.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0abfd648dac9ac9c5b3aa9b523d27f1789046640b58dcd5652a720ddb325e1fc", size = 167713, upload-time = "2026-06-20T23:48:52.598Z" }, - { url = "https://files.pythonhosted.org/packages/4d/9a/d1bd36f6d088c8e652a9383cabbd49af30b8c576302a7eccddbab6963e3f/wrapt-2.2.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f4bfd8d1eb438153eff8b8cfe87f032ba65731e1ce06138b5090f745a33f6f95", size = 166779, upload-time = "2026-06-20T23:48:54.33Z" }, - { url = "https://files.pythonhosted.org/packages/4c/ae/24ffacd4187fac2740a1972093929e836dea092d42c87d728cd98fee11a6/wrapt-2.2.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c427c9d06d859848a69f0d928fe28b5c33a941b2265d10a0e1f15cd244f1ee33", size = 158407, upload-time = "2026-06-20T23:48:55.944Z" }, - { url = "https://files.pythonhosted.org/packages/a3/ed/974427668249a356051e8d67d47fa54ef6c777f0fcf3bae9d292c047d4b6/wrapt-2.2.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4250b43d1a129d947e083c4dc6baf333c9bb34edd26f912d5b0457841fc858ab", size = 166594, upload-time = "2026-06-20T23:48:57.617Z" }, - { url = "https://files.pythonhosted.org/packages/fb/5f/e1d7c6e4523f78db2fbd7826babd0348da1d5e0834c4f918b9ab5757dfae/wrapt-2.2.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:173e5bb5ca350a6e0abab60b7ec7cdd7992a814cb14b4de670a28f067f105663", size = 157068, upload-time = "2026-06-20T23:48:59.171Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c1/7ebd1027f00700c0b0233b20aceef2b4784294ed64971424c4a78e069e34/wrapt-2.2.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aa14b01804bce36c6d63d7b6a4f55df390f29f8648cc13a1f40b166f4d54680d", size = 166470, upload-time = "2026-06-20T23:49:00.737Z" }, - { url = "https://files.pythonhosted.org/packages/4e/ea/1a89e6d3b7a83c3affe5c09cde77792c947e63e4bc85ad84cd5bb9abb0d8/wrapt-2.2.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:45c2f2768e790c9f8db90f239ef23a2af8e7570f25a35619ef902df4a738447f", size = 203362, upload-time = "2026-06-20T23:49:09.811Z" }, - { url = "https://files.pythonhosted.org/packages/19/d8/3b58763d9863b5a73771c0d97110f9595d248db454009e07e1535ee905a4/wrapt-2.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbf00ee0cb55ec24e2b0995a71942b85b21a066db8f3f46e1dbfdb9433ffba81", size = 210449, upload-time = "2026-06-20T23:49:11.521Z" }, - { url = "https://files.pythonhosted.org/packages/2d/6f/17fd9e053103d8be148d20d5d7505facc72d5fe1f9127973904ceaed79cf/wrapt-2.2.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2252f77663651b89255895f58cc6ac08fcb206d4371813e5af61bb62d4f7689c", size = 195349, upload-time = "2026-06-20T23:49:13.346Z" }, - { url = "https://files.pythonhosted.org/packages/ef/04/d0d1ccaaa12cb7dccf28a23f0279a608ba498f71e81d949d5ed54bcfd5c1/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cd7181ab1c31192ff5219269830744b5a62020b3a6d433588c4f1c95b8f8bff", size = 206099, upload-time = "2026-06-20T23:49:15.051Z" }, - { url = "https://files.pythonhosted.org/packages/44/b3/e8aa07b619890a2aa6cde1931b1887abb08820721b564a5f80b7ca3f3aa0/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:6fe35fd51b74867d8b80174c277bd6bbf6a73e443f908129dc531c4b688a20d5", size = 192728, upload-time = "2026-06-20T23:49:16.854Z" }, - { url = "https://files.pythonhosted.org/packages/b7/f0/1819fb50f0d3c9bd758d8a83b56f1b470dee8b5b8eac8702b7c137cea9d4/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:11d95fc2fbad3163596c39d440e6f21ca9fccece74b56e30a37ac2fca786a07c", size = 198842, upload-time = "2026-06-20T23:49:18.504Z" }, - { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, -] - -[[package]] -name = "yte" -version = "1.9.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argparse-dataclass", marker = "sys_platform == 'linux'" }, - { name = "dpath", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/44/f5/7e44620e6e077bfe624b9a17c329b8e0d0159e176e1f1a93c2790428ab2c/yte-1.9.4.tar.gz", hash = "sha256:86a47e6d722cec9419a7ac88be57d0d6c4ce28f02860393b71a66f2c674069f6", size = 8101, upload-time = "2025-11-27T12:55:00.85Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/63/6a44729fdc60eb255a7b156a84e7552290174a9bf151e3b6c18e83d6fbfa/yte-1.9.4-py3-none-any.whl", hash = "sha256:5dac63303d3e6bc2ebadc36ece3c3fb09343772fe6e25e9356d9baf8f9dfaf6d", size = 10618, upload-time = "2025-11-27T12:55:01.685Z" }, -] - -[[package]] -name = "zarr" -version = "3.2.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "donfig", marker = "sys_platform == 'linux'" }, - { name = "google-crc32c", marker = "sys_platform == 'linux'" }, - { name = "numcodecs", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/93/8d/aeb164004f87543b06ef54f885d02c342c31ceb274e2bbec470a98927621/zarr-3.2.1.tar.gz", hash = "sha256:71565b738a0e7e8ed226f0516eba8c6bb53440ad7669a8c48ebb3534a161d035", size = 675161, upload-time = "2026-05-05T12:37:22.383Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/0a/469e2bd01be1490336e6c8707386845655d59261543315778a3ccc7e8019/zarr-3.2.1-py3-none-any.whl", hash = "sha256:f78cdd3d9687ad0e9f9cba2c5683b64f0c52589c19f685eeabe872e93cc0d2c7", size = 319617, upload-time = "2026-05-05T12:37:20.66Z" }, -] - -[[package]] -name = "zipp" -version = "4.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, -] - -[[package]] -name = "zope-interface" -version = "8.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/08/dc/50550cfcbb2ea3cbca5f1d7ed05c8aa840f831a0f2d63aec0a953f7c590e/zope_interface-8.5.tar.gz", hash = "sha256:7a3ba1c5877f0f3e3906b02ddf793abed2becc2948116414ce0e1dd820b68d6d", size = 257957, upload-time = "2026-05-26T06:50:14.574Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/c3/005032195ff3b210c139b7c560ed5c534e844b0907d8e44d2b3d8919305e/zope_interface-8.5-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:e6347b8d8d12c5eca6502450a92be30079b7acfade2c4f693efa0deb8871b06e", size = 265296, upload-time = "2026-05-26T06:49:29.741Z" }, - { url = "https://files.pythonhosted.org/packages/c5/66/1036543d6a66bc04c19df3cf650f3ad938a002ab0a443c24e23e8de5e8b9/zope_interface-8.5-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5e970dabea777a24b0b0bbf9dae3ab75ce8b2d8e948edf4875627034b21f3560", size = 270689, upload-time = "2026-05-26T06:49:31.767Z" }, - { url = "https://files.pythonhosted.org/packages/30/4c/8b56259558cace4414e753ca6740396a1f59d4a95ddb55b4658600408670/zope_interface-8.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f0b48ccadaa9839e09ff81e969703cecb3f402c813bfe8b958652e699bea69f5", size = 270280, upload-time = "2026-05-26T06:49:33.489Z" }, - { url = "https://files.pythonhosted.org/packages/aa/a5/692a2b8d70f78e848793231d5fae5fecbf8d0cccd73430fdc34802a6d3c1/zope_interface-8.5-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:7849ad8fa90763cc1087f4dda78ca3a233e950b3e08fac7079297c9cafbbd7bb", size = 265191, upload-time = "2026-05-26T06:49:43.449Z" }, - { url = "https://files.pythonhosted.org/packages/70/8d/454a9cfc7a050c394ab4f11b3371f7897828b7415e096afff724637e65e0/zope_interface-8.5-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5578c9421ca409a1f39f153d6f7803e4cde01da592ec75a9ac5e1b777d18d33b", size = 270626, upload-time = "2026-05-26T06:49:45.425Z" }, - { url = "https://files.pythonhosted.org/packages/51/8c/db8409cfa3575b8e9b4800babd7d49f8228433cd1f0c56814bd0ada49c33/zope_interface-8.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e1bd7d96b4ca5fa311f54c9eac16dce4886b428c1531dbe06067763ccdf123b4", size = 270444, upload-time = "2026-05-26T06:49:47.025Z" }, - { url = "https://files.pythonhosted.org/packages/32/67/bc8b4f465d388039255003e230c284a175cedf1203c692f23cb7bff64efe/zope_interface-8.5-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:3090e3a663d20194756a59a272e0c8508b889341e31d5894223331fe6b4f9b21", size = 266827, upload-time = "2026-05-26T06:49:54.873Z" }, - { url = "https://files.pythonhosted.org/packages/a7/eb/37d05b935ede53d79690fecc8d201440084418e590bcfc05f384451c7593/zope_interface-8.5-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9342fb74e2afefdb081bf1df727d209ea56995c6e13f5a0540e6d7aff4beafb8", size = 270139, upload-time = "2026-05-26T06:49:57.116Z" }, - { url = "https://files.pythonhosted.org/packages/8b/0b/fd0c54579e2ce8dc6cf1a757903f3374bc6fbda929a46af9e0f53cb0e5f0/zope_interface-8.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6c54725d818f1b57a7efb8b16528326e1f3c257b602b32393fd255c45af8799d", size = 270338, upload-time = "2026-05-26T06:49:58.698Z" }, - { url = "https://files.pythonhosted.org/packages/c2/95/d97430abd5ae9677e8b9295b58720c0064a5b557dbb6b8bf5928484cf0d8/zope_interface-8.5-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:14b0e9799351d4c34fe99afd67f0cdd76e55ba15c66a98699d5fc22ea8241e08", size = 294905, upload-time = "2026-05-26T06:50:07.384Z" }, - { url = "https://files.pythonhosted.org/packages/41/ec/a0f8f3dad6e74992f4654bdd94802be0929eabca7b871cac3b6fbb5e961b/zope_interface-8.5-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cd6a732ac84b94eb1ef9222a117347a27efd294ee16810ffdf7ecd307677ed5", size = 300885, upload-time = "2026-05-26T06:50:08.997Z" }, - { url = "https://files.pythonhosted.org/packages/0f/da/6881b48803a0ee8d23eb5efa30fce3ed218a2bd9de5758ce489d224fee81/zope_interface-8.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:798b7c87d0e59a7d5d086d642208d0d8700ff0d55c4029134b3c479c3bfb110f", size = 304672, upload-time = "2026-05-26T06:50:10.563Z" }, -] diff --git a/workflow/README.md b/workflow/README.md index f20ffe8f..47015158 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -24,36 +24,3 @@ What does *not* belong here: Runs stay modular, not monolithic: a paper or run composes these rules with Snakemake's `module` directive under its own config and an output `prefix`, so each namespaces cleanly under `results//`. - -## Running on the cluster — the candide profile - -`profiles/candide/config.yaml` is the committed SLURM profile: it hands -Snakemake the candide executor, account, partition, node excludes, and per-job -floor, so scheduling is repo state rather than an operator's shell. Drive any -target with one command: - -```bash -snakemake --profile workflow/profiles/candide \ - -s workflow/image_sims/Snakefile \ - --configfile -``` - -For example, the image-sim m-bias chain end to end (`im_mbias` fans out one -SLURM job per branch × tile, MPI-free): - -```bash -snakemake --profile workflow/profiles/candide \ - -s workflow/image_sims/Snakefile \ - im_mbias --configfile my_run.yaml -``` - -Give the target *before* `--configfile`: `--configfile` takes one-or-more -paths, so a target after it is read as a config file ("No such file: -im_mbias"). Always dry-run first with `-n`. - -The profile carries only cluster policy — no container settings (the image-sims -rules own their `apptainer exec` call) and no `OMP_NUM_THREADS` (pinned to 1 at -that same `apptainer exec` line, since the slurm executor's `--export=ALL` -propagates the driver's env, not a profile flag). Per-rule `mem_mb` / `runtime` -stay on the rules. Off-cluster, drop `--profile` and add `-j N`. See the -profile's own comments for the full rationale. diff --git a/workflow/Snakefile b/workflow/Snakefile index 7d91db81..6f59827b 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -47,8 +47,3 @@ include: "rules/glass_mock.smk" # guarded so paper configs without it (e.g. bmodes) don't trip on the lookups. if "cosmo_val" in config: include: "rules/cosmo_val.smk" - -# Image-simulation m/c-bias chain (image_sims.smk). Active only when the config -# carries an `image_sims` block; standalone runs use workflow/image_sims/Snakefile. -if "image_sims" in config: - include: "rules/image_sims.smk" diff --git a/workflow/common.py b/workflow/common.py index 3df3413b..17e60ce7 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -5,6 +5,16 @@ import re from pathlib import Path +# Absolute path to the generic workflow's scripts, anchored on this module's own +# location (common.py lives in workflow/, is `from common import *`'d into every +# Snakefile, and so resolves to the generic workflow dir of the running checkout +# regardless of which paper composes it — unlike workflow.basedir, which under +# `module` composition reflects the composing paper). Rules that shell out to a +# script directly (the MPI xi_highres run can't go through Snakemake's `script:` +# directive) interpolate this instead of a hardcoded pure_eb/ compat-symlink +# path. /automnt/n17data is the automount of the container-bound /n17data. +WORKFLOW_SCRIPTS = os.path.join(os.path.dirname(os.path.realpath(__file__)), "scripts") + # Output roots are env-overridable so a reproduction run can write into a # fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL = Path( diff --git a/workflow/image_sims/Snakefile b/workflow/image_sims/Snakefile deleted file mode 100644 index d7a89065..00000000 --- a/workflow/image_sims/Snakefile +++ /dev/null @@ -1,46 +0,0 @@ -"""Standalone entry point for the image-simulation m-bias workflow. - -Run the sp_validation-side chain (merge -> extract -> calibrate -> m-bias), -optionally including the ShapePipe pipeline stage, without pulling in the -cosmology-validation config the top-level ``workflow/Snakefile`` requires. - -Layer a run config over the operational defaults -- the workflow config.yaml -carries operational defaults but *no* science keys, so it is incomplete on its -own (by design); the run config supplies the science knobs. The one drive -command on candide, with the committed SLURM profile owning all scheduling: - - snakemake --profile workflow/profiles/candide \\ - -s workflow/image_sims/Snakefile \\ - im_mbias --configfile my_run.yaml - -``configfile: "workflow/image_sims/config.yaml"`` below loads the operational -defaults automatically, so only ``my_run.yaml`` (the science knobs, and any -operational override that run wants) is passed on the command line; Snakemake -deep-merges the two. The profile supplies the executor, account, partition, -node excludes and job floor -- no ``-j`` needed (the slurm executor sets the -job cap). Off-cluster, drop ``--profile`` and add ``-j N`` to run locally. - -The target (``im_mbias``) is given *before* ``--configfile``: Snakemake's -``--configfile`` takes one-or-more paths, so a target placed after it is -swallowed as a config path ("No such file: im_mbias"). Put targets ahead of -``--configfile`` (or make ``--configfile`` the last flag on the line). Always -dry-run first with ``-n``. - -The same rules are also available inside the main workflow: they are included -there under ``if "image_sims" in config``. -""" - -configfile: "workflow/image_sims/config.yaml" - - -# The image-sims rules own their container invocation explicitly, so no -# top-level container is needed here. -container: None - - -include: "../rules/image_sims.smk" - - -rule all: - input: - f"{GRIDS_BASE}/results/m_bias_results.yaml", diff --git a/workflow/image_sims/config.yaml b/workflow/image_sims/config.yaml deleted file mode 100644 index 8f719c68..00000000 --- a/workflow/image_sims/config.yaml +++ /dev/null @@ -1,89 +0,0 @@ -# Image-simulation m-bias workflow configuration. -# -# Two kinds of keys live under `image_sims:`, and the split is the point: -# -# * OPERATIONAL keys default here (active lines below) and *nowhere else* -- -# the .smk reads them bare, so this file is their single home. Override in -# a run config only when a run genuinely differs from the shared setup. -# -# * SCIENCE keys have NO default -- not here, not in code. They fix the -# estimator's scientific behaviour and must be stated per run, so they -# appear below only as commented template lines. Supply them in a run -# config layered on top: -# -# snakemake -s workflow/image_sims/Snakefile \ -# --configfile workflow/image_sims/config.yaml \ -# --configfile my_run.yaml \ -# -j 4 im_mbias -# -# A run config that omits a science key fails at DAG parse, naming the key; an -# unknown key under `image_sims:` fails as a typo. The structural keys below -# (sif, repos, data roots, num, tile_ids) also have no default and must be set. - -image_sims: - - # --- containers ------------------------------------------------------- - # Two images, one per half of the chain (the split gate766 ran). One image - # is the eventual target -- the sp_validation image is FROM the ShapePipe - # image -- but until sp_validation is uv-locked with cosmo_numba declared, - # its published image can drift NumPy past numba's window (seen 2026-07-11: - # "Numba needs NumPy 2.4 or less. Got NumPy 2.5" at ngmix). PYTHONPATH - # shadows pure-Python code only, never binary deps. - sif: /n17data/cdaley/containers/sp_validation_im_sims.sif # extract/calibrate/m-bias - sif_pipeline: /n17data/cdaley/containers/shapepipe_im_sims-runtime.sif # pipeline/merge - # Apptainer bind mounts. /automnt is required when repos/data are - # automounted (candide gotcha); harmless otherwise. [operational] - binds: /n17data,/n09data,/home,/automnt - - # --- repositories ----------------------------------------------------- - # Bound into the image; both repos' src go on PYTHONPATH so this branch's - # code wins over the baked copies: ShapePipe's #766 build, and sp_validation's - # image_sims.py / catalog.match_catalogs_radec. - shapepipe_repo: /n17data/cdaley/unions/code/shapepipe - sp_validation_repo: /n17data/cdaley/unions/code/sp_validation - - # --- data and run directories ---------------------------------------- - # grids_base is the run/output root: one sub-directory per simulation. - grids_base: /n17data/cdaley/unions/scratch-wf/imsims-run/grids - input_sims_base: /n09data/hervas/skills_out - psf_dict: /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle - - # --- simulation grid -------------------------------------------------- - sims_type: grid # 'grid' -> *_grid_{num}; anything else -> *_{num} [operational] - num: 1 - # Branches this run requests: the unsheared reference plus the four +/- - # sheared branches. Injected shear is NOT set here -- it is parsed from each - # branch's basic_info.txt by im_manifest. [operational] - branches: ["1z2z", "1p2z", "1m2z", "1z2p", "1z2m"] - # Tiles to process: an explicit list, the one tile-input mechanism. - tile_ids: ["233.293", "237.292", "238.292"] - - # --- calibration ------------------------------------------------------ - shape: ngmix # [operational] - # ShapePipe cfis configs (final_cat.param etc.); default is - # {shapepipe_repo}/example/cfis_image_sims once #766 lands. [operational] - config_dir: /n17data/cdaley/unions/scratch-wf/imsims-run/grids/_cfis_image_sims - psf_model: psfex # [operational] - n_smp: -1 # [operational] - # Extract/calibrate scripts run from the sp_validation repo checkout (branch - # code, not the baked copies). Point elsewhere for a different checkout. - # [operational] - extract_script: /n17data/cdaley/unions/code/sp_validation/scripts/calibration/extract_info.py - calibrate_script: /n17data/cdaley/unions/code/sp_validation/scripts/calibration/calibrate_comprehensive_cat.py - - # --- science knobs (REQUIRED in the run config; no default) ----------- - # Copy these into your run config and set them. There is deliberately no - # default: each fixes the estimator's scientific behaviour, so a run must - # state it. The injected |g| is separate -- parsed from basic_info.txt by - # im_manifest, never set here. - # - # mask_config: config/calibration/mask_v1.X.9_im_sim.yaml # relative to sp_validation_repo - # match_radius_deg: 0.0002 # RA/Dec pair-match radius, degrees - # w_cols: [none, w_iv] # weight schemes to compute in one run; "none" - # # (or null) = unit weights (#227); first entry is - # # the primary/headline result -- lead with "none" - # # for the fiducial unweighted estimator. The - # # deprecated scalar `w_col` is still accepted. - # pair_match: true # match +g/-g object-by-object (per-object cancellation) - # n_bootstrap: 500 # bootstrap resamples for the errors - # bootstrap_seed: 42 # seed for the bootstrap RNG (makes errors reproducible) diff --git a/workflow/image_sims/params_im_sim.py b/workflow/image_sims/params_im_sim.py deleted file mode 100644 index e87124d0..00000000 --- a/workflow/image_sims/params_im_sim.py +++ /dev/null @@ -1,228 +0,0 @@ -""" - -:Name: params.py - -:Description: This script contains parameters to run the validation notebook. - -:Author: Martin Kilbinger - -:Date: 2021 - -:Package: sp_validation - -""" - -import os - -import numpy as np - -# Control - -## Verbose output -verbose = True - -## Math output -np.set_printoptions(precision=3, formatter={"float": "{: .3g}".format}) - - -# Survey parameters - -## Field or patch name -- derived from the run directory, which is named -## after the simulation (e.g. '1z2z_grid_1'), so one shared params file -## serves every sim. -name = os.path.basename(os.getcwd()) -print("Field name = {}".format(name)) - -## Area of a tile in deg^2 -area_tile = 0.25 - -## Pixel size in arcsec -pixel_size = 0.187 - -## Shape measurement method, implemented is -## 'ngix': multi-epoch model fitting -## 'galsim': stacked-image moments (experimental) -shape = "ngmix" - -# Paths - -## Input paths - -### Input data directory -data_dir = "." - -### Tile IDs -path_tile_ID = f"{data_dir}/tiles_{name}.txt" - -### Weak-lensing galaxy catalog name -galaxy_cat_path = f"{data_dir}/final_cat_{name}.hdf5" -print(f"Galaxy catalogue = {galaxy_cat_path}") - -## Parameter list; optional, set to `None` if not required -param_list_path = f"{data_dir}/cfis/final_cat.param" - -### Star and PSF catalog name; optional, set to `None` if not required -star_cat_path = None - -# HDU number of star and PSF catalogue -hdu_star_cat = 1 - -### External mask; optional, set to `None` if not required -mask_external_path = None - -## Output paths - -### Output base directory -output_dir = f"{data_dir}" - -### Galaxy shape catalogue base name -output_shape_cat_base = f"{output_dir}/shape_catalog" - -### PSF output catalogue base name. -output_PSF_cat_base = f"{output_dir}/psf_catalog" - -### File for found tile IDs -path_found_ID = f"{output_dir}/found_ID.txt" - -### File for missing tile IDs -path_missing_ID = f"{output_dir}/missing_ID.txt" - -### Plot directory and subdirs -plot_dir = f"{output_dir}/plots/" - -### Statistics text file -stats_file_name = "stats_file.txt" - -# Other IO options - -## Input - -### Coordinate column names -col_name_ra = "XWIN_WORLD" -col_name_dec = "YWIN_WORLD" - -### Memory mode, set to None unless very large file -mmap_mode = None - -## Output - -### Output file format extension: '.fits' or '.hdf5' -output_format = ".fits" - -### Additional output columns -add_cols = [ - "FLUX_RADIUS", - "FWHM_IMAGE", - "FWHM_WORLD", - "MAGERR_AUTO", - "MAG_WIN", - "MAGERR_WIN", - "FLUX_AUTO", - "FLUXERR_AUTO", - "FLUX_APER", - "FLUXERR_APER", - "NGMIX_T_NOSHEAR", - "NGMIX_T_PSF_RECONV_NOSHEAR", -] - -## Pre-calibration catalogue, including masked objects and mask flags. -## ShapePipe-v2 (post-#761) ngmix grammar: ellipticity in named scalar -## components NGMIX_G{1,2}_*, PSF size split into NGMIX_T_PSF_ORIG/RECONV. -## IMAFLAGS_ISO (present in the data-path params) is omitted: the simulation -## pipeline runs no imaging-flag masking stage, so the column does not exist. -## NGMIX_MCAL_TYPES_FAIL is kept -- it is the metacal moments-failure flag the -## calibration mask cuts on, identically to the data path. -add_cols_pre_cal = [ - "TILE_ID", - "NUMBER", - "FLAGS", - "NGMIX_MCAL_FLAGS", - "NGMIX_MCAL_TYPES_FAIL", - "N_EPOCH", - "NGMIX_N_EPOCH", - "NGMIX_G1_PSF_ORIG_NOSHEAR", - "NGMIX_G2_PSF_ORIG_NOSHEAR", - "NGMIX_G1_ERR_NOSHEAR", - "NGMIX_G2_ERR_NOSHEAR", -] - -### Set flag columns as integer format -add_cols_pre_cal_format = {} -for key in ( - "NUMBER", - "FLAGS", - "NGMIX_MCAL_FLAGS", - "NGMIX_MCAL_TYPES_FAIL", - "N_EPOCH", - "NGMIX_N_EPOCH", -): - add_cols_pre_cal_format[key] = "I" - -add_cols_pre_cal_format["TILE_ID"] = "A7" -add_cols_pre_cal_format["NUMBER"] = "J" - -# Create key names for metacal information -prefix = "NGMIX" -suffixes = ["1M", "1P", "2M", "2P", "NOSHEAR"] -centers = ["FLAGS", "G1", "G2", "FLUX", "FLUX_ERR", "T", "T_ERR", "T_PSF_RECONV"] -for center in centers: - for suffix in suffixes: - add_cols_pre_cal.append(f"{prefix}_{center}_{suffix}") - -for suffix in suffixes: - add_cols_pre_cal_format[f"FLAGS_{suffix}"] = "I" - - -# Catalog parameters - -## Star matching threshold [deg] -thresh = 0.0002 - -## Number of jackknife resamples for additive bias -## (0: no jackknife computation). -## If < 2000 the jackknife mean fluctuates a lot. -n_jack = 0 - - -## Galaxy selection - -# Flag to output selected and calibrated galaxy catalogue (<= SP v1.4.1). -# If False, only output comprehensive catalogue. -do_selection_calibration = False - -## Magnitude limits -gal_mag_bright = 15 -gal_mag_faint = 30 - -### Spread-model -do_spread_model = False - -### SExtractor flags to keep in addition to FLAGS=0 -### (bit-coded; list of powers of 2); -### Empty list if no flags -flags_keep = [] - -## Minimum number of epochs -n_epoch_min = 2 - -### Signal-to-noise (selection within metacal) -#### minimum to cut noisy objects -gal_snr_min = 10 -#### maximum to cut too bright objects, potentially too large for the postage stamp -gal_snr_max = 500 - -### Relative size, T_gal / T_psf (selection within metacal) -### to select objects that are not too small compared to the PSF, thus not likely to be point-like, -### or to big as they seem to bias the correlation functions -gal_rel_size_min = 0.5 -gal_rel_size_max = 3.0 - -### Correct galaxy size for ellipticity -gal_size_corr_ell = False - -### prior ellipticity dispersion (one component), *only* used for galaxy weight -sigma_eps_prior = 0.34 - - -## Wrap coordinates around this value [deg], set to != 0 if ra=0 is within coordinate range -wrap_ra = 0 diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml deleted file mode 100644 index 0316b909..00000000 --- a/workflow/profiles/candide/config.yaml +++ /dev/null @@ -1,85 +0,0 @@ -# Committed SLURM profile for the candide cluster (IAP). -# -# This is the "one run command" half of the workflow: drive any target with -# -# snakemake --profile workflow/profiles/candide \ -# -s workflow/image_sims/Snakefile \ -# --configfile -# -# and Snakemake owns all scheduling -- it fans out one SLURM job per branch x -# tile and drives them against the cluster, MPI-free. Everything here is -# cluster policy (executor, account, partition, node excludes, per-job -# defaults); it carries no science and no workflow logic. -# -# What is deliberately NOT here: -# -# * Container / apptainer settings. The image-sims rules set -# ``container: None`` and own their ``apptainer exec`` call through the -# shared ``EXEC`` prefix (one image for every stage, with PYTHONPATH / -# PSF_DICT / OMP_NUM_THREADS injected there). So no -# ``software-deployment-method: apptainer`` / ``apptainer-args`` -- those -# would wrap a *second*, redundant container around jobs that already run -# inside one. -# -# * OMP_NUM_THREADS. It is pinned to 1 on the ``apptainer exec`` line in -# workflow/rules/image_sims.smk, not here. The slurm executor submits with -# ``--export=ALL``, which propagates the *driver's* ambient environment; a -# profile only sets CLI flags, never the driver's own env, so an -# ``OMP_NUM_THREADS`` set here would silently depend on the operator having -# exported it by hand. Injecting it at the container boundary puts it where -# the compute runs, committed and independent of the launching shell. -# -# * Per-rule resources (mem_mb, runtime). Those live on each rule in the -# .smk; the ``default-resources`` below are only the floor for rules that -# set none. - -executor: slurm - -# Cluster policy applied to every job unless a rule overrides it. The excludes -# are the flaky/no-internet candide nodes (n17 mount issues, n09 no internet, -# n36); ``slurm_extra`` is passed verbatim onto the sbatch line by the executor -# plugin, so the quoting is what sbatch must see. -# -# Three candide-specific SLURM lessons are baked into the values below (learned -# the hard way on the earlier hand-driven im-sims runs; see shapepipe's retired -# image_sims_pipeline/Snakefile docstring): -# -# * ``runtime`` MUST carry a unit (``60m``, ``6h``, ``2d``). Snakemake's -# resource parser reads a *bare* number as SECONDS, so ``runtime: 60`` would -# silently give every job a 60-second wall clock and kill it on start. The -# quoted-with-unit form here is deliberate; keep it that way, and prefer the -# same in any ``--default-resources`` passed on the command line. (A bare -# integer in a *rule's* ``resources: runtime=720`` is fine -- snakemake -# reads rule-level numeric runtime as minutes -- the seconds trap is only -# the CLI/default-resources parser.) -# -# * ``cpus_per_task`` is pinned to 12 to CAP JOBS PER NODE, not because a job -# needs 12 cores (the chain is MPI-free and pins ``OMP_NUM_THREADS=1`` at -# the container). candide's per-user process limit is ``ulimit -u 1200`` -# *per node*, and apptainer crashes ("can't start new thread") beyond ~4 -# concurrent jobs on a 48-core node. Requesting 12 CPUs/job holds SLURM to -# ~4 jobs per 48-core node, under the ceiling. Dropping this to 1 would let -# SLURM pack ~48 jobs onto a node and crash the compute-heavy im_pipeline -# stage (which inherits this default -- it sets mem/runtime but not cpus). -# -# * After launching a real fan-out, VERIFY the request actually landed: -# ``squeue -u $USER -o "%C %l"`` must show 12 (CPUs) and the wall clock you -# intended (e.g. 12:00:00 for im_pipeline). A silently-misparsed runtime or -# cpus shows up here before it wastes a queue slot. -default-resources: - slurm_account: "cusers" - slurm_partition: "comp,pscomp" - runtime: "60m" - cpus_per_task: 12 - slurm_extra: "'--exclude=n17,n09,n36'" - -# Give an appearing output file a moment on candide's automounted filesystems -# before Snakemake calls a job failed for a missing output, and retry a job -# once on transient node failure. -latency-wait: 5 -retries: 1 - -# Keep the SLURM logs of successful jobs (candide debugging), and rerun a job -# when its code / params / inputs change, not only on mtime. -slurm-keep-successful-logs: true -rerun-triggers: ["mtime", "params", "input", "code"] diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 704b4fb0..4f2a2e46 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -104,8 +104,74 @@ def cv_cosebis_npz(version): ) -def cv_pseudo_cl_fits(version): - return str(COSMO_VAL / f"pseudo_cl_{version}.fits") +def cv_pseudo_cl_sacc(version): + """Untagged pseudo-Cl SACC part cv_pseudo_cl writes (B-mode diagnostic). + + This is the harmonic-space BB diagnostic cv_summarize_bmodes reads. The + *analysis* file's pseudo-Cl part is the tagged, blinded inference product + instead (see cv_pseudo_cl_analysis_sacc) so {version}.sacc stays byte- + comparable against today's cosmosis_fitting.py assembly (PR-3's converter). + """ + return str(COSMO_VAL / f"pseudo_cl_{version}.sacc") + + +# Fiducial harmonic-binning tag the pseudo-Cl producer (twopoint.smk rules +# pseudo_cl / pseudo_cl_cov) stamps into the analysis-grade filename. Mirrors +# inference.smk's PSEUDO_CL_TAG so the analysis file carries the same pseudo-Cl +# the inference pipeline consumes (canonical: blind=A, powspace, nbins=32). +_HARMONIC_FIDUCIAL = config["harmonic"]["fiducial"] +_PSEUDO_CL_TAG = ( + f"blind={_HARMONIC_FIDUCIAL['blind']}" + f"_{_HARMONIC_FIDUCIAL['binning']}" + f"_nbins={_HARMONIC_FIDUCIAL['nbins']}" +) + + +def cv_pseudo_cl_analysis_sacc(version): + """Tagged, blinded pseudo-Cl SACC part the analysis file carries.""" + return str(COSMO_VAL / f"pseudo_cl_{version}_{_PSEUDO_CL_TAG}.sacc") + + +def cv_pseudo_cl_cov(version): + """NaMaster pseudo-Cl covariance FITS (COVAR_EE_EE/BB_BB/EB_EB extensions).""" + return str(COSMO_VAL / f"pseudo_cl_cov_{version}_{_PSEUDO_CL_TAG}.fits") + + +def cv_cosebis_sacc(version): + """COSEBIs SACC part (fiducial scale cut) the cv_cosebis rule writes.""" + return str(COSMO_VAL / f"{version}_cosebis.sacc") + + +def cv_pure_eb_sacc(version): + """Pure-E/B SACC part the cv_pure_eb rule writes.""" + return str(COSMO_VAL / f"{version}_pure_eb.sacc") + + +def cv_rho_tau_sacc(version): + """ρ/τ SACC part calculate_rho_tau_stats writes (rho_tau_{base}.sacc).""" + return str( + COSMO_VAL / "rho_tau_stats" / f"rho_tau_{cv_basename(version, CV_FIDUCIAL)}.sacc" + ) + + +def cv_xi_coarse_sacc(version): + """Coarse ξ± SACC part the xi rule (run_2pcf.py) writes for a version. + + Carries the reporting-binning suffix so requesting it binds the xi job's + wildcards (the rule's txt + coarse .sacc outputs share one wildcard set). + """ + return str( + COSMO_VAL + / ( + f"{version}_xi_coarse_minsep={CV['theta_min']}_maxsep={CV['theta_max']}" + f"_nbins={CV['nbins']}_npatch={CV['npatch']}.sacc" + ) + ) + + +def cv_analysis_sacc(version): + """Terminal assembled analysis file {version}.sacc.""" + return str(COSMO_VAL / f"{version}.sacc") # Common params block shared by every cosmo_val rule: the cv constructor kwargs @@ -275,9 +341,9 @@ rule cv_ratio_xi_sys_xi: # --------------------------------------------------------------------------- rule cv_pseudo_cl: - """Pseudo-Cl E/B spectra for all versions (NaMaster).""" + """Pseudo-Cl E/B spectra for all versions (NaMaster), born as SACC parts.""" output: - pseudo_cl=[cv_pseudo_cl_fits(v) for v in CV_VERSIONS], + pseudo_cl=[cv_pseudo_cl_sacc(v) for v in CV_VERSIONS], params: **cv_params(), threads: 12 @@ -298,6 +364,7 @@ rule cv_pure_eb: xi=lambda w: cv_xi_txt(w.version), output: npz=cv_pure_eb_npz("{version}"), + sacc=cv_pure_eb_sacc("{version}"), params: version="{version}", min_sep_int=CV["pure_eb"]["min_sep_int"], @@ -320,6 +387,7 @@ rule cv_cosebis: xi=lambda w: cv_xi_txt(w.version), output: npz=cv_cosebis_npz("{version}"), + sacc=cv_cosebis_sacc("{version}"), params: version="{version}", min_sep_int=CV["cosebis"]["min_sep_int"], @@ -345,7 +413,7 @@ rule cv_summarize_bmodes: pure_eb=[cv_pure_eb_npz(v) for v in CV_VERSIONS], cosebis=[cv_cosebis_npz(v) for v in CV_VERSIONS], pseudo_cl=( - [cv_pseudo_cl_fits(v) for v in CV_VERSIONS] + [cv_pseudo_cl_sacc(v) for v in CV_VERSIONS] if CV.get("include_pseudo_cl", False) else [] ), output: @@ -371,6 +439,86 @@ rule cv_summarize_bmodes: "../scripts/cv_summarize_bmodes.py" +# --------------------------------------------------------------------------- +# Terminal analysis file: assemble the per-statistic SACC parts into {version}.sacc +# --------------------------------------------------------------------------- +# The five born-as-SACC parts (xi_coarse, pseudo_cl, cosebis, pure_eb, rho_tau) +# are each written by their own rule carrying its own covariance block, except +# ξ± coarse and pseudo-Cℓ which are born cov-less by design. assemble_sacc.py +# loads the parts in canonical order and rebuilds one {version}.sacc with a +# single FullCovariance (point-insertion order = block order). +# +# The pseudo-Cℓ part is the TAGGED, blinded inference product (blind=A, powspace, +# nbins=32) — the same pseudo-Cℓ today's cosmosis_fitting.py consumes — so the +# analysis file stays byte-comparable against it (PR-3's converter). Its real +# NaMaster covariance is injected here from the matching pseudo_cl_cov FITS +# (COVAR_EE_EE/BB_BB/EB_EB → block-diagonal, dropping cross-spectra, matching the +# B-mode PTE's use of COVAR_BB_BB). The ξ± coarse block is the one piece not yet +# sourced from its real covariance: the CosmoCov theory .txt is blind/gaussian/ +# mask-keyed and lives deep in the inference tree, so wiring it couples cosmo_val +# to the whole inference covariance DAG — that sourcing is PR-3's converter +# territory. Until then a documented diagonal placeholder keeps the ξ block (and +# so the FullCovariance) structurally valid; it is a flagged stand-in, never a +# science covariance, and plugs out via --xi-cov the moment PR 3 lands. + + +def cv_assemble_inputs(version): + """The per-statistic SACC parts + covariance inputs assemble_sacc consumes. + + Each part's filename carries enough to bind its producing rule's wildcards + (the coarse ξ± and ρ/τ parts their reporting binning; the pseudo-Cℓ part its + fiducial harmonic tag). pseudo_cl (+ its cov) is included only when the + config toggles the harmonic-space BB into the analysis. + """ + parts = dict( + xi_coarse=cv_xi_coarse_sacc(version), + cosebis=cv_cosebis_sacc(version), + pure_eb=cv_pure_eb_sacc(version), + rho_tau=cv_rho_tau_sacc(version), + ) + if CV.get("include_pseudo_cl", False): + parts["pseudo_cl"] = cv_pseudo_cl_analysis_sacc(version) + parts["pseudo_cl_cov"] = cv_pseudo_cl_cov(version) + return parts + + +rule assemble_sacc: + """Assemble the terminal {version}.sacc from the per-statistic SACC parts.""" + input: + unpack(lambda w: cv_assemble_inputs(w.version)), + output: + sacc=cv_analysis_sacc("{version}"), + params: + version="{version}", + # Statistics this rule wired (same toggles as cv_assemble_inputs). The + # script validates part_paths against this so a typo'd input keyword + # can't silently drop a statistic from the terminal file. + expected=lambda w: [ + k for k in cv_assemble_inputs(w.version) if k != "pseudo_cl_cov" + ], + # ξ± coarse has no real covariance wired yet (its CosmoCov theory block is + # PR-3's converter territory, plugging in via --xi-cov). By DEFAULT this + # is fatal: assemble_sacc.py raises rather than ship {version}.sacc — the + # terminal science file — with a var=1.0 placeholder as its LEADING + # covariance block (~20 orders off the real ξ± variance → silent + # catastrophic χ²/PTE for any consumer). Only an explicit config opt-in + # (cosmo_val.allow_placeholder_cov: true — dry-run / test configs) attaches + # the flagged diagonal placeholder. The pseudo-Cℓ block is real (from the + # pseudo_cl_cov input); COSEBIs / pure-E/B / ρ/τ carry their own. + placeholder_var=(1.0 if CV.get("allow_placeholder_cov", False) else None), + resources: + mem_mb=8000, + runtime=20, + script: + "../scripts/assemble_sacc.py" + + +rule assemble_sacc_all: + """Assemble the analysis SACC file for every version.""" + input: + [cv_analysis_sacc(v) for v in CV_VERSIONS], + + # --------------------------------------------------------------------------- # Aggregate target: the whole validation suite # --------------------------------------------------------------------------- @@ -392,3 +540,5 @@ rule cosmo_val_all: str(COSMO_VAL / "ratio_xi_sys_xi.png"), # B-modes str(COSMO_VAL / "bmode_summary.json"), + # Terminal analysis file: the assembled {version}.sacc per version + [cv_analysis_sacc(v) for v in CV_VERSIONS], diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk deleted file mode 100644 index a4eb92d0..00000000 --- a/workflow/rules/image_sims.smk +++ /dev/null @@ -1,568 +0,0 @@ -"""Image-simulation orchestration: raw SKiLLS sim images -> shear m/c bias. - -This rule set drives the image-simulation validation chain end to end and is -the sp_validation-side half of the split described in -``UNIONS-WL/MultiBand_ImSim#1``: ShapePipe turns the simulated tiles into -per-tile shape catalogues, then sp_validation merges, extracts, calibrates and -finally measures the multiplicative/additive shear bias. - -Two images, one prefix shape. Architecturally one image could run every -stage -- the sp_validation image is built ``FROM`` the ShapePipe image, so it -carries both stacks -- but the *published* sp_validation image's environment -is not yet trustworthy for the ShapePipe half: sp_validation has no lockfile -and does not declare its numba-bearing dependency (``cosmo_numba``), so -unpinned install layers can drift NumPy past numba's window (a 2026-07-11 -gate run hit exactly this: ``Numba needs NumPy 2.4 or less. Got NumPy 2.5`` -at the ngmix stage). PYTHONPATH shadowing covers pure-Python *code*, never -binary deps, so until sp_validation is uv-locked with its deps declared -(spun off as its own task), each half runs in its own repo's image -- the -same split the gate766 baseline ran: - -* ShapePipe stages -> ``pipeline`` (raw images -> per-tile cats) and ``merge`` - (``create_final_cat`` -> ``final_cat_{sim}.hdf5``) run in ``sif_pipeline`` - (the ShapePipe image). -* sp_validation stages -> ``manifest``, ``extract`` (-> comprehensive cat), - ``calibrate`` (-> cut cat) and ``m_bias`` (-> ``m_bias_results.yaml``) run - in ``sif`` (the sp_validation image). - -Every rule sets ``container: None`` and calls ``apptainer exec`` explicitly -through a shared prefix template (``EXEC_PIPELINE`` / ``EXEC`` -- identical -env injections, different image), because the images are not the workflow's -top-level container. Everything is parameterised under -``config["image_sims"]`` -- the two ``sif`` keys, repository roots, data roots, -the PSF dictionary, the explicit ``tile_ids`` list and the sim/calibration -knobs -- so a fresh user drives it from config alone, with no hard-coded clone -layout. Configuration is fail-fast: a schema check at load rejects an unknown -key (typo) and a missing science key (see ``workflow/image_sims/config.yaml`` -for the operational/science split). The ``PYTHONPATH`` override injects both -repos' ``src`` so the *branch* source (ShapePipe's ``#766`` build; -sp_validation's ``image_sims.py``, ``catalog.match_catalogs_radec``) wins over -whatever is baked into the image. - -The five simulations per grid are the reference ``1z2z`` (no input shear) plus -the ``+/-`` shear pairs ``1p2z``/``1m2z`` (g1) and ``1z2p``/``1z2m`` (g2); the -m-bias estimator matches each to the reference by RA/Dec. -""" - -import os -from pathlib import Path - -IMSIM = config["image_sims"] - -# --- fail-fast schema check ---------------------------------------------- -# One home for every fact: the run config carries the science knobs, the -# workflow config.yaml carries the operational defaults, and *this* block is -# where a typo or a missing knob dies -- at DAG parse, before any compute. -# -# Every key must be declared below. An unknown key under ``image_sims:`` is a -# hard error (typo protection); a missing *science* key is a hard error naming -# the key (no silent code default anywhere). Operational keys default in the -# workflow config.yaml and nowhere else: the .smk reads them as bare -# ``IMSIM[key]`` (never ``.get`` with a second literal), so their value comes -# from config.yaml alone -- the single home for an operational default. -# -# Science keys: required from the *run* config; no default in config.yaml (only -# a commented template line) and no default in code. These fix the estimator's -# scientific behaviour, so they must be stated per run, never inherited. -_SCIENCE_KEYS = { - "w_cols", - "pair_match", - "match_radius_deg", - "n_bootstrap", - "bootstrap_seed", - "mask_config", -} -# Deprecated science keys: accepted (so a pre-``w_cols`` run config still parses -# and the estimator's back-compat path runs) but not *required* -- our configs -# state ``w_cols``. Listed here only to keep them out of the unknown-key error. -_DEPRECATED_KEYS = { - "w_col", -} -# Operational keys: default (visibly) in the workflow config.yaml; the .smk -# reads them bare, so config.yaml is their one home. -_OPERATIONAL_KEYS = { - "binds", - "sims_type", - "branches", - "shape", - "config_dir", - "psf_model", - "n_smp", - "extract_script", - "calibrate_script", -} -# Structural keys: paths/identifiers the run must supply (no sensible default). -_STRUCTURAL_KEYS = { - "sif", - "sif_pipeline", - "shapepipe_repo", - "sp_validation_repo", - "grids_base", - "input_sims_base", - "psf_dict", - "num", - "tile_ids", -} -_ALLOWED_KEYS = ( - _SCIENCE_KEYS | _DEPRECATED_KEYS | _OPERATIONAL_KEYS | _STRUCTURAL_KEYS -) - -_unknown = set(IMSIM) - _ALLOWED_KEYS -if _unknown: - raise ValueError( - "image_sims: unknown config key(s) " - f"{sorted(_unknown)} -- check for a typo (allowed keys: " - f"{sorted(_ALLOWED_KEYS)})" - ) -_missing_science = sorted(_SCIENCE_KEYS - set(IMSIM)) -if _missing_science: - raise ValueError( - "image_sims: missing required science key(s) " - f"{_missing_science} -- these have no default and must be set in the " - "run config (see the commented template in workflow/image_sims/config.yaml)" - ) -_missing_structural = sorted(_STRUCTURAL_KEYS - set(IMSIM)) -if _missing_structural: - raise ValueError( - "image_sims: missing required key(s) " - f"{_missing_structural} -- set them in the run config" - ) - -# --- containers ----------------------------------------------------------- -# Two images (see module docstring): the ShapePipe image for the pipeline and -# merge stages, the sp_validation image for everything downstream. Collapse -# back to one image once sp_validation's env is lock-managed. -SIF = IMSIM["sif"] # sp_validation stages -SIF_PIPELINE = IMSIM["sif_pipeline"] # ShapePipe stages -BINDS = IMSIM["binds"] - -# --- repositories (bound into the image; branch code overrides) ----------- -SHAPEPIPE_REPO = IMSIM["shapepipe_repo"] -SPV_REPO = IMSIM["sp_validation_repo"] - -# --- data and run directories -------------------------------------------- -GRIDS_BASE = IMSIM["grids_base"] # run/output root; one sub-dir per sim -INPUT_SIMS_BASE = IMSIM["input_sims_base"] # SKiLLS sim images -PSF_DICT = IMSIM["psf_dict"] # Herve's Full_psf_dict.pickle - -# --- simulation grid ------------------------------------------------------ -# SIM_BASES is the set of branches *this run requests* -- the reference plus the -# four +/- sheared branches. Their injected shear (amplitude, per-branch -# (g1,g2), pairing) is NOT a literal here: it lives only in manifest.yaml, built -# by im_manifest from each branch's basic_info.txt and read back by im_mbias. -NUM = IMSIM["num"] -SIMS_TYPE = IMSIM["sims_type"] -_SUFFIX = f"_{SIMS_TYPE}_{NUM}" if SIMS_TYPE == "grid" else f"_{NUM}" -SIM_BASES = list(IMSIM["branches"]) -SIMS = [f"{base}{_SUFFIX}" for base in SIM_BASES] -MANIFEST = f"{GRIDS_BASE}/manifest.yaml" -BUILD_MANIFEST = f"{SPV_REPO}/workflow/scripts/im_build_manifest.py" - -# --- tiles ---------------------------------------------------------------- -# tile_ids is the one tile-input mechanism: an explicit list in the run config. -TILE_IDS = list(IMSIM["tile_ids"]) - -# --- calibration / m-bias knobs ------------------------------------------ -SHAPE = IMSIM["shape"] -MASK_CONFIG = IMSIM["mask_config"] # e.g. config/calibration/mask_v1.X.9_im_sim.yaml -PARAMS_TEMPLATE = f"{SPV_REPO}/workflow/image_sims/params_im_sim.py" -# ShapePipe cfis_image_sims config dir (per-tile/exposure configs + final_cat.param). -CONFIG_DIR = IMSIM["config_dir"] - -# ShapePipe scripts live in the ShapePipe repo (also baked into its image). -CREATE_FINAL_CAT = f"{SHAPEPIPE_REPO}/scripts/python/create_final_cat.py" -RUN_JOB = f"{SHAPEPIPE_REPO}/scripts/sh/run_job_sp_canfar_v2.0.bash" -# Extract/calibrate run from the sp_validation *repo* checkout (bind-mounted), -# not the baked copies: the container tracks the branch but lags it, and the -# image-sims path needs branch-only fixes (star-catalogue-optional extract, -# FITS-aware CalibrateCat.read_cat). Overridable for a different checkout. -EXTRACT_INFO = IMSIM["extract_script"] -CALIBRATE = IMSIM["calibrate_script"] -# m-bias is *this branch's* extracted core, injected on PYTHONPATH. -COMPUTE_M_BIAS = f"{SPV_REPO}/scripts/compute_m_bias_image_sims.py" - -# --- container exec prefixes ---------------------------------------------- -# One prefix *shape* for every stage -- two instances, one per image. Three -# env injections make the on-disk branch -# code and the sim PSF win over the image's baked copies: -# -# * PYTHONPATH prepends BOTH repos' ``src`` (ShapePipe first, then -# sp_validation), so Python resolves the worktree build before -# ``/app``/``/sp_validation`` -- the local-testing counterpart of the -# git-ref deps, letting the branch code run without an image rebuild. This -# covers the Python *packages* only: the bash entry points (run_job) and -# the ShapePipe/sp_validation *scripts* are still invoked at the repo paths -# resolved from config (RUN_JOB, CREATE_FINAL_CAT, EXTRACT_INFO, ...), not -# shadowed by PYTHONPATH. -# * PSF_DICT points the fake_psf module (PSF_DICT_PATH = $PSF_DICT, expanded -# via getexpanded) at this run's PSF dictionary. -# -# The SLURM env vars are stripped (``env -u ...``) so that when the ShapePipe -# pipeline stage's OpenMPI initialises inside the image it does not try to -# attach to the host SLURM launcher (cf. apptainer_noslurm.sh). The strip is -# harmless for the pure-Python sp_validation stages, so one prefix serves all. -# -# ``OMP_NUM_THREADS=1`` is injected here, at the ``apptainer exec`` call, and -# not left to the SLURM profile. The chain is MPI-free: Snakemake fans out one -# job per branch x tile and each job's parallelism is ShapePipe's own internal -# multiprocessing (``-N n_smp``), so the OpenMP/BLAS thread pool inside the -# container must be pinned to 1 to avoid oversubscription. The SLURM profile -# cannot pin it reliably: the slurm executor submits with ``--export=ALL``, -# which propagates the *driver's* ambient environment -- but a Snakemake -# profile only sets CLI flags, never the driver's own env, so an -# ``OMP_NUM_THREADS`` there would depend on the operator having exported it by -# hand (the implicit, uncommitted state the "one run command" is meant to -# retire). Injecting it on the ``apptainer exec`` line puts it where the -# compute actually runs -- inside the container, independent of the driver's -# env -- the same lever this prefix already uses for PYTHONPATH/PSF_DICT. -_EXEC_PREFIX = ( - "env -u SLURM_JOBID -u SLURM_JOB_ID -u SLURM_PROCID " - f"apptainer exec --bind {BINDS} " - f"--env PYTHONPATH={SHAPEPIPE_REPO}/src:{SPV_REPO}/src " - f"--env PSF_DICT={PSF_DICT} --env OMP_NUM_THREADS=1 " -) -EXEC = _EXEC_PREFIX + SIF # sp_validation stages -EXEC_PIPELINE = _EXEC_PREFIX + SIF_PIPELINE # ShapePipe stages - -JOB_MASK = sum([1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]) - - -wildcard_constraints: - sim="|".join(SIMS), - tile="|".join(t.replace(".", r"\.") for t in TILE_IDS), - - -# ========================================================================== -# Convenience targets (run in order) -# ========================================================================== -rule im_manifest_only: - input: - MANIFEST, - - -rule im_init_all: - input: - expand(f"{GRIDS_BASE}/{{sim}}/params.py", sim=SIMS), - - -rule im_pipeline_all: - input: - expand( - f"{GRIDS_BASE}/{{sim}}/logs/pipeline_{{tile}}.done", - sim=SIMS, - tile=TILE_IDS, - ), - - -rule im_merge_all: - input: - expand(f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", sim=SIMS), - - -rule im_extract_all: - input: - expand( - f"{GRIDS_BASE}/{{sim}}/shape_catalog_comprehensive_{SHAPE}.fits", - sim=SIMS, - ), - - -rule im_calibrate_all: - input: - expand( - f"{GRIDS_BASE}/{{sim}}/shape_catalog_cut_{SHAPE}.fits", sim=SIMS - ), - - -# ========================================================================== -# Rules -# ========================================================================== -rule im_manifest: - """Build the campaign manifest at the head of the DAG. - - Parses ``g_cosmic`` from every requested branch's ``basic_info.txt``, - cross-checks each against its ``1{X}2{Y}`` name and the (0,0) reference, - derives the single injected amplitude, and writes ``manifest.yaml`` into the - run root. This is the one home for the injected-shear facts; im_mbias reads - the amplitude and branch map from here, nowhere else. Pure sp_validation - stage (stdlib parse of basic_info; PyYAML to write). - """ - input: - # basic_info.txt for each requested branch, so editing a sim's record - # rebuilds the manifest (and re-validates) rather than reusing a stale one. - basic_info=expand( - f"{INPUT_SIMS_BASE}/{{sim}}/basic_info.txt", sim=SIMS - ), - output: - manifest=MANIFEST, - params: - branch_args=lambda wc: " ".join(f"--branch {b}" for b in SIM_BASES), - input_sims_base=INPUT_SIMS_BASE, - sims_type=SIMS_TYPE, - num=NUM, - shell: - "{EXEC} python {BUILD_MANIFEST} " - "--input-sims-base {params.input_sims_base} " - "--sims-type {params.sims_type} --num {params.num} " - "{params.branch_args} -o {output.manifest}" - - -rule im_init: - """Stage per-sim run directory: params.py, mask config, ShapePipe configs, - and the raw SKiLLS image inputs. - - ``params_im_sim.py`` derives the field name from the directory basename, so - the same template serves every sim; ``config_mask.yaml`` and ``cfis`` are - symlinks the downstream calibration and merge steps read from cwd. - - ``input_tiles``/``input_exp`` are top-level symlinks to the raw SKiLLS tile - and exposure images; ShapePipe's ``get_images_runner`` resolves them via - ``$SP_DIR/input_{tiles,exp}`` (``$SP_DIR`` is the run dir). ``run_job`` does - not stage these, so ``im_init`` must -- this is what makes ``im_pipeline`` - runnable from raw images, not just from pre-staged intermediates. - """ - input: - # Tracked so that editing the params template or mask config re-stages - # them into every run dir (a plain params: value would not retrigger, - # silently leaving stale params.py behind after a grammar change). - template=PARAMS_TEMPLATE, - mask_src=os.path.join(SPV_REPO, MASK_CONFIG), - output: - params=f"{GRIDS_BASE}/{{sim}}/params.py", - mask=f"{GRIDS_BASE}/{{sim}}/config_mask.yaml", - params: - config_dir=CONFIG_DIR, - run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", - cfis=lambda wc: f"{GRIDS_BASE}/{wc.sim}/cfis", - sim_tiles=lambda wc: f"{INPUT_SIMS_BASE}/{wc.sim}/images/SP_tiles", - sim_exp=lambda wc: f"{INPUT_SIMS_BASE}/{wc.sim}/images/SP_exp", - shell: - # cfis / input_tiles / input_exp are stable read-only symlinks (used by - # get_images, merge, extract); created here but not tracked as outputs, - # which snakemake will not accept for a symlink/directory. - "mkdir -p $(dirname {output.params}) && " - "cp {input.template} {output.params} && " - "ln -sf {input.mask_src} {output.mask} && " - "ln -sfT {params.config_dir} {params.cfis} && " - "ln -sfT {params.sim_tiles} {params.run_dir}/input_tiles && " - "ln -sfT {params.sim_exp} {params.run_dir}/input_exp" - - -rule im_pipeline: - """Run ShapePipe on one simulated tile (ShapePipe stage). - - Delegates the module DAG to ShapePipe's own job runner; the sentinel log - marks tile completion for the merge step. This is the compute-heavy, - MPI-bearing stage. - """ - input: - # ``params.py`` is a *tracked* output of ``im_init``, so this one input - # supplies the im_init -> im_pipeline edge. The ``cfis`` symlink the - # shell reads (via {RUN_JOB}) is created by that same im_init shell block - # as an *untracked* side effect -- no rule declares it as an output - # (snakemake will not track a symlink/directory output). Declaring it an - # input here therefore asked the DAG for a file no rule produces: on a - # fresh grids_base it aborted the build with MissingInputException before - # any job ran. It is safe to drop -- cfis exists whenever params does, - # since im_init stages both together. - params=f"{GRIDS_BASE}/{{sim}}/params.py", - output: - done=touch(f"{GRIDS_BASE}/{{sim}}/logs/pipeline_{{tile}}.done"), - params: - run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", - psf=IMSIM["psf_model"], - n_smp=IMSIM["n_smp"], - resources: - mem_mb=16000, - runtime=720, - shell: - "cd {params.run_dir} && " - "{EXEC_PIPELINE} bash {RUN_JOB} " - "-e {wildcards.tile} -t image_sims -j {JOB_MASK} " - "-p {params.psf} -N {params.n_smp}" - - -rule im_merge: - """Merge per-tile ShapePipe catalogues into final_cat_{sim}.hdf5. - - ``create_final_cat.py`` lives in the ShapePipe repo/image; run in image_sims - mode (``-I``) it walks the per-tile output under the run directory. - """ - input: - tiles=expand( - f"{GRIDS_BASE}/{{{{sim}}}}/logs/pipeline_{{tile}}.done", - tile=TILE_IDS, - ), - output: - cat=f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", - params: - run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", - shell: - "cd {params.run_dir} && " - "{EXEC_PIPELINE} python {CREATE_FINAL_CAT} " - "-I -m final_cat_{wildcards.sim}.hdf5 -i .. " - "-p cfis/final_cat.param -P {wildcards.sim} " - "-o n_tiles_final.txt -v" - - -rule im_extract: - """Extract the comprehensive ngmix catalogue (sp_validation stage). - - ``extract_info.py`` reads ``params.py`` from cwd and the merged catalogue, - writing ``shape_catalog_comprehensive_{shape}``. - """ - input: - cat=f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", - params=f"{GRIDS_BASE}/{{sim}}/params.py", - output: - cat=f"{GRIDS_BASE}/{{sim}}/shape_catalog_comprehensive_{SHAPE}.fits", - params: - run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", - shell: - "cd {params.run_dir} && {EXEC} python {EXTRACT_INFO}" - - -rule im_calibrate: - """Calibrate and cut the comprehensive catalogue (sp_validation stage). - - ``calibrate_comprehensive_cat.py`` reads ``config_mask.yaml`` from cwd, - applies the metacal calibration and selection, and writes - ``shape_catalog_cut_{shape}.fits``. - """ - input: - cat=f"{GRIDS_BASE}/{{sim}}/shape_catalog_comprehensive_{SHAPE}.fits", - mask=f"{GRIDS_BASE}/{{sim}}/config_mask.yaml", - output: - cat=f"{GRIDS_BASE}/{{sim}}/shape_catalog_cut_{SHAPE}.fits", - params: - run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", - shell: - "cd {params.run_dir} && " - "{EXEC} python {CALIBRATE} -s calibrate" - - -rule im_mbias: - """Multiplicative/additive shear bias from the calibrated grids. - - Produces the workflow's headline artifact, ``m_bias_results.yaml``. The - injected shear (``shear_amplitude`` and the branch map) comes from - ``manifest.yaml`` alone -- no literal amplitude here or in config.yaml. The - generated ``m_bias_config.yaml`` carries the manifest's ``branches`` and - ``pairs``, so the estimator's sim list and pairing are the campaign's, not a - hard-coded default. - """ - input: - manifest=MANIFEST, - cats=expand( - f"{GRIDS_BASE}/{{sim}}/shape_catalog_cut_{SHAPE}.fits", sim=SIMS - ), - output: - results=f"{GRIDS_BASE}/results/m_bias_results.yaml", - params: - cfg=f"{GRIDS_BASE}/results/m_bias_config.yaml", - grids_base=GRIDS_BASE, - num=NUM, - cat_name=f"shape_catalog_cut_{SHAPE}.fits", - sif=SIF, - sif_pipeline=SIF_PIPELINE, - shapepipe_repo=SHAPEPIPE_REPO, - sp_validation_repo=SPV_REPO, - # Science knobs, read bare from the run config (no default here). - match_radius_deg=IMSIM["match_radius_deg"], - w_cols=IMSIM["w_cols"], - n_bootstrap=IMSIM["n_bootstrap"], - pair_match=IMSIM["pair_match"], - bootstrap_seed=IMSIM["bootstrap_seed"], - run: - import hashlib - import re - import subprocess - - import yaml - - with open(input.manifest) as fh: - manifest = yaml.safe_load(fh) - - def _git(repo, *args): - """Read a git fact from ``repo``; ``None`` if it is not a checkout.""" - try: - return subprocess.run( - ["git", "-C", repo, *args], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - except (subprocess.CalledProcessError, FileNotFoundError): - return None - - def _sif_revision(sif_path): - """GHCR revision baked into the SIF's OCI labels. - - A plain-text scan of the image file (login-safe: no exec, no - container start), reading org.opencontainers.image.revision -- the - source commit GHCR built the image from. ``None`` if absent. - """ - try: - with open(sif_path, "rb") as fh: - blob = fh.read() - except OSError: - return None - m = re.search( - rb'org\.opencontainers\.image\.revision"?[:=]"?([0-9a-f]{7,40})', - blob, - ) - return m.group(1).decode() if m else None - - # Manifest hash: sha256 of the exact bytes im_manifest wrote, so the - # result records which injected-shear facts it was computed against. - with open(input.manifest, "rb") as fh: - manifest_sha256 = hashlib.sha256(fh.read()).hexdigest() - - provenance = { - "manifest_sha256": manifest_sha256, - "sp_validation": { - "branch": _git(params.sp_validation_repo, "rev-parse", "--abbrev-ref", "HEAD"), - "commit": _git(params.sp_validation_repo, "rev-parse", "HEAD"), - }, - "shapepipe": { - "branch": _git(params.shapepipe_repo, "rev-parse", "--abbrev-ref", "HEAD"), - "commit": _git(params.shapepipe_repo, "rev-parse", "HEAD"), - }, - "containers": { - "sif": params.sif, - "ghcr_revision": _sif_revision(params.sif), - "sif_pipeline": params.sif_pipeline, - "ghcr_revision_pipeline": _sif_revision(params.sif_pipeline), - }, - } - - os.makedirs(os.path.dirname(output.results), exist_ok=True) - # Emit *every* key the estimator requires -- pair_match and - # bootstrap_seed included. Requiring a key without emitting it would - # be a KeyError at run time, so the generated config is the complete - # contract between rule and estimator. ``provenance`` rides along as a - # top-level block: the compute script copies it verbatim into the output - # results yaml, so a result file is self-describing (which manifest, - # which repo commits, which container built the number). - mbias_cfg = { - "grids_dir": params.grids_base, - "num": params.num, - "catalog_name": params.cat_name, - # Injected shear: from the manifest, the single source of truth. - "shear_amplitude": manifest["shear_amplitude"], - "branches": list(manifest["branches"]), - "pairs": manifest["pairs"], - "match_radius_deg": params.match_radius_deg, - "w_cols": list(params.w_cols), - "pair_match": params.pair_match, - "n_bootstrap": params.n_bootstrap, - "bootstrap_seed": params.bootstrap_seed, - "results_dir": os.path.dirname(output.results), - "output_path": output.results, - "provenance": provenance, - } - with open(params.cfg, "w") as fh: - yaml.safe_dump(mbias_cfg, fh) - shell( - "{EXEC} python {COMPUTE_M_BIAS} -c {params.cfg} -v" - ) diff --git a/workflow/rules/inference.smk b/workflow/rules/inference.smk index d2847976..6bf68645 100644 --- a/workflow/rules/inference.smk +++ b/workflow/rules/inference.smk @@ -29,7 +29,7 @@ GLASS_MOCK_FITS_PATTERN = str( ) GLASS_MOCK_CONFIG_PATTERN = str( COSMO_INFERENCE_PROD - / f"cosmosis_config/output/cosmosis_pipeline_glass_mocks_{GLASS_MOCK_VERSION}_glass_mock_{{mock_id}}.ini" + / f"cosmosis_config/cosmosis_pipeline_glass_mocks_{GLASS_MOCK_VERSION}_glass_mock_{{mock_id}}.ini" ) # Fiducial harmonic-binning tag the pseudo-Cl producer (twopoint.smk) stamps @@ -55,11 +55,26 @@ def pseudo_cl_assets(version): cov_path = PSEUDO_CL_DIR / f"pseudo_cl_cov_{version}_{PSEUDO_CL_TAG}.fits" return str(cl_path), str(cov_path) +# --------------------------------------------------------------------------- +# DORMANT — pre-SACC cosmosis assembly. Migration to native SACC deferred to +# PR 7 (native-SACC inference consumption); do NOT deep-migrate here. +# +# The SACC migration (PR 4) removed the data products several of these inputs +# name, so this rule's DAG no longer resolves and is NOT reachable from the +# cosmo_val suite (cosmo_val_all never requests it). Stale inputs: +# - xi_plus / xi_minus FITS: the `xi` rule now emits the coarse ξ± SACC part +# ({version}_xi_coarse_...sacc), not per-sign FITS. +# - pseudo_cl / pseudo_cl_cov via pseudo_cl_assets(): the `pseudo_cl` rule now +# writes .sacc (pseudo_cl_assets still requests .fits). +# PR 7 rewires this to consume the assembled {version}.sacc (built by +# cosmo_val.smk's assemble_sacc rule) directly, retiring cosmosis_fitting.py's +# per-product FITS assembly. Until then the inference target is knowingly red. +# --------------------------------------------------------------------------- rule inference_prep: input: # Processed covariance matrix - use centralized covariance_path() cov_matrix=lambda w: covariance_path(w.version, w.blind, min_sep=w.min_sep, max_sep=w.max_sep, nbins=w.nbins), - # Xi FITS files + # Xi FITS files — PRE-SACC (no longer produced; see dormant note above) xi_plus=str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), xi_minus=str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), # n(z) file (using new location with base version mapping) @@ -69,6 +84,7 @@ rule inference_prep: tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), # tau covariance (tracked as dependency) tau_cov=str(COSMO_VAL / "rho_tau_stats/cov_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}_th.npy"), + # pseudo_cl / pseudo_cl_cov — PRE-SACC (.fits path; producer now writes .sacc) pseudo_cl=lambda w: pseudo_cl_assets(w.version)[0], pseudo_cl_cov=lambda w: pseudo_cl_assets(w.version)[1], output: @@ -78,7 +94,7 @@ rule inference_prep: ), config_file=str( COSMO_INFERENCE_PROD - / "cosmosis_config/output/cosmosis_pipeline_{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.ini" + / "cosmosis_config/cosmosis_pipeline_{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.ini" ) params: cosmosis_root="{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}", diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 22c09db2..ebd19f91 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -1,13 +1,20 @@ # Two-point data-vector rules: xi, rho/tau, and pseudo-Cl products. +# WORKFLOW_SCRIPTS (from common.py) is the generic workflow's scripts dir, +# resolved from the running checkout — used by the raw-shell MPI xi_highres rule. rule xi: input: catalog=get_shear_catalog, output: - str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), - str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + # Raw TreeCorr .txt byproduct (read back by covariance + skip-if-exists) + # and the born-as-SACC coarse ξ± part (no covariance until the + # assemble_sacc rule injects the CosmoCov block). Both outputs carry the + # same reporting-binning wildcards — Snakemake requires every output of a + # rule to share one wildcard set, and it keeps the coarse .sacc name + # self-describing so requesting it binds the xi job unambiguously. + txt=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), + xi_coarse=str(COSMO_VAL / "{version}_xi_coarse_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 24 params: ver="{version}", @@ -15,7 +22,6 @@ rule xi: max_sep="{max_sep}", nbins="{nbins}", npatch="{npatch}", - fits=False, resources: mem_mb=30000, disk_mb=20000, @@ -25,12 +31,16 @@ rule xi: rule xi_highres: - """High-resolution xi for COSEBIS integration.""" + """High-resolution xi for COSEBIS integration. + + Terminal born-as-SACC product: {version}_xi_fine.sacc (a DiagonalCovariance + from TreeCorr varxip/varxim). COSEBIs and pure-E/B consume it. The raw .txt + dump is kept as a convergence byproduct. + """ container: None output: txt=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.txt"), - xi_plus=str(COSMO_VAL / f"xi_plus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), - xi_minus=str(COSMO_VAL / f"xi_minus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), + xi_fine=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_fine.sacc"), resources: tasks=30, cpus_per_task=12, @@ -45,7 +55,7 @@ rule xi_highres: "--bind /home,/n09data,/n17data,/n23data1,/softs " "--env LD_LIBRARY_PATH=/softs/openmpi/5.0.5-slurm-CentOS8/lib " "/n17data/cdaley/containers/containers " - "python /automnt/n17data/cdaley/unions/pure_eb/code/sp_validation/workflow/scripts/run_2pcf_highres.py" + f"python {WORKFLOW_SCRIPTS}/run_2pcf_highres.py" rule run_cosmo_val: @@ -70,6 +80,11 @@ rule rho_tau_stats: output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + # Born-as-SACC ρ/τ part (ρ_0…ρ_5 autos + τ_0/τ_2/τ_5 leakage, carrying + # its own covariance block) that the assemble_sacc rule consumes; + # calculate_rho_tau_stats writes it alongside the FITS via + # rho_tau_to_sacc_part. + rho_tau=str(COSMO_VAL / "rho_tau_stats/rho_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 48 params: ver="{version}", @@ -92,9 +107,9 @@ wildcard_constraints: rule pseudo_cl: - """Generate pseudo-Cl data vector with configurable binning.""" + """Generate pseudo-Cl data vector (born as SACC) with configurable binning.""" output: - pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.fits"), + pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.sacc"), wildcard_constraints: blind="[ABC]", params: @@ -146,7 +161,7 @@ rule pseudo_cl_all: """Generate pseudo-Cls for all versions.""" input: expand( - str(COSMO_VAL / "pseudo_cl_{version}_blind=A_powspace_nbins=32.fits"), + str(COSMO_VAL / "pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc"), version=PSEUDO_CL_VERSIONS, ), @@ -164,7 +179,7 @@ rule pseudo_cl_fine_all: """Generate fine pseudo-Cls for COSEBIS.""" input: expand( - str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_linear_nbins=2040.fits"), + str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_linear_nbins=2040.sacc"), version=config["versions"], blind=BLINDS, ), diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py new file mode 100644 index 00000000..513ec976 --- /dev/null +++ b/workflow/scripts/assemble_sacc.py @@ -0,0 +1,254 @@ +"""Assemble the terminal ``{version}.sacc`` analysis file from per-statistic parts. + +Dual-mode. Under Snakemake (``script:`` directive) the injected ``snakemake`` +object supplies the parts + covariance inputs; as a standalone CLI (argparse) +the same assembly runs from explicit flags (the lightcone/ASTRA path). + +Each per-statistic ``*.sacc`` *part* (written born-as-SACC by the mixins and the +run_2pcf / generate_pseudo_cl scripts) holds one statistic. The assembler loads +them in canonical order — ξ± coarse, pseudo-Cℓ, COSEBIs, pure-E/B, ρ/τ — and +calls :func:`sacc_writers.assemble_analysis_sacc`, which rebuilds one Sacc with a +single block-diagonal ``FullCovariance`` (point-insertion order = block order, +validated by ``sacc_io.assemble_covariance``). + +Covariance sourcing (the part-by-part decision) +----------------------------------------------- +``assemble_analysis_sacc`` REQUIRES every part to carry its own covariance block. +The COSEBIs, pure-E/B and ρ/τ parts already do (their writers attach it). The +ξ± coarse and pseudo-Cℓ parts are born cov-less by design; this script injects +their blocks before assembly: + +* **ξ± coarse** — the CosmoCov theory covariance ``.txt`` (``--xi-cov``). For the + single-bin round it is already ``[ξ+; ξ−]``-ordered (CosmoCov / covdat_to_fits: + ``STRT_0=0`` XI_PLUS, ``STRT_1=len/2`` XI_MINUS), which is exactly the SACC + ξ insertion order, so ``np.loadtxt`` → ``add_covariance`` needs no permutation. +* **pseudo-Cℓ** — the NaMaster iNKA / OneCovariance covariance FITS + (``--pseudo-cl-cov`` + ``--pseudo-cl-cov-hdu``). The FITS carries the 16 + EE/EB/BE/BB cross-blocks (each ``nbp × nbp``); SACC stores EE, BB, EB (in that + order), so we assemble the block-diagonal ``[EE_EE; BB_BB; EB_EB]``. The + cross-spectrum blocks (EE↔BB, …) are dropped — matching how the B-mode PTE + today reads only ``COVAR_BB_BB``. **TODO(PR-cov):** carry the full dense + EE/BB/EB cross-covariance once the analysis needs cross-spectrum correlations. + +When a cov input is absent the assembly cannot proceed on a real product; pass +``--allow-placeholder`` to attach a documented diagonal placeholder +(``placeholder_var`` on every point of the cov-less parts) so the DAG dry-run and +the fast test can still produce a structurally-valid ``FullCovariance``. The +placeholder is a flagged stand-in, never a science covariance. +""" + +import argparse + +import numpy as np + +from sp_validation import sacc_io +from sp_validation.cosmo_val.sacc_writers import assemble_analysis_sacc + +# NaMaster iNKA covariance FITS: per-spectrum HDU names. SACC insertion order is +# EE, BB, EB, so the block-diagonal is assembled in that order. +_CL_HDU = {"EE": "COVAR_EE_EE", "BB": "COVAR_BB_BB", "EB": "COVAR_EB_EB"} +_CL_ORDER = ("EE", "BB", "EB") + +# Canonical part order — the order assemble_analysis_sacc inserts points in, which +# must match the covariance block order. Missing parts are simply skipped. +CANONICAL = ("xi_coarse", "pseudo_cl", "cosebis", "pure_eb", "rho_tau") + + +def _pseudo_cl_cov_block(cov_fits, hdu): + """Block-diagonal ``[EE_EE; BB_BB; EB_EB]`` from the NaMaster iNKA cov FITS. + + ``hdu`` selects the file flavor: for the per-spectrum iNKA file we read the + three named diagonal HDUs; for a single dense HDU (OneCovariance / g+ng + ``COVAR_FULL``) that already spans EE/BB/EB we return it as-is. + """ + from astropy.io import fits + + with fits.open(cov_fits) as hdul: + names = {h.name for h in hdul} + if all(_CL_HDU[s] in names for s in _CL_ORDER): + blocks = [np.asarray(hdul[_CL_HDU[s]].data, float) for s in _CL_ORDER] + n = blocks[0].shape[0] + full = np.zeros((3 * n, 3 * n)) + for i, block in enumerate(blocks): + full[i * n : (i + 1) * n, i * n : (i + 1) * n] = block + return full + return np.asarray(hdul[hdu].data, float) + + +def _attach_cov(part, name, xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var): + """Ensure ``part`` carries a covariance, injecting the xi/pseudo-Cℓ block. + + ``part`` is mutated in place. cosebis/pure_eb/rho_tau parts already carry + their covariance and pass straight through. Raises loudly if a required xi / + pseudo-Cℓ block is missing and no placeholder was requested. + """ + if part.covariance is not None: + return part + if name == "xi_coarse": + if xi_cov is not None: + part.add_covariance(np.loadtxt(xi_cov)) + return part + elif name == "pseudo_cl": + if pseudo_cl_cov is not None: + part.add_covariance(_pseudo_cl_cov_block(pseudo_cl_cov, pseudo_cl_cov_hdu)) + return part + if placeholder_var is None: + raise ValueError( + f"the {name!r} part carries no covariance and no covariance input was " + f"given (--xi-cov / --pseudo-cl-cov). Supply the block, or pass " + "--allow-placeholder to attach a documented diagonal placeholder." + ) + part.add_covariance(np.full(len(part.mean), float(placeholder_var))) + return part + + +def assemble_sacc( + version, + part_paths, + out_path, + *, + expected=None, + xi_cov=None, + pseudo_cl_cov=None, + pseudo_cl_cov_hdu="COVAR_FULL", + placeholder_var=None, +): + """Assemble ``{version}.sacc`` from the per-statistic ``part_paths`` mapping. + + Parameters + ---------- + version : str + Catalogue version (stored in the assembled file's metadata). + part_paths : dict + ``{statistic: path}`` with statistic in :data:`CANONICAL`. Only the + present statistics are assembled; order is forced to canonical. + out_path : str + Destination ``{version}.sacc``. + expected : sequence of str, optional + Statistics that MUST be present in ``part_paths`` (from the caller's + config toggles). Raises loudly if any is missing or has no path — so a + typo'd input keyword (``cosebi`` for ``cosebis``) can't silently drop a + statistic from the terminal file. Names not in :data:`CANONICAL` are + rejected too (catches a typo in the expected list itself). + xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var + Covariance sourcing — see the module docstring. + """ + if expected is not None: + unknown = [name for name in expected if name not in CANONICAL] + if unknown: + raise ValueError( + f"expected parts {unknown} are not assemblable statistics; " + f"valid names are {CANONICAL}" + ) + missing = [name for name in expected if not part_paths.get(name)] + if missing: + raise ValueError( + f"expected parts {missing} missing from part_paths for {version} " + f"(got {sorted(part_paths)}); a required statistic would be " + "silently dropped from the terminal analysis file" + ) + parts = [] + nz = metadata = None + for name in CANONICAL: + path = part_paths.get(name) + if path is None: + continue + part = sacc_io.load(path) + if nz is None: + # The nz tracers + metadata are identical across parts (same version); + # take them from the first loaded part for the assembled file. + nz = {i: sacc_io.get_nz(part, i) for i in range(_n_source_bins(part))} + metadata = dict(part.metadata) + parts.append( + _attach_cov( + part, name, xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var + ) + ) + if not parts: + raise ValueError(f"no parts found for {version}: {part_paths}") + s = assemble_analysis_sacc(nz, metadata, parts) + sacc_io.save(s, out_path) + print(f"Assembled {len(parts)} parts -> {out_path}") + return s + + +def _n_source_bins(part): + """Count the ``source_{i}`` NZ tracers on a part (single-bin round -> 1).""" + i = 0 + while sacc_io.source_name(i) in part.tracers: + i += 1 + return i + + +def _from_snakemake(smk): + p = smk.params + inp = smk.input + part_paths = { + name: getattr(inp, name) + for name in CANONICAL + if hasattr(inp, name) and getattr(inp, name) + } + # The rule declares which statistics it wired (from its config toggles); a + # typo in an input keyword drops the part from part_paths above, so validate + # against this expected list rather than trusting the hasattr filter. + expected = list(p["expected"]) + assemble_sacc( + version=p["version"], + part_paths=part_paths, + out_path=str(smk.output[0]), + expected=expected, + xi_cov=getattr(inp, "xi_cov", None), + pseudo_cl_cov=getattr(inp, "pseudo_cl_cov", None), + pseudo_cl_cov_hdu=p.get("pseudo_cl_cov_hdu", "COVAR_FULL"), + placeholder_var=p.get("placeholder_var", None), + ) + + +def _from_cli(argv=None): + ap = argparse.ArgumentParser( + description="Assemble the terminal {version}.sacc from per-statistic parts." + ) + ap.add_argument("--version", required=True, help="Catalogue version") + ap.add_argument("--out", required=True, help="Output {version}.sacc path") + for name in CANONICAL: + ap.add_argument( + f"--{name.replace('_', '-')}", default=None, help=f"{name} part" + ) + ap.add_argument("--xi-cov", default=None, help="CosmoCov ξ covariance .txt") + ap.add_argument( + "--pseudo-cl-cov", + default=None, + help="NaMaster/OneCovariance pseudo-Cℓ cov FITS", + ) + ap.add_argument( + "--pseudo-cl-cov-hdu", + default="COVAR_FULL", + help="HDU name for a single dense pseudo-Cℓ cov (EE/BB/EB-spanning)", + ) + ap.add_argument( + "--allow-placeholder", + type=float, + default=None, + metavar="VAR", + help="Attach a diagonal placeholder (variance VAR) to cov-less parts", + ) + a = ap.parse_args(argv) + part_paths = {name: getattr(a, name) for name in CANONICAL if getattr(a, name)} + assemble_sacc( + version=a.version, + part_paths=part_paths, + out_path=a.out, + xi_cov=a.xi_cov, + pseudo_cl_cov=a.pseudo_cl_cov, + pseudo_cl_cov_hdu=a.pseudo_cl_cov_hdu, + placeholder_var=a.allow_placeholder, + ) + + +if __name__ == "__main__": + try: + snakemake # noqa: F821 — injected by Snakemake's script: directive + except NameError: + _from_cli() + else: + _from_snakemake(snakemake) # noqa: F821 diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index 182cda99..b90ebc5b 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -3,8 +3,11 @@ Compute + plot rule (per version). plot_cosebis calls calculate_cosebis over a fine integration binning (the 2000-bin TreeCorr is the dominant cost) and evaluates the configured scale cuts. Writes the {version}_eb_..._data.npz -COSEBIs data product (declared output) plus figures, and the per-version -COSEBIs PTE that cv_summarize_bmodes collects. +COSEBIs data product plus figures, and the per-version COSEBIs PTE that +cv_summarize_bmodes collects. It also writes the born-as-SACC COSEBIs part +({version}_cosebis.sacc, the fiducial scale cut's {En,Bn,cov}) that the +assemble_sacc rule consumes — the multi-cut .npz sidecar stays the diagnostic +PTE scan. """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs @@ -13,8 +16,9 @@ _unbuffer_streams() cv = make_cv(snakemake) p = snakemake.params +version = p["version"] cv.plot_cosebis( - version=p["version"], + version=version, min_sep_int=p["min_sep_int"], max_sep_int=p["max_sep_int"], nbins_int=p["nbins_int"], @@ -23,4 +27,12 @@ scale_cuts=[tuple(sc) for sc in p["scale_cuts"]], fiducial_scale_cut=tuple(p["fiducial_scale_cut"]), ) +# Born-as-SACC COSEBIs part at the fiducial scale cut (plot_cosebis stored the +# multi-cut results on the instance). +cv.cosebis_to_sacc_part( + version, + snakemake.output["sacc"], + cv._cosebis_results[version], + fiducial_scale_cut=tuple(p["fiducial_scale_cut"]), +) verify_outputs(snakemake) diff --git a/workflow/scripts/cv_pseudo_cl.py b/workflow/scripts/cv_pseudo_cl.py index cf04e8e8..43923e4e 100644 --- a/workflow/scripts/cv_pseudo_cl.py +++ b/workflow/scripts/cv_pseudo_cl.py @@ -1,8 +1,10 @@ """Rule cv_pseudo_cl: harmonic-space pseudo-Cl B-mode spectra. -plot_pseudo_cl triggers calculate_pseudo_cl, which writes pseudo_cl_{version}.fits -for every version (the BB spectrum cv_summarize_bmodes reads) and the cell_ee.png -figure. The per-version FITS files are the declared outputs. +plot_pseudo_cl triggers calculate_pseudo_cl, which writes the born-as-SACC +pseudo_cl_{version}.sacc part for every version (EE/BB/EB with the shared +bandpower window — the BB spectrum cv_summarize_bmodes reads) and the +cell_ee.png figure. The per-version SACC parts are the declared outputs and +feed both cv_summarize_bmodes and the assemble_sacc rule. """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index d15a763f..7a453b16 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -3,9 +3,10 @@ Compute + plot rule (per version). plot_pure_eb calls calculate_pure_eb, which runs two TreeCorr correlations (reporting + integration binning); the reporting binning reuses the cv_2pcf data vector via calculate_2pcf's skip-if-exists -path. Writes the {version}_eb_..._data.npz data product (declared output) plus -companion figures, and the per-version E/B PTEs that cv_summarize_bmodes -collects. +path. Writes the {version}_eb_..._data.npz data product plus companion figures, +and the per-version E/B PTEs that cv_summarize_bmodes collects. It also writes +the born-as-SACC pure-E/B part ({version}_pure_eb.sacc, the six PURE_KEYS blocks ++ covariance) that the assemble_sacc rule consumes. """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs @@ -14,12 +15,15 @@ _unbuffer_streams() cv = make_cv(snakemake) p = snakemake.params +version = p["version"] cv.plot_pure_eb( - versions=[p["version"]], + versions=[version], min_sep_int=p["min_sep_int"], max_sep_int=p["max_sep_int"], nbins_int=p["nbins_int"], fiducial_xip_scale_cut=tuple(p["fiducial_scale_cut"]), fiducial_xim_scale_cut=tuple(p["fiducial_scale_cut"]), ) +# Born-as-SACC pure-E/B part (plot_pure_eb stored the results on the instance). +cv.pure_eb_to_sacc_part(version, snakemake.output["sacc"], cv._pure_eb_results[version]) verify_outputs(snakemake) diff --git a/workflow/scripts/cv_summarize_bmodes.py b/workflow/scripts/cv_summarize_bmodes.py index 90df5999..0fe7b95a 100644 --- a/workflow/scripts/cv_summarize_bmodes.py +++ b/workflow/scripts/cv_summarize_bmodes.py @@ -3,8 +3,8 @@ The terminal diagnostic. summarize_bmodes reads the in-memory _pure_eb_results / _cosebis_results / _pseudo_cls dicts, which are populated by plot_pure_eb / plot_cosebis / plot_pseudo_cl. The per-version E/B and COSEBIs -npz products and the pseudo-Cl FITS are declared as inputs (so the DAG forces -those rules first), but the summary still needs the live result objects (it +npz products and the pseudo-Cl SACC parts are declared as inputs (so the DAG +forces those rules first), but the summary still needs the live result objects (it reads each version's TreeCorr `gg`, which the npz cannot hold). So this rule re-runs the three B-mode methods in-process: they reload the existing 2pcf / data-vector files via their skip-if-exists paths and recompute only the cheap diff --git a/workflow/scripts/generate_cosmocov_ini.py b/workflow/scripts/generate_cosmocov_ini.py new file mode 100644 index 00000000..11e785cf --- /dev/null +++ b/workflow/scripts/generate_cosmocov_ini.py @@ -0,0 +1,149 @@ +"""Generate a CosmoCov ``.ini`` for one (version, blind, grid, flavour, mask). + +CLI refactor of the former ``rule covariance_ini`` heredoc. Cosmology is read +from the frozen ``planck18.json`` snapshot (the cosmology_snapshot lc output — +source of truth is cs_util.cosmo.PLANCK18); survey (area, n_eff, +sigma_e) from the catalog config's per-version ``cov_th``; n(z) via the same +path convention as workflow/common.build_redshift_path; the footprint mask +power spectrum is passed explicitly (empty string for the unmasked variant). +The emitted ``.ini`` is byte-compatible with the paper's covariance_ini rule. + + python generate_cosmocov_ini.py \ + --version SP_v1.4.6.3_leak_corr --blind A \ + --planck18-json /planck18.json \ + --cat-config \ + --min-sep 0.5 --max-sep 300.0 --nbins 1000 --gaussian g \ + --mask-cls \ + --out-ini +""" + +import argparse +import json +import os +import re + +import yaml + + +def build_redshift_path(version, blind): + """Replicate workflow/common.build_redshift_path.""" + base_version = re.sub(r"_leak_corr$", "", version) + base_version = re.sub(r"_ecut\d+", "", base_version) + if "v1.4.11" in base_version: + base_version = "SP_v1.4.6" + version_dir = base_version.replace("SP_", "") + return ( + f"/n17data/sguerrini/UNIONS/WL/nz/{version_dir}/nz_{base_version}_{blind}.txt" + ) + + +INI_TEMPLATE = """\ +# +# Cosmological parameters +# +Omega_m : {Omega_m} +Omega_v : {Omega_v} +sigma_8 : {sigma_8} +n_spec : {n_s} +w0 : -1 +wa : 0 +omb : {Omega_b} +h0 : {h} + + +# Survey and galaxy parameters +# +# area in degrees +# n_gal,lens_n_gal in gals/arcmin^2 + +area : {area} +sourcephotoz : multihisto +lensphotoz : multihisto +source_tomobins : 1 +lens_tomobins : 1 +sigma_e : {sigma_e} +source_n_gal : {n_e} +lens_n_gal : {n_e} + + +shear_REDSHIFT_FILE : {nz} +clustering_REDSHIFT_FILE : {nz} +c_footprint_file : {mask} + + +# IA parameters +IA : 1 +A_ia : 0.0 +eta_ia : 0.0 + + +# Covariance parameters +# +# tmin,tmax in arcminutes +tmin : {min_sep} +tmax : {max_sep} +ntheta : {nbins} +ng : {ng} +cng : {ng} + + +outdir : ./ +filename : cov_tmp +ss : true +ls : false +ll : false +""" + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--version", required=True) + ap.add_argument("--blind", default="A") + ap.add_argument("--planck18-json", required=True) + ap.add_argument("--cat-config", required=True) + ap.add_argument("--min-sep", required=True, help="tmin arcmin (string, e.g. 0.5)") + ap.add_argument("--max-sep", required=True, help="tmax arcmin (string, e.g. 300.0)") + ap.add_argument("--nbins", required=True, help="ntheta (string, e.g. 1000)") + ap.add_argument("--gaussian", required=True, choices=["g", "ng"]) + ap.add_argument( + "--mask-cls", default="", help="footprint mask Cl path ('' = unmasked)" + ) + ap.add_argument("--out-ini", required=True) + a = ap.parse_args(argv) + + with open(a.planck18_json) as f: + cosmo = json.load(f) + with open(a.cat_config) as f: + cat_config = yaml.safe_load(f) + + base_version = a.version.replace("_leak_corr", "") + cov_th = cat_config[base_version]["cov_th"] + + ng_value = "1" if a.gaussian == "ng" else "0" + + ini = INI_TEMPLATE.format( + Omega_m=cosmo["Omega_m"], + Omega_v=cosmo["Omega_v"], + sigma_8=cosmo["sigma_8"], + n_s=cosmo["n_s"], + Omega_b=cosmo["Omega_b"], + h=cosmo["h"], + area=cov_th["A"], + sigma_e=cov_th["sigma_e"], + n_e=cov_th["n_e"], + nz=build_redshift_path(a.version, a.blind), + mask=a.mask_cls, + min_sep=a.min_sep, + max_sep=a.max_sep, + nbins=a.nbins, + ng=ng_value, + ) + + os.makedirs(os.path.dirname(os.path.abspath(a.out_ini)), exist_ok=True) + with open(a.out_ini, "w") as f: + f.write(ini) + print(f"Wrote {a.out_ini}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index a5c19a56..9bfabf4a 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -4,12 +4,13 @@ object supplies the parameters and the native product is renamed to the tagged output filename the rule declares; as a standalone CLI (argparse) the same compute runs from explicit flags and the primitive's native -``pseudo_cl_{ver}.fits`` is left in place under ``--out`` (no rename — each +``pseudo_cl_{ver}.sacc`` is left in place under ``--out`` (no rename — each lc/ASTRA recipe gets its own output directory, so the untagged native name is unambiguous and the primitives' skip-if-exists never collides across nbins -runs). The CLI form is what the lightcone/ASTRA recipe calls, so the -measurement is driven directly (no nested Snakemake) with lc handling -orchestration: +runs). The C_ell data vector is born as SACC (EE/BB/EB with a shared bandpower +window) — see ``sp_validation.cosmo_val.sacc_writers.pseudo_cl_to_sacc``. The +CLI form is what the lightcone/ASTRA recipe calls, so the measurement is driven +directly (no nested Snakemake) with lc handling orchestration: python generate_pseudo_cl.py \ --ver SP_v1.4.6.3_leak_corr \ @@ -28,14 +29,13 @@ import json import os -from astropy.io import fits - +from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation def generate_pseudo_cl( version: str, - output_dir: str, + out_path: str, cat_config: str, nside: int = 1024, npatch: int = 1, @@ -45,16 +45,16 @@ def generate_pseudo_cl( nbins: int = None, power: float = 0.5, ): - """Generate a pseudo-Cl data vector into ``output_dir``. + """Generate a pseudo-Cl data vector, born as a SACC part at ``out_path``. Parameters ---------- version : str Catalog version (e.g., "SP_v1.4.6_leak_corr") - output_dir : str - Directory the pseudo-Cl FITS file is written into. The primitive writes - its native ``pseudo_cl_{version}.fits`` here; callers that need a tagged - filename rename it themselves (see ``_from_snakemake``). + out_path : str + Exact destination the SACC part is *born at* — its final (possibly + tagged) name. No native-basename + rename step, so this producer's + skip-if-exists never collides with the untagged cv_pseudo_cl diagnostic. cat_config : str Path to catalog configuration YAML nside : int @@ -76,8 +76,9 @@ def generate_pseudo_cl( Returns ------- str - Path to the primitive's native ``pseudo_cl_{version}.fits`` product. + ``out_path`` (the SACC part written). """ + output_dir = os.path.dirname(out_path) os.makedirs(output_dir, exist_ok=True) blind_str = f" blind={blind}" if blind else "" @@ -135,26 +136,27 @@ def generate_pseudo_cl( cv = CosmologyValidation(**cv_kwargs) - # Calculate pseudo-Cls only (no covariance) - cv.calculate_pseudo_cl() + # Calculate pseudo-Cls only (no covariance). The data vector is born as a + # SACC part directly at out_path (its final, possibly-tagged name) — no + # shared native basename, no rename, so this producer's skip-if-exists never + # collides with the untagged cv_pseudo_cl diagnostic (which would otherwise + # let one rule adopt + delete the other's differently-blinded file). + cv.calculate_pseudo_cl(out_path=out_path) - # Report on the native product (renamed by the Snakemake caller, if any) - src_cl = os.path.join(output_dir, f"pseudo_cl_{version}.fits") - if os.path.exists(src_cl): - with fits.open(src_cl) as hdul: - data = hdul["PSEUDO_CELL"].data - n_ell = len(data["ELL"]) - print(f"Generated pseudo-Cl with {n_ell} ell bins") - print(f"ell range: [{data['ELL'].min():.1f}, {data['ELL'].max():.1f}]") - return src_cl + if os.path.exists(out_path): + s = sacc_io.load(out_path) + ell = sacc_io.get_pseudo_cl(s, (0, 0))[0] + print(f"Generated pseudo-Cl with {len(ell)} ell bins") + print(f"ell range: [{ell.min():.1f}, {ell.max():.1f}]") + return out_path def _from_snakemake(smk): p = smk.params - output_cl = smk.output.pseudo_cl - src_cl = generate_pseudo_cl( + # Born directly at the rule's declared (tagged) output — no rename step. + generate_pseudo_cl( version=p["version"], - output_dir=os.path.dirname(output_cl), + out_path=smk.output.pseudo_cl, cat_config=p["cat_config"], nside=int(p["nside"]), npatch=int(p["npatch"]), @@ -164,10 +166,6 @@ def _from_snakemake(smk): nbins=int(p["nbins"]), power=float(p.get("power", 0.5)), ) - # Snakemake declares a tagged output filename; rename the native product to it. - if os.path.exists(src_cl) and src_cl != output_cl: - os.rename(src_cl, output_cl) - print(f"Saved to: {output_cl}") def _from_cli(argv=None): @@ -220,9 +218,13 @@ def _from_cli(argv=None): with open(a.cosmo_json) as f: cosmo_params = json.load(f) + # lc/ASTRA path: --out is a per-recipe directory; the untagged native name + # is unambiguous there (each recipe gets its own tree, so no cross-nbins or + # cross-blind collision). + out_path = os.path.join(a.out, f"pseudo_cl_{a.ver}.sacc") generate_pseudo_cl( version=a.ver, - output_dir=a.out, + out_path=out_path, cat_config=a.cat_config, nside=a.nside, npatch=a.npatch, diff --git a/workflow/scripts/im_build_manifest.py b/workflow/scripts/im_build_manifest.py deleted file mode 100644 index 6f0a0faf..00000000 --- a/workflow/scripts/im_build_manifest.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python -"""Build the image-simulation campaign manifest from the sims' own records. - -This is the head of the image-simulation DAG: it reads the injected shear that -the sim campaign recorded for each requested branch, cross-checks it against the -branch's *name*, and writes a single ``manifest.yaml`` that every downstream -stage reads. The point is one home for the injected-shear facts -- amplitude, -per-branch ``(g1, g2)``, the reference branch, and the ``+/-`` pairing -- so no -literal amplitude or hard-coded branch list survives anywhere else. - -The source of truth is each branch's ``basic_info.txt``, written by the sim -campaign. The one line this parser needs looks like:: - - g_cosmic = 0.025 0.0 - -i.e. the literal key ``g_cosmic``, run-together whitespace, an ``=``, then the -two injected shear components ``g1 g2`` as space-separated floats (sign as a -leading ``-``; ``0.0`` for an un-sheared component). We parse *only* that line, -by stdlib string ops -- no YAML/regex dependency -- so the script runs inside -the container with nothing but the standard library. - -Branch names follow the ``1{X}2{Y}`` convention: the character after ``1`` is -the g1 sign, the character after ``2`` is the g2 sign, each one of ``p`` (+), -``m`` (-), ``z`` (0). So ``1p2z`` injects ``(+|g|, 0)``, ``1z2m`` injects -``(0, -|g|)``, ``1z2z`` is the un-sheared reference. The suffix -(``_grid_1`` etc.) is appended by the workflow and is not part of the sign code. - -Validation (all fail-loud, each message naming the offending file and field): - -* every branch's parsed ``(g1, g2)`` sign/axis matches its name's sign code; -* the reference branch parses to exactly ``(0, 0)``; -* the derived ``|g|`` (the single nonzero magnitude of a sheared branch) is - equal across all four sheared branches -- one injected amplitude for the - whole campaign. - -Only the branches this run requests are read and validated. -""" - -import argparse -import os -import sys - -import yaml - -# Branch-name sign code: the character after "1" (g1) and after "2" (g2). -_SIGN = {"p": +1, "m": -1, "z": 0} - - -def die(msg): - """Abort with a clear, prefixed message on stderr.""" - sys.exit(f"im_build_manifest: {msg}") - - -def basic_info_path(input_sims_base, branch): - """Path to a branch's basic_info.txt (the sim campaign's own record).""" - return os.path.join(input_sims_base, branch, "basic_info.txt") - - -def parse_g_cosmic(path): - """Parse the ``g_cosmic = g1 g2`` line from a basic_info.txt file. - - Returns ``(g1, g2)`` as floats. Fails loud, naming the file, if the line - is absent, malformed, or does not carry exactly two float components. - """ - if not os.path.isfile(path): - die(f"basic_info.txt not found: {path}") - - with open(path) as fh: - lines = fh.readlines() - - matches = [ln for ln in lines if ln.split("=", 1)[0].strip() == "g_cosmic"] - if not matches: - die(f"no 'g_cosmic' line in {path}") - if len(matches) > 1: - die(f"multiple 'g_cosmic' lines in {path}") - - rhs = matches[0].split("=", 1)[1].split() - if len(rhs) != 2: - die( - f"'g_cosmic' in {path}: expected two components 'g1 g2', " - f"got {len(rhs)}: {matches[0].strip()!r}" - ) - try: - return float(rhs[0]), float(rhs[1]) - except ValueError: - die(f"'g_cosmic' in {path}: components not floats: {matches[0].strip()!r}") - - -def sign_code(branch): - """Extract the ``(g1_sign, g2_sign)`` code from a ``1{X}2{Y}`` branch name. - - ``branch`` is the bare sign code (e.g. ``1p2z``), suffix already stripped. - Fails loud if the name does not match the convention. - """ - if ( - len(branch) != 4 - or branch[0] != "1" - or branch[2] != "2" - or branch[1] not in _SIGN - or branch[3] not in _SIGN - ): - die( - f"branch name {branch!r} does not match the 1{{X}}2{{Y}} convention " - f"(X, Y each one of p/m/z)" - ) - return _SIGN[branch[1]], _SIGN[branch[3]] - - -def build_manifest(input_sims_base, sims_type, num, branches): - """Parse + validate every requested branch, return the manifest dict. - - ``branches`` are bare sign codes (``1z2z``, ``1p2z``, ...); the on-disk - directory name is ``{branch}{suffix}`` with ``suffix`` derived from - ``sims_type``/``num`` exactly as the workflow builds it. - """ - suffix = f"_{sims_type}_{num}" if sims_type == "grid" else f"_{num}" - - parsed = {} # branch -> (g1, g2) from basic_info.txt - for branch in branches: - dirname = f"{branch}{suffix}" - g1, g2 = parse_g_cosmic(basic_info_path(input_sims_base, dirname)) - s1, s2 = sign_code(branch) - - # Sign/axis must agree with the name: a component is nonzero iff its - # sign code is nonzero, and its sign matches. - for comp, (g, s) in enumerate(((g1, s1), (g2, s2)), start=1): - path = basic_info_path(input_sims_base, dirname) - if s == 0 and g != 0.0: - die( - f"{path}: branch {branch!r} names g{comp} un-sheared (z) but " - f"g_cosmic gives g{comp} = {g}" - ) - if s != 0 and (g == 0.0 or (g > 0) != (s > 0)): - die( - f"{path}: branch {branch!r} names g{comp} sign {'+' if s > 0 else '-'} " - f"but g_cosmic gives g{comp} = {g}" - ) - parsed[branch] = (g1, g2) - - # Reference branch: the one whose name codes (0, 0). Must exist and be (0,0). - refs = [b for b in branches if sign_code(b) == (0, 0)] - if len(refs) != 1: - die( - f"expected exactly one reference branch (name code 1z2z) among " - f"{branches}, found {refs}" - ) - reference = refs[0] - if parsed[reference] != (0.0, 0.0): - die( - f"{basic_info_path(input_sims_base, f'{reference}{suffix}')}: reference " - f"branch {reference!r} must inject (0, 0), got {parsed[reference]}" - ) - - # Derived amplitude: the single nonzero magnitude of each sheared branch, - # cross-checked equal across all four. - amplitudes = {} # branch -> |g| - for branch in branches: - if branch == reference: - continue - g1, g2 = parsed[branch] - amplitudes[branch] = abs(g1) if g1 != 0.0 else abs(g2) - distinct = sorted(set(amplitudes.values())) - if len(distinct) != 1: - die( - "injected |g| differs across sheared branches (must be one campaign " - f"amplitude): {amplitudes} " - f"[files under {input_sims_base}/{suffix}/basic_info.txt]" - ) - shear_amplitude = distinct[0] - - # Pairs: (+component, -component) for each sheared axis, in branch order so - # the estimator's per-pair processing order is stable. - plus = {} # component (0/1) -> branch with +|g| on that component - minus = {} - for branch in branches: - if branch == reference: - continue - s1, s2 = sign_code(branch) - comp = 0 if s1 != 0 else 1 - (plus if (s1 or s2) > 0 else minus)[comp] = branch - pairs = [ - {"plus": plus[comp], "minus": minus[comp], "component": comp} - for comp in sorted(set(plus) & set(minus)) - ] - - return { - "input_sims_base": input_sims_base, - "sims_type": sims_type, - "num": num, - "shear_amplitude": shear_amplitude, - "reference": reference, - # Branch order preserved (dict insertion order round-trips through - # yaml.safe_dump with sort_keys=False) so downstream load order is fixed. - "branches": { - branch: {"g1": parsed[branch][0], "g2": parsed[branch][1]} - for branch in branches - }, - "pairs": pairs, - } - - -def parse_args(): - p = argparse.ArgumentParser(description=__doc__) - p.add_argument( - "--input-sims-base", - required=True, - help="root under which each branch dir holds basic_info.txt", - ) - p.add_argument( - "--sims-type", required=True, help="'grid' -> _grid_{num} suffix, else _{num}" - ) - p.add_argument("--num", required=True, type=int, help="run number") - p.add_argument( - "--branch", - required=True, - action="append", - dest="branches", - help="bare branch sign code (1z2z, 1p2z, ...); repeatable", - ) - p.add_argument("-o", "--output", required=True, help="manifest.yaml output path") - return p.parse_args() - - -def main(): - args = parse_args() - manifest = build_manifest( - args.input_sims_base, args.sims_type, args.num, args.branches - ) - os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) - with open(args.output, "w") as fh: - yaml.safe_dump(manifest, fh, sort_keys=False) - print(f"im_build_manifest: wrote {args.output}") - print(f" shear_amplitude = {manifest['shear_amplitude']}") - print(f" reference = {manifest['reference']}") - print(f" branches = {list(manifest['branches'])}") - for pair in manifest["pairs"]: - print( - f" pair g{pair['component'] + 1}: {pair['plus']} (+) / {pair['minus']} (-)" - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/workflow/scripts/im_compose_mask.py b/workflow/scripts/im_compose_mask.py deleted file mode 100644 index fb602a11..00000000 --- a/workflow/scripts/im_compose_mask.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python -"""Compose the image-sim mask/calibration config from base + declared overlay. - -The image-sim calibration differs from the data calibration in only a handful -of places (input path, dropped coverage cuts, unweighted global response, -additive-bias off). Instead of maintaining a second full copy of the config -- -which can silently drift from the base it was branched from -- we keep the -*data* config (``mask_v1.X.9.yaml``) as the single home for the shared cuts and -declare the sim-specific delta in an overlay -(``mask_v1.X.9_im_sim.overlay.yaml``). This script applies the overlay to the -base and emits the resolved config, which is byte-for-byte the committed runtime -file ``mask_v1.X.9_im_sim.yaml``. A test locks that equality, so the declaration -and the runtime file cannot diverge. - -The overlay is a list of block operations on the base *text* (not on parsed -YAML), so the resolved file preserves the base's exact formatting and comments --- the property that makes byte-identity with a hand-maintained runtime file -achievable, and the delta legible as a plain diff. Each op: - -* ``drop:`` remove a verbatim block of base text; -* ``replace:`` / ``with:`` swap a verbatim block for new text. - -Every anchor (the ``drop`` block, or a ``replace`` block) must occur **exactly -once** in the base -- zero or multiple matches is a hard error, so an overlay -that has fallen out of sync with the base fails loudly instead of composing -something wrong. ``why`` is prose for the human reader and is ignored here. - -Stdlib + PyYAML only, so it runs inside the sp_validation container with nothing -extra. -""" - -import argparse -import os -import sys - -import yaml - - -def die(msg): - """Abort with a clear, prefixed message on stderr.""" - sys.exit(f"im_compose_mask: {msg}") - - -def _apply_once(text, anchor, replacement, *, kind, i): - """Replace the single occurrence of ``anchor`` in ``text`` with ``replacement``. - - ``anchor`` must occur exactly once; anything else (missing, or ambiguous) - means the overlay no longer matches the base and is a hard error naming the - offending op. - """ - n = text.count(anchor) - if n != 1: - die( - f"op {i} ({kind}): anchor block occurs {n} time(s) in the base, " - "expected exactly 1 -- the overlay is out of sync with the base.\n" - f"--- anchor ---\n{anchor}\n--------------" - ) - return text.replace(anchor, replacement) - - -def compose(base_text, overlay): - """Apply ``overlay['ops']`` to ``base_text`` and return the resolved text.""" - text = base_text - for i, op in enumerate(overlay["ops"]): - if "drop" in op: - text = _apply_once(text, op["drop"], "", kind="drop", i=i) - elif "replace" in op: - if "with" not in op: - die(f"op {i} (replace): missing 'with:' block") - text = _apply_once(text, op["replace"], op["with"], kind="replace", i=i) - else: - die(f"op {i}: needs a 'drop:' or 'replace:'/'with:' block") - return text - - -def main(argv=None): - ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - ap.add_argument( - "overlay", - help="overlay yaml declaring base + block ops " - "(e.g. mask_v1.X.9_im_sim.overlay.yaml)", - ) - ap.add_argument( - "-o", - "--output", - help="write resolved config here; default: stdout", - ) - args = ap.parse_args(argv) - - with open(args.overlay) as fh: - overlay = yaml.safe_load(fh) - - # The base path is stated in the overlay, relative to the overlay's own dir, - # so the pair travels together (both live in config/calibration/). - base_path = os.path.join(os.path.dirname(args.overlay), overlay["base"]) - with open(base_path) as fh: - base_text = fh.read() - - resolved = compose(base_text, overlay) - - if args.output: - with open(args.output, "w") as fh: - fh.write(resolved) - else: - sys.stdout.write(resolved) - - -if __name__ == "__main__": - main() diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 2e1ccabf..a773d039 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -13,14 +13,21 @@ --out The measurement itself is unchanged — ``CosmologyValidation.calculate_2pcf`` -does the TreeCorr work and writes the ``.txt`` dump plus ξ+/ξ- FITS files into -``output_dir``. ``output_dir`` is passed explicitly (rather than via the +does the TreeCorr work and writes the ``.txt`` dump (a raw byproduct the +covariance machinery reads back). The analysis ξ± data product is then born as +SACC here: ``{ver}_xi_coarse.sacc``, a *part* on the coarse grid via +``xi_to_sacc(grid="coarse", ...)`` carrying ``theta_nom``/``npairs``/``weight`` +tags but NO covariance (the ξ block is supplied at assembly from the CosmoCov +theory covariance). ``output_dir`` is passed explicitly (rather than via the ``COSMO_VAL`` env hook) so lc can point each run at its own ``{output}`` tree. """ import argparse +import os +from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation +from sp_validation.cosmo_val.sacc_writers import xi_to_sacc def run_2pcf( @@ -31,30 +38,57 @@ def run_2pcf( npatch, cat_config, output_dir, - save_fits=True, + sacc_out=None, ): - """Measure ξ±(θ) for ``ver`` and write it under ``output_dir``. + """Measure ξ±(θ) for ``ver`` and write its coarse SACC part. Parameters mirror the TreeCorr reporting/integration grids: ``min_sep`` / ``max_sep`` in arcmin, ``nbins`` logarithmic bins, ``npatch`` spatial patches (1 for the paper fiducial). ``cat_config`` is an absolute path to the catalog configuration; ``output_dir`` overrides - ``cat_config['paths']['output']`` so products land where lc expects. + ``cat_config['paths']['output']`` so the ``.txt`` byproduct lands where lc + expects. ``sacc_out`` is the exact destination for the coarse ξ± SACC part + (the Snakemake-declared output); it defaults to ``{ver}_xi_coarse.sacc`` + under the resolved output directory for the CLI path. + + Returns + ------- + treecorr.GGCorrelation + The measured correlation object (also the source of the SACC part). """ cv = CosmologyValidation( versions=[ver], catalog_config=cat_config, output_dir=output_dir, ) - return cv.calculate_2pcf( + gg = cv.calculate_2pcf( ver=ver, npatch=npatch, - save_fits=save_fits, min_sep=min_sep, max_sep=max_sep, nbins=nbins, ) + # Born-as-SACC coarse ξ± part: no covariance here (added at assembly from + # the CosmoCov theory covariance). theta = meanr; theta_nom = rnom. + s = xi_to_sacc( + cv.sacc_nz(ver), + cv.sacc_metadata(ver), + gg.meanr, + gg.xip, + gg.xim, + grid="coarse", + theta_nom=gg.rnom, + npairs=gg.npairs, + weight=gg.weight, + ) + out_path = sacc_out or os.path.join( + output_dir or cv.cc["paths"]["output"], f"{ver}_xi_coarse.sacc" + ) + sacc_io.save(s, out_path) + print(f"Wrote coarse ξ± SACC part: {out_path}") + return gg + def _from_snakemake(smk): p = smk.params @@ -70,7 +104,9 @@ def _from_snakemake(smk): # class defaults (./cat_config.yaml, COSMO_VAL env) otherwise. cat_config=p.get("cat_config", "./cat_config.yaml"), output_dir=p.get("output_dir", None), - save_fits=True, + # Write the SACC part exactly where the rule declares it (the .txt + # byproduct still lands under the resolved output dir via _output_path). + sacc_out=smk.output["xi_coarse"], ) @@ -97,7 +133,6 @@ def _from_cli(argv=None): "--cat-config", required=True, help="Absolute path to cat_config.yaml" ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") - ap.add_argument("--no-fits", action="store_true", help="Skip ξ+/ξ- FITS export") a = ap.parse_args(argv) run_2pcf( ver=a.ver, @@ -107,7 +142,6 @@ def _from_cli(argv=None): npatch=a.npatch, cat_config=a.cat_config, output_dir=a.out, - save_fits=not a.no_fits, ) diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py index eb31d2c7..ed3660bf 100644 --- a/workflow/scripts/run_2pcf_highres.py +++ b/workflow/scripts/run_2pcf_highres.py @@ -27,6 +27,12 @@ import treecorr from astropy.io import fits +# sacc_io depends only on numpy + sacc (no healpy/cs_util), so the born-as-SACC +# fine ξ± write works on the bare-host MPI path too, where the full cosmo_val +# stack is unavailable. +from sp_validation import sacc_io +from sp_validation.cosmo_val.sacc_writers import xi_to_sacc + try: # In-container path: full sp_validation stack available. from sp_validation.cosmo_val import CosmologyValidation @@ -76,6 +82,7 @@ E1_COL = None E2_COL = None W_COL = None +REDSHIFT_PATH = None # n(z) file for the SACC tracer TMIN = None # arcmin TMAX = None # arcmin NBINS = None @@ -196,35 +203,46 @@ def compute_patch_centers(ra, dec): del cat_sub -def write_xi_fits(gg, prefix, xi_data): - """Write ξ+ or ξ- to FITS matching CosmologyValidation format.""" - out_path = os.path.join( - OUTPUT_DIR, - f"{prefix}_{VERSION}_minsep={TMIN}_maxsep={TMAX}_nbins={NBINS}_npatch=1.fits", +def write_xi_fine_sacc(gg): + """Write the terminal fine-grid ξ± SACC part (``{version}_xi_fine.sacc``). + + This is a terminal product in its own right — COSEBIs and pure-E/B consume + it. It carries a ``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim`` + (npatch=1 leaves shot-noise variance as the only covariance estimate). + Both run paths land here: in-container this uses the full SACC stack; on the + bare-host MPI run only ``sacc_io`` + the n(z) file are needed (no healpy). + """ + z, nz = np.loadtxt(REDSHIFT_PATH, unpack=True) + metadata = { + "catalogue_version": VERSION, + "sp_validation_version": _sp_validation_version(), + "npatch": 1, + } + s = xi_to_sacc( + {0: (z, nz)}, + metadata, + gg.meanr, + gg.xip, + gg.xim, + grid="fine", + theta_nom=gg.rnom, + variances=np.concatenate([gg.varxip, gg.varxim]), ) - n = len(xi_data) - cols = [ - fits.Column(name="BIN1", format="K", array=np.ones(n, dtype=int)), - fits.Column(name="BIN2", format="K", array=np.ones(n, dtype=int)), - fits.Column(name="ANGBIN", format="K", array=np.arange(1, n + 1)), - fits.Column(name="VALUE", format="D", array=xi_data), - fits.Column(name="ANG", format="D", unit="arcmin", array=gg.meanr), - ] - ext_name = "XI_PLUS" if "plus" in prefix else "XI_MINUS" - hdu = fits.BinTableHDU.from_columns(cols, name=ext_name) - for key, val in { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - }.items(): - hdu.header[key] = val - hdu.writeto(out_path, overwrite=True) + out_path = os.path.join(OUTPUT_DIR, f"{VERSION}_xi_fine.sacc") + sacc_io.save(s, out_path) log(f" Wrote {out_path}") +def _sp_validation_version(): + """Best-effort package version for the SACC metadata (empty if unavailable).""" + try: + from sp_validation import __version__ + + return __version__ + except Exception: + return "" + + def resolve_shear_config(cat_config_path, version): """Standalone shear-config resolver (bare-host fallback for CosmologyValidation). @@ -274,7 +292,7 @@ def resolve_paths(ver): def main(): - global CAT_PATH, VERSION, E1_COL, E2_COL, W_COL + global CAT_PATH, VERSION, E1_COL, E2_COL, W_COL, REDSHIFT_PATH global TMIN, TMAX, NBINS, NPATCH, OUTPUT_DIR, PATCH_FILE args = parse_args() @@ -301,6 +319,7 @@ def main(): E1_COL = shear_cfg["e1_col"] E2_COL = shear_cfg["e2_col"] W_COL = shear_cfg["w_col"] + REDSHIFT_PATH = shear_cfg["redshift_path"] PATCH_FILE = os.path.join( OUTPUT_DIR, @@ -383,8 +402,7 @@ def main(): gg.write(out_txt, write_patch_results=False, write_cov=False) log(f" Wrote {out_txt}") - write_xi_fits(gg, "xi_plus", gg.xip) - write_xi_fits(gg, "xi_minus", gg.xim) + write_xi_fine_sacc(gg) elapsed = time.time() - t0 log(f"Done! Total time: {elapsed / 3600:.1f}h ({elapsed:.0f}s)") diff --git a/workflow/scripts/run_cosmocov_chain.sh b/workflow/scripts/run_cosmocov_chain.sh new file mode 100644 index 00000000..80a80370 --- /dev/null +++ b/workflow/scripts/run_cosmocov_chain.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# CosmoCov covariance chain (lc-native, container:none recipe). +# +# Faithful port of covariance_ini -> covariance_cosmocov (x3 blocks) -> +# covariance_cat -> covariance_process. The CosmoCov C++ binary runs on the +# bare host (module load gcc/intelpython/openmpi, as in the original +# container:None rule); the .ini generation and cosmocov_process step run inside +# the sp_validation apptainer container. The 3 shear-shear blocks (++,--,+-) are +# independent and run in parallel. +# +# Usage: +# run_cosmocov_chain.sh --version SP_v1.4.6.3_leak_corr --blind A \ +# --min-sep 0.5 --max-sep 300.0 --nbins 1000 --gaussian g \ +# --planck18-json /planck18.json \ +# --cat-config --mask-cls \ +# --out +set -euo pipefail + +CONTAINER=/n17data/cdaley/containers/containers/ +WT=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra +SRC=$WT/src +BIND=/home,/scratch,/automnt,/n17data,/n23data1,/n09data +COSMOCOV=/n23data1/n06data/lgoh/scratch/UNIONS/CosmoCov/covs/cov + +VERSION=""; BLIND="A"; MINSEP=""; MAXSEP=""; NBINS=""; GAUSSIAN="" +PLANCK18=""; CATCONFIG=""; MASKCLS=""; OUT="" +while [ $# -gt 0 ]; do + case "$1" in + --version) VERSION="$2"; shift 2;; + --blind) BLIND="$2"; shift 2;; + --min-sep) MINSEP="$2"; shift 2;; + --max-sep) MAXSEP="$2"; shift 2;; + --nbins) NBINS="$2"; shift 2;; + --gaussian) GAUSSIAN="$2"; shift 2;; + --planck18-json) PLANCK18="$2"; shift 2;; + --cat-config) CATCONFIG="$2"; shift 2;; + --mask-cls) MASKCLS="$2"; shift 2;; + --cosmocov) COSMOCOV="$2"; shift 2;; + --out) OUT="$2"; shift 2;; + *) echo "unknown arg: $1" >&2; exit 2;; + esac +done + +mkdir -p "$OUT" +# Absolutize OUT before any `cd` below: the CosmoCov binary writes its block +# files into cwd, so we cd into OUT (line ~61); every other OUT-relative path +# ($INI, block logs, covariance.txt, cosmocov_process output) must therefore be +# absolute or it re-resolves against the new cwd and double-nests. lc templates +# {output} as a project-relative path, so this makes the recipe robust to both +# relative (lc) and absolute (direct-run) --out. +OUT="$(cd "$OUT" && pwd)" +INI="$OUT/covariance.ini" + +echo "[cosmocov] generating .ini" +apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" "$CONTAINER" \ + /usr/local/bin/python "$WT/workflow/scripts/generate_cosmocov_ini.py" \ + --version "$VERSION" --blind "$BLIND" \ + --planck18-json "$PLANCK18" --cat-config "$CATCONFIG" \ + --min-sep "$MINSEP" --max-sep "$MAXSEP" --nbins "$NBINS" --gaussian "$GAUSSIAN" \ + --mask-cls "$MASKCLS" --out-ini "$INI" + +echo "[cosmocov] loading modules + running 3 blocks (parallel)" +source /etc/profile.d/modules.sh +module unload gcc 2>/dev/null || true; module load gcc +module unload intelpython 2>/dev/null || true; module load intelpython/3-2024.1.0 +module load openmpi + +cd "$OUT" +# BLOCK_PAIRS = [("++","1"), ("--","2"), ("+-","3")] — one CosmoCov invocation per block +for idx in 1 2 3; do + ( "$COSMOCOV" "$idx" "$INI" > "$OUT/cosmocov_block_${idx}.log" 2>&1 ) & +done +wait + +# Concatenate blocks in BLOCK_PAIRS order (++, --, +-) — as covariance_cat does +CAT="$OUT/covariance.txt" +: > "$CAT" +for pm_idx in "++:1" "--:2" "+-:3"; do + pm="${pm_idx%%:*}"; idx="${pm_idx##*:}" + blk="$OUT/cov_tmp_ssss_${pm}_cov_Ntheta${NBINS}_Ntomo1_${idx}" + [ -f "$blk" ] || { echo "MISSING block $blk (see cosmocov_block_${idx}.log)" >&2; exit 1; } + cat "$blk" >> "$CAT" +done +echo "[cosmocov] concatenated -> $CAT" + +echo "[cosmocov] processing (positive-definite check, G/G+NG extract, QA plot)" +apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" "$CONTAINER" \ + /usr/local/bin/python "$WT/cosmo_inference/scripts/cosmocov_process.py" \ + "$CAT" "$OUT/covariance_processed" +echo "[cosmocov] done -> $OUT/covariance_processed.txt (+_g.txt, +_plot.pdf)" diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index ea2f35bc..9df0bbfd 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -48,9 +48,11 @@ cv.calculate_rho_tau_stats() -# Confirm CosmologyValidation produced the requested outputs +# Confirm CosmologyValidation produced the requested outputs. calculate_rho_tau_stats +# writes the rho/tau FITS *and* the born-as-SACC rho_tau part (via +# rho_tau_to_sacc_part); the part feeds the assemble_sacc rule. outputs = snakemake.output # type: ignore -for label in ("rho_stats", "tau_stats"): +for label in ("rho_stats", "tau_stats", "rho_tau"): target = Path(outputs[label]) if not target.exists(): raise FileNotFoundError( From c09b063f2c46d89d7ced93d438e2e6394e1a0607 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 11:50:22 +0200 Subject: [PATCH 011/160] test(blinding): adapt to canonical sacc_io save/load and relocated config Three integration-drift fixes surfaced by the reconciled base: - test_ac6_ac8 / test_ac8_dotted: the end-to-end fixtures now pass type="mock" to sio.save (PR-2 requires the provenance stamp); the parts are mocks, blind_part re-saves inheriting that type. - test_ac13: the CosmoSIS halofit config moved to cosmo_inference/cosmosis_config/templates/ on develop; point the AC13 assertion at the new path (token unchanged: mead2020_feedback). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WzUt7VbtXwr2SCHUdiQTyt --- src/sp_validation/tests/test_blinding.py | 4 ++-- src/sp_validation/tests/test_camb_ccl_crosscheck.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 7013a81a..5cd0f0f7 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -975,7 +975,7 @@ def test_ac6_ac8_end_to_end_init_parts_gather_unblind(tmp_path): part_files, out_paths = {}, {} for name in ("xi_reporting", "xi_integration", "cl"): path = tmp_path / f"{name}.fits" - sio.save(parts[name], str(path)) + sio.save(parts[name], str(path), type="mock") out_paths[name] = bd.blind_part(str(path), str(blind_dir), log=_NOLOG) part_files[name] = path @@ -1081,7 +1081,7 @@ def test_ac8_dotted_versioned_part_names_escrow_and_restore(tmp_path): out_paths, part_files = {}, {} for name in ("xi_reporting", "xi_integration"): path = tmp_path / f"{version}_{name}.fits" - sio.save(parts[name], str(path)) + sio.save(parts[name], str(path), type="mock") out_paths[name] = bd.blind_part(str(path), str(blind_dir), log=_NOLOG) part_files[name] = path diff --git a/src/sp_validation/tests/test_camb_ccl_crosscheck.py b/src/sp_validation/tests/test_camb_ccl_crosscheck.py index b4cb3b3f..3f286474 100644 --- a/src/sp_validation/tests/test_camb_ccl_crosscheck.py +++ b/src/sp_validation/tests/test_camb_ccl_crosscheck.py @@ -139,6 +139,7 @@ def test_ac13_halofit_token_matches_inference_config(): pathlib.Path(__file__).resolve().parents[3] / "cosmo_inference" / "cosmosis_config" + / "templates" / "cosmosis_pipeline_A_ia_cell.ini" ) match = re.search(r"^halofit_version\s*=\s*(\S+)", ini.read_text(), re.MULTILINE) From 72e680f7cb4bc446f2aff491037f7d7ceae642ee Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 11:50:29 +0200 Subject: [PATCH 012/160] Migrate to canonical sacc_io API: typed save, fail-closed load, grid rename Sweep the migration code onto PR2's canonical vocabulary and contracts: - grid='coarse'/'fine' -> 'reporting'/'integration' everywhere (writer calls, readers, tests), including the internal DAG intermediates: {version}_xi_coarse* -> _xi_reporting*, _xi_fine -> _xi_integration, the xi_coarse/xi_fine Snakemake output keys, CANONICAL part names, cv_xi_reporting_sacc, write_xi_integration_sacc. - save(s, path) -> save(s, path, type=...): 'data' at every production writer (the cosmo_val pipeline measures the real UNIONS catalogues; GLASS mocks do not flow through these writers), 'mock' for synthetic test fixtures. assemble_sacc propagates its parts' type stamp rather than hardcoding, so mock parts assemble into a mock analysis file. - load(path) -> fail-closed load: pipeline-internal readbacks of freshly written pre-blind data parts pass allow_unblinded=True (blinding is a downstream Smokescreen step); mock fixtures load freely. Readers raising on unmatched selections needed no call-site changes: every get_* reads a statistic guaranteed present in the file just written or assembled. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WzUt7VbtXwr2SCHUdiQTyt --- papers/bmodes/scripts/run_xi_sweep.py | 4 +-- src/sp_validation/cosmo_val/cosebis.py | 2 +- src/sp_validation/cosmo_val/pseudo_cl.py | 8 ++++-- .../cosmo_val/psf_systematics.py | 2 +- src/sp_validation/cosmo_val/pure_eb.py | 2 +- src/sp_validation/cosmo_val/sacc_writers.py | 13 +++++---- src/sp_validation/tests/test_assemble_sacc.py | 28 +++++++++---------- .../tests/test_bmodes_workflow_dry_run.py | 2 +- src/sp_validation/tests/test_cli_seams.py | 2 +- src/sp_validation/tests/test_pseudo_cl.py | 4 ++- src/sp_validation/tests/test_sacc_writers.py | 27 ++++++++++-------- workflow/rules/cosmo_val.smk | 20 ++++++------- workflow/rules/inference.smk | 4 +-- workflow/rules/twopoint.smk | 10 +++---- workflow/scripts/assemble_sacc.py | 20 ++++++++----- workflow/scripts/generate_pseudo_cl.py | 4 ++- workflow/scripts/run_2pcf.py | 22 +++++++-------- workflow/scripts/run_2pcf_highres.py | 16 +++++------ 18 files changed, 105 insertions(+), 85 deletions(-) diff --git a/papers/bmodes/scripts/run_xi_sweep.py b/papers/bmodes/scripts/run_xi_sweep.py index 49076435..0aa5f859 100644 --- a/papers/bmodes/scripts/run_xi_sweep.py +++ b/papers/bmodes/scripts/run_xi_sweep.py @@ -74,13 +74,13 @@ def _from_cli(argv=None): for grid in a.grids: # The sweep consumes only the .txt dump (cosebis_version_comparison # reconstructs it by binning). run_2pcf is born-as-SACC, so give its - # coarse part a grid-qualified name — the default {ver}_xi_coarse.sacc + # reporting part a grid-qualified name — the default {ver}_xi_reporting.sacc # carries no binning, so the two grids per version would collide. run_2pcf( ver=ver, cat_config=a.cat_config, output_dir=a.out, - sacc_out=os.path.join(a.out, f"{ver}_xi_coarse_{grid}.sacc"), + sacc_out=os.path.join(a.out, f"{ver}_xi_reporting_{grid}.sacc"), **GRIDS[grid], ) diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index 6b1d6146..9ca048c3 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -180,7 +180,7 @@ def cosebis_to_sacc_part(self, version, out_path, results, fiducial_scale_cut=No result, scale_cut, ) - sacc_io.save(s, out_path) + sacc_io.save(s, out_path, type="data") def plot_cosebis( self, diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 29ff7846..2a55934a 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -514,7 +514,11 @@ def calculate_pseudo_cl(self, out_path=None): @staticmethod def _load_pseudo_cl_sacc(out_path): """Read a pseudo-Cl SACC part into the ELL/EE/EB/BB dict consumers use.""" - s = sacc_io.load(out_path) + # Pipeline-internal readback of a part this producer just wrote: the + # born-as-SACC parts are unblinded real-data measurements (blinding is a + # downstream Smokescreen step), so the fail-closed load must be told this + # is a legitimate pre-blind consumer. + s = sacc_io.load(out_path, allow_unblinded=True) ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(s, SACC_BIN) return {"ELL": ell, "EE": ee, "EB": eb, "BB": bb} @@ -704,7 +708,7 @@ def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): cl_all, wsp, ) - sacc_io.save(s, out_path) + sacc_io.save(s, out_path, type="data") def plot_pseudo_cl(self): """ diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index af819977..6f72ac1f 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -80,7 +80,7 @@ def rho_tau_to_sacc_part( tau_cov_th=tau_cov_th, ) out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") - sacc_io.save(s, out_path) + sacc_io.save(s, out_path, type="data") @property def rho_stat_handler(self): diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 416ad8ef..85d3beb8 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -151,7 +151,7 @@ def pure_eb_to_sacc_part(self, version, out_path, results): eb, covariance=results["cov"], ) - sacc_io.save(s, out_path) + sacc_io.save(s, out_path, type="data") def plot_pure_eb( self, diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index 76e9e6e0..72594650 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -11,9 +11,10 @@ ``sacc.concatenate_data_sets``, whose ``BlockDiagonalCovariance`` output the contract rules out). -The fine-grid ``{version}_xi_fine.sacc`` is a terminal product in its own right -(:func:`xi_to_sacc` with ``grid="fine"`` and a ``DiagonalCovariance`` from -TreeCorr ``varxip``/``varxim``); COSEBIs and pure-E/B consume it. +The integration-grid ``{version}_xi_integration.sacc`` is a terminal product in +its own right (:func:`xi_to_sacc` with ``grid="integration"`` and a +``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim``); COSEBIs and pure-E/B +consume it. Everything here is single-bin today (``bins=(0, 0)``); the interface is tomography-native so a future round supplies real bin pairs unchanged. @@ -47,10 +48,10 @@ def xi_to_sacc( weight=None, variances=None, ): - """One ξ± part (``bins=(0, 0)``) on the coarse or fine grid. + """One ξ± part (``bins=(0, 0)``) on the reporting or integration grid. ``variances`` (the concatenated ``[varxip; varxim]``) attaches a - ``DiagonalCovariance`` — used for the terminal fine file, where npatch=1 + ``DiagonalCovariance`` — used for the terminal integration file, where npatch=1 leaves TreeCorr shot-noise variance as the only covariance estimate. """ s = sio.new_sacc(nz, metadata) @@ -220,7 +221,7 @@ def assemble_analysis_sacc(nz, metadata, parts): Each part is a single-statistic Sacc (from a ``*_to_sacc`` writer, loaded from disk) carrying its own covariance = its block. This re-adds every part's data points into one Sacc in the order the parts are given — which - must be the canonical order (ξ± coarse, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ) + must be the canonical order (ξ± reporting, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ) — and assembles a single ``FullCovariance`` from the per-part covariance blocks. Point insertion order and block order therefore agree by construction, which ``sacc_io.assemble_covariance`` validates (contiguous, diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py index ccddab97..26ff5f15 100644 --- a/src/sp_validation/tests/test_assemble_sacc.py +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -53,11 +53,11 @@ def _theta(n=6): META = {"catalogue_version": "vSYNTH", "npatch": 1} -def _write_parts(tmp_path, *, with_pseudo_cl=True, cov_less=("xi_coarse",)): +def _write_parts(tmp_path, *, with_pseudo_cl=True, cov_less=("xi_reporting",)): """Write per-statistic parts to disk; return the ``{name: path}`` mapping. Parts named in ``cov_less`` are written without a covariance (mimicking the - born-cov-less ξ± coarse / pseudo-Cℓ parts); the rest carry their own block. + born-cov-less ξ± reporting / pseudo-Cℓ parts); the rest carry their own block. """ nz = {0: _nz()} theta = _theta() @@ -72,9 +72,9 @@ def get_bandpower_windows(self): return w xi = sw.xi_to_sacc( - nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) - if "xi_coarse" not in cov_less: + if "xi_reporting" not in cov_less: xi.add_covariance(_spd(len(xi.mean), 1)) cl_all = np.vstack( @@ -119,7 +119,7 @@ def get_bandpower_windows(self): rt = sw.rho_tau_to_sacc(nz, META, rho, tau) parts = { - "xi_coarse": xi, + "xi_reporting": xi, "pseudo_cl": cl, "cosebis": co, "pure_eb": eb, @@ -131,7 +131,7 @@ def get_bandpower_windows(self): paths = {} for name, part in parts.items(): p = tmp_path / f"{name}.sacc" - sio.save(part, str(p)) + sio.save(part, str(p), type="mock") paths[name] = str(p) return paths @@ -139,7 +139,7 @@ def get_bandpower_windows(self): def test_assemble_sacc_placeholder_canonical_order(tmp_path): """The cov-less ξ± part gets a placeholder; every point is covered and the blocks land in canonical order (ξ±, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ).""" - paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) out = tmp_path / "vSYNTH.sacc" s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) assert out.exists() @@ -171,7 +171,7 @@ def test_assemble_sacc_placeholder_canonical_order(tmp_path): def test_assemble_sacc_injects_real_xi_covariance(tmp_path): """A CosmoCov ξ covariance .txt is loaded into the cov-less ξ± block.""" - paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) # ξ± part has 12 points ([ξ+; ξ−] over 6 θ); supply a matching cov .txt. xi_cov = _spd(12, 21) cov_path = tmp_path / "xi_cov.txt" @@ -189,7 +189,7 @@ def test_assemble_sacc_injects_pseudo_cl_covariance(tmp_path): block (the live default: ξ± placeholder + real pseudo-Cℓ cov).""" from astropy.io import fits - paths = _write_parts(tmp_path, cov_less=("xi_coarse", "pseudo_cl")) + paths = _write_parts(tmp_path, cov_less=("xi_reporting", "pseudo_cl")) # pseudo-Cℓ part is 3 ell × {EE, BB, EB} = 9 points; per-spectrum 3×3 blocks. ee, bb, eb = _spd(3, 31), _spd(3, 32), _spd(3, 33) cov_fits = tmp_path / "pseudo_cl_cov.fits" @@ -222,7 +222,7 @@ def test_assemble_sacc_injects_pseudo_cl_covariance(tmp_path): def test_assemble_sacc_missing_cov_raises(tmp_path): """A cov-less part with no injected block and no placeholder fails loudly.""" - paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) out = tmp_path / "vSYNTH.sacc" with pytest.raises(ValueError, match="carries no covariance"): asm.assemble_sacc("vSYNTH", paths, str(out)) @@ -230,7 +230,7 @@ def test_assemble_sacc_missing_cov_raises(tmp_path): def test_assemble_sacc_respects_pseudo_cl_toggle(tmp_path): """With pseudo_cl absent, assembly still succeeds and omits the Cℓ points.""" - paths = _write_parts(tmp_path, with_pseudo_cl=False, cov_less=("xi_coarse",)) + paths = _write_parts(tmp_path, with_pseudo_cl=False, cov_less=("xi_reporting",)) assert "pseudo_cl" not in paths out = tmp_path / "vSYNTH.sacc" s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) @@ -245,7 +245,7 @@ def test_assemble_sacc_respects_pseudo_cl_toggle(tmp_path): def test_assemble_sacc_expected_part_missing_raises(tmp_path): """A typo'd input keyword drops a part from part_paths; the expected list catches it rather than silently omitting the statistic.""" - paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) # Simulate a rule-input typo: cosebis wired under the wrong key. paths["cosebi"] = paths.pop("cosebis") out = tmp_path / "vSYNTH.sacc" @@ -254,14 +254,14 @@ def test_assemble_sacc_expected_part_missing_raises(tmp_path): "vSYNTH", paths, str(out), - expected=["xi_coarse", "pseudo_cl", "cosebis", "pure_eb", "rho_tau"], + expected=["xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau"], placeholder_var=1.0, ) def test_assemble_sacc_expected_rejects_unknown_name(tmp_path): """A typo in the expected list itself is rejected (not a valid statistic).""" - paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) out = tmp_path / "vSYNTH.sacc" with pytest.raises(ValueError, match="not assemblable statistics"): asm.assemble_sacc( diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index 87afc416..cc6a293c 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -94,5 +94,5 @@ def test_cosmo_val_workflow_assemble_dry_runs(): assert "rule assemble_sacc:" in out, out assert f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" in out, out assert f"pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits" in out, out - for part in ("_xi_coarse_", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): + for part in ("_xi_reporting_", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): assert part in out, f"missing {part} part in assemble DAG:\n{out}" diff --git a/src/sp_validation/tests/test_cli_seams.py b/src/sp_validation/tests/test_cli_seams.py index 697b3426..069d8ccc 100644 --- a/src/sp_validation/tests/test_cli_seams.py +++ b/src/sp_validation/tests/test_cli_seams.py @@ -45,7 +45,7 @@ def test_run_xi_sweep_run_2pcf_call_binds(): ver="V", cat_config="/cfg.yaml", output_dir="/out", - sacc_out="/out/V_xi_coarse_reporting.sacc", + sacc_out="/out/V_xi_reporting_reporting.sacc", min_sep=1.0, max_sep=250.0, nbins=20, diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 637b10c2..8c095a71 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -525,7 +525,9 @@ def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): cv.calculate_pseudo_cl_catalog(ver, out_path) assert os.path.exists(out_path) - s = sacc_io.load(out_path) + # The born-as-SACC part is unblinded type='data'; reading it back for the + # round-trip assertion is a pre-blind consumer. + s = sacc_io.load(out_path, allow_unblinded=True) ell, ee, bb, eb, window = sacc_io.get_pseudo_cl(s, SACC_BIN) # A shared BandpowerWindow rides the part per the SACC layout contract. assert window is not None diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index d886fa0a..26072c11 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -31,7 +31,7 @@ def _theta(n=6): def _roundtrip(s, tmp_path, name): p = tmp_path / f"{name}.sacc" - sio.save(s, str(p)) + sio.save(s, str(p), type="mock") return sio.load(str(p)) @@ -41,20 +41,20 @@ def _roundtrip(s, tmp_path, name): # --------------------------------------------------------------------------- # # Per-writer parts # --------------------------------------------------------------------------- # -def test_xi_to_sacc_coarse(tmp_path): +def test_xi_to_sacc_reporting(tmp_path): theta = _theta() xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 s = sw.xi_to_sacc( - {0: _nz()}, META, theta, xip, xim, grid="coarse", theta_nom=theta * 1.01 + {0: _nz()}, META, theta, xip, xim, grid="reporting", theta_nom=theta * 1.01 ) s2 = _roundtrip(s, tmp_path, "xic") - th, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + th, p, m = sio.get_xi(s2, (0, 0), grid="reporting") assert np.array_equal(th, theta) assert np.array_equal(p, xip) and np.array_equal(m, xim) - assert s2.covariance is None # coarse part has no cov until assembly + assert s2.covariance is None # reporting part has no cov until assembly -def test_xi_to_sacc_fine_diagonal(tmp_path): +def test_xi_to_sacc_integration_diagonal(tmp_path): theta = np.geomspace(0.5, 300.0, 30) xip, xim = np.arange(30) * 1e-5, np.arange(30) * 2e-5 varxip, varxim = np.arange(1, 31) * 1e-12, np.arange(1, 31) * 2e-12 @@ -64,12 +64,12 @@ def test_xi_to_sacc_fine_diagonal(tmp_path): theta, xip, xim, - grid="fine", + grid="integration", variances=np.concatenate([varxip, varxim]), ) assert type(s.covariance).__name__ == "DiagonalCovariance" s2 = _roundtrip(s, tmp_path, "xif") - th, p, _ = sio.get_xi(s2, (0, 0), grid="fine") + th, p, _ = sio.get_xi(s2, (0, 0), grid="integration") assert np.array_equal(th, theta) and np.array_equal(p, xip) assert np.array_equal( np.diag(s2.covariance.dense), np.concatenate([varxip, varxim]) @@ -239,7 +239,7 @@ def get_bandpower_windows(self): return w xi = sw.xi_to_sacc( - nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) xi.add_covariance(_spd(len(xi.mean), 1)) cl_all = np.vstack( @@ -293,7 +293,12 @@ def test_assemble_analysis_sacc_requires_covariance(): parts = _make_parts(nz) parts.append( sw.xi_to_sacc( - nz, META, _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + nz, + META, + _theta(), + np.arange(6) * 1e-5, + np.arange(6) * 2e-5, + grid="reporting", ) ) # no covariance with pytest.raises(ValueError, match="own covariance block"): @@ -306,7 +311,7 @@ def test_assemble_from_reloaded_parts(tmp_path): parts = _make_parts(nz) reloaded = [] for i, part in enumerate(parts): - sio.save(part, str(tmp_path / f"part{i}.sacc")) + sio.save(part, str(tmp_path / f"part{i}.sacc"), type="mock") reloaded.append(sio.load(str(tmp_path / f"part{i}.sacc"))) s = sw.assemble_analysis_sacc(nz, META, reloaded) assert type(s.covariance).__name__ == "FullCovariance" diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 4f2a2e46..e8eb332d 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -154,16 +154,16 @@ def cv_rho_tau_sacc(version): ) -def cv_xi_coarse_sacc(version): - """Coarse ξ± SACC part the xi rule (run_2pcf.py) writes for a version. +def cv_xi_reporting_sacc(version): + """Reporting ξ± SACC part the xi rule (run_2pcf.py) writes for a version. Carries the reporting-binning suffix so requesting it binds the xi job's - wildcards (the rule's txt + coarse .sacc outputs share one wildcard set). + wildcards (the rule's txt + reporting .sacc outputs share one wildcard set). """ return str( COSMO_VAL / ( - f"{version}_xi_coarse_minsep={CV['theta_min']}_maxsep={CV['theta_max']}" + f"{version}_xi_reporting_minsep={CV['theta_min']}_maxsep={CV['theta_max']}" f"_nbins={CV['nbins']}_npatch={CV['npatch']}.sacc" ) ) @@ -442,9 +442,9 @@ rule cv_summarize_bmodes: # --------------------------------------------------------------------------- # Terminal analysis file: assemble the per-statistic SACC parts into {version}.sacc # --------------------------------------------------------------------------- -# The five born-as-SACC parts (xi_coarse, pseudo_cl, cosebis, pure_eb, rho_tau) +# The five born-as-SACC parts (xi_reporting, pseudo_cl, cosebis, pure_eb, rho_tau) # are each written by their own rule carrying its own covariance block, except -# ξ± coarse and pseudo-Cℓ which are born cov-less by design. assemble_sacc.py +# ξ± reporting and pseudo-Cℓ which are born cov-less by design. assemble_sacc.py # loads the parts in canonical order and rebuilds one {version}.sacc with a # single FullCovariance (point-insertion order = block order). # @@ -453,7 +453,7 @@ rule cv_summarize_bmodes: # analysis file stays byte-comparable against it (PR-3's converter). Its real # NaMaster covariance is injected here from the matching pseudo_cl_cov FITS # (COVAR_EE_EE/BB_BB/EB_EB → block-diagonal, dropping cross-spectra, matching the -# B-mode PTE's use of COVAR_BB_BB). The ξ± coarse block is the one piece not yet +# B-mode PTE's use of COVAR_BB_BB). The ξ± reporting block is the one piece not yet # sourced from its real covariance: the CosmoCov theory .txt is blind/gaussian/ # mask-keyed and lives deep in the inference tree, so wiring it couples cosmo_val # to the whole inference covariance DAG — that sourcing is PR-3's converter @@ -466,12 +466,12 @@ def cv_assemble_inputs(version): """The per-statistic SACC parts + covariance inputs assemble_sacc consumes. Each part's filename carries enough to bind its producing rule's wildcards - (the coarse ξ± and ρ/τ parts their reporting binning; the pseudo-Cℓ part its + (the reporting ξ± and ρ/τ parts their reporting binning; the pseudo-Cℓ part its fiducial harmonic tag). pseudo_cl (+ its cov) is included only when the config toggles the harmonic-space BB into the analysis. """ parts = dict( - xi_coarse=cv_xi_coarse_sacc(version), + xi_reporting=cv_xi_reporting_sacc(version), cosebis=cv_cosebis_sacc(version), pure_eb=cv_pure_eb_sacc(version), rho_tau=cv_rho_tau_sacc(version), @@ -496,7 +496,7 @@ rule assemble_sacc: expected=lambda w: [ k for k in cv_assemble_inputs(w.version) if k != "pseudo_cl_cov" ], - # ξ± coarse has no real covariance wired yet (its CosmoCov theory block is + # ξ± reporting has no real covariance wired yet (its CosmoCov theory block is # PR-3's converter territory, plugging in via --xi-cov). By DEFAULT this # is fatal: assemble_sacc.py raises rather than ship {version}.sacc — the # terminal science file — with a var=1.0 placeholder as its LEADING diff --git a/workflow/rules/inference.smk b/workflow/rules/inference.smk index 6bf68645..ff0c982d 100644 --- a/workflow/rules/inference.smk +++ b/workflow/rules/inference.smk @@ -62,8 +62,8 @@ def pseudo_cl_assets(version): # The SACC migration (PR 4) removed the data products several of these inputs # name, so this rule's DAG no longer resolves and is NOT reachable from the # cosmo_val suite (cosmo_val_all never requests it). Stale inputs: -# - xi_plus / xi_minus FITS: the `xi` rule now emits the coarse ξ± SACC part -# ({version}_xi_coarse_...sacc), not per-sign FITS. +# - xi_plus / xi_minus FITS: the `xi` rule now emits the reporting ξ± SACC part +# ({version}_xi_reporting_...sacc), not per-sign FITS. # - pseudo_cl / pseudo_cl_cov via pseudo_cl_assets(): the `pseudo_cl` rule now # writes .sacc (pseudo_cl_assets still requests .fits). # PR 7 rewires this to consume the assembled {version}.sacc (built by diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index ebd19f91..8202ed66 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -8,13 +8,13 @@ rule xi: catalog=get_shear_catalog, output: # Raw TreeCorr .txt byproduct (read back by covariance + skip-if-exists) - # and the born-as-SACC coarse ξ± part (no covariance until the + # and the born-as-SACC reporting ξ± part (no covariance until the # assemble_sacc rule injects the CosmoCov block). Both outputs carry the # same reporting-binning wildcards — Snakemake requires every output of a - # rule to share one wildcard set, and it keeps the coarse .sacc name + # rule to share one wildcard set, and it keeps the reporting .sacc name # self-describing so requesting it binds the xi job unambiguously. txt=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), - xi_coarse=str(COSMO_VAL / "{version}_xi_coarse_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), + xi_reporting=str(COSMO_VAL / "{version}_xi_reporting_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 24 params: ver="{version}", @@ -33,14 +33,14 @@ rule xi: rule xi_highres: """High-resolution xi for COSEBIS integration. - Terminal born-as-SACC product: {version}_xi_fine.sacc (a DiagonalCovariance + Terminal born-as-SACC product: {version}_xi_integration.sacc (a DiagonalCovariance from TreeCorr varxip/varxim). COSEBIs and pure-E/B consume it. The raw .txt dump is kept as a convergence byproduct. """ container: None output: txt=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.txt"), - xi_fine=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_fine.sacc"), + xi_integration=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_integration.sacc"), resources: tasks=30, cpus_per_task=12, diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 513ec976..fb61a7c4 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -6,7 +6,7 @@ Each per-statistic ``*.sacc`` *part* (written born-as-SACC by the mixins and the run_2pcf / generate_pseudo_cl scripts) holds one statistic. The assembler loads -them in canonical order — ξ± coarse, pseudo-Cℓ, COSEBIs, pure-E/B, ρ/τ — and +them in canonical order — ξ± reporting, pseudo-Cℓ, COSEBIs, pure-E/B, ρ/τ — and calls :func:`sacc_writers.assemble_analysis_sacc`, which rebuilds one Sacc with a single block-diagonal ``FullCovariance`` (point-insertion order = block order, validated by ``sacc_io.assemble_covariance``). @@ -15,10 +15,10 @@ ----------------------------------------------- ``assemble_analysis_sacc`` REQUIRES every part to carry its own covariance block. The COSEBIs, pure-E/B and ρ/τ parts already do (their writers attach it). The -ξ± coarse and pseudo-Cℓ parts are born cov-less by design; this script injects +ξ± reporting and pseudo-Cℓ parts are born cov-less by design; this script injects their blocks before assembly: -* **ξ± coarse** — the CosmoCov theory covariance ``.txt`` (``--xi-cov``). For the +* **ξ± reporting** — the CosmoCov theory covariance ``.txt`` (``--xi-cov``). For the single-bin round it is already ``[ξ+; ξ−]``-ordered (CosmoCov / covdat_to_fits: ``STRT_0=0`` XI_PLUS, ``STRT_1=len/2`` XI_MINUS), which is exactly the SACC ξ insertion order, so ``np.loadtxt`` → ``add_covariance`` needs no permutation. @@ -51,7 +51,7 @@ # Canonical part order — the order assemble_analysis_sacc inserts points in, which # must match the covariance block order. Missing parts are simply skipped. -CANONICAL = ("xi_coarse", "pseudo_cl", "cosebis", "pure_eb", "rho_tau") +CANONICAL = ("xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau") def _pseudo_cl_cov_block(cov_fits, hdu): @@ -84,7 +84,7 @@ def _attach_cov(part, name, xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholde """ if part.covariance is not None: return part - if name == "xi_coarse": + if name == "xi_reporting": if xi_cov is not None: part.add_covariance(np.loadtxt(xi_cov)) return part @@ -153,7 +153,10 @@ def assemble_sacc( path = part_paths.get(name) if path is None: continue - part = sacc_io.load(path) + # Assembly runs pre-blind on unblinded real-data parts (Smokescreen + # conceals the assembled analysis file downstream), so the fail-closed + # load is told this is a legitimate pre-blind consumer. + part = sacc_io.load(path, allow_unblinded=True) if nz is None: # The nz tracers + metadata are identical across parts (same version); # take them from the first loaded part for the assembled file. @@ -167,7 +170,10 @@ def assemble_sacc( if not parts: raise ValueError(f"no parts found for {version}: {part_paths}") s = assemble_analysis_sacc(nz, metadata, parts) - sacc_io.save(s, out_path) + # Assembly preserves its parts' provenance: every part was written by + # sacc_io.save and therefore carries the type=data|mock stamp in its + # metadata (copied into the assembled file above). + sacc_io.save(s, out_path, type=metadata["type"]) print(f"Assembled {len(parts)} parts -> {out_path}") return s diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index 9bfabf4a..0cec054e 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -144,7 +144,9 @@ def generate_pseudo_cl( cv.calculate_pseudo_cl(out_path=out_path) if os.path.exists(out_path): - s = sacc_io.load(out_path) + # Pipeline-internal readback of the unblinded data part just written + # (blinding is a downstream Smokescreen step). + s = sacc_io.load(out_path, allow_unblinded=True) ell = sacc_io.get_pseudo_cl(s, (0, 0))[0] print(f"Generated pseudo-Cl with {len(ell)} ell bins") print(f"ell range: [{ell.min():.1f}, {ell.max():.1f}]") diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index a773d039..3e513479 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -15,8 +15,8 @@ The measurement itself is unchanged — ``CosmologyValidation.calculate_2pcf`` does the TreeCorr work and writes the ``.txt`` dump (a raw byproduct the covariance machinery reads back). The analysis ξ± data product is then born as -SACC here: ``{ver}_xi_coarse.sacc``, a *part* on the coarse grid via -``xi_to_sacc(grid="coarse", ...)`` carrying ``theta_nom``/``npairs``/``weight`` +SACC here: ``{ver}_xi_reporting.sacc``, a *part* on the reporting grid via +``xi_to_sacc(grid="reporting", ...)`` carrying ``theta_nom``/``npairs``/``weight`` tags but NO covariance (the ξ block is supplied at assembly from the CosmoCov theory covariance). ``output_dir`` is passed explicitly (rather than via the ``COSMO_VAL`` env hook) so lc can point each run at its own ``{output}`` tree. @@ -40,15 +40,15 @@ def run_2pcf( output_dir, sacc_out=None, ): - """Measure ξ±(θ) for ``ver`` and write its coarse SACC part. + """Measure ξ±(θ) for ``ver`` and write its reporting SACC part. Parameters mirror the TreeCorr reporting/integration grids: ``min_sep`` / ``max_sep`` in arcmin, ``nbins`` logarithmic bins, ``npatch`` spatial patches (1 for the paper fiducial). ``cat_config`` is an absolute path to the catalog configuration; ``output_dir`` overrides ``cat_config['paths']['output']`` so the ``.txt`` byproduct lands where lc - expects. ``sacc_out`` is the exact destination for the coarse ξ± SACC part - (the Snakemake-declared output); it defaults to ``{ver}_xi_coarse.sacc`` + expects. ``sacc_out`` is the exact destination for the reporting ξ± SACC part + (the Snakemake-declared output); it defaults to ``{ver}_xi_reporting.sacc`` under the resolved output directory for the CLI path. Returns @@ -69,7 +69,7 @@ def run_2pcf( nbins=nbins, ) - # Born-as-SACC coarse ξ± part: no covariance here (added at assembly from + # Born-as-SACC reporting ξ± part: no covariance here (added at assembly from # the CosmoCov theory covariance). theta = meanr; theta_nom = rnom. s = xi_to_sacc( cv.sacc_nz(ver), @@ -77,16 +77,16 @@ def run_2pcf( gg.meanr, gg.xip, gg.xim, - grid="coarse", + grid="reporting", theta_nom=gg.rnom, npairs=gg.npairs, weight=gg.weight, ) out_path = sacc_out or os.path.join( - output_dir or cv.cc["paths"]["output"], f"{ver}_xi_coarse.sacc" + output_dir or cv.cc["paths"]["output"], f"{ver}_xi_reporting.sacc" ) - sacc_io.save(s, out_path) - print(f"Wrote coarse ξ± SACC part: {out_path}") + sacc_io.save(s, out_path, type="data") + print(f"Wrote reporting ξ± SACC part: {out_path}") return gg @@ -106,7 +106,7 @@ def _from_snakemake(smk): output_dir=p.get("output_dir", None), # Write the SACC part exactly where the rule declares it (the .txt # byproduct still lands under the resolved output dir via _output_path). - sacc_out=smk.output["xi_coarse"], + sacc_out=smk.output["xi_reporting"], ) diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py index ed3660bf..689187ea 100644 --- a/workflow/scripts/run_2pcf_highres.py +++ b/workflow/scripts/run_2pcf_highres.py @@ -2,7 +2,7 @@ """ High-resolution ξ± measurement for COSEBIS integration. -Computes TreeCorr GGCorrelation with fine angular binning (10,000+ bins) +Computes TreeCorr GGCorrelation with integration angular binning (10,000+ bins) required for accurate COSEBIS mode integration. Uses MPI for patch-pair distribution across nodes when available; falls back to multi-threaded single-process otherwise. @@ -28,7 +28,7 @@ from astropy.io import fits # sacc_io depends only on numpy + sacc (no healpy/cs_util), so the born-as-SACC -# fine ξ± write works on the bare-host MPI path too, where the full cosmo_val +# integration ξ± write works on the bare-host MPI path too, where the full cosmo_val # stack is unavailable. from sp_validation import sacc_io from sp_validation.cosmo_val.sacc_writers import xi_to_sacc @@ -203,8 +203,8 @@ def compute_patch_centers(ra, dec): del cat_sub -def write_xi_fine_sacc(gg): - """Write the terminal fine-grid ξ± SACC part (``{version}_xi_fine.sacc``). +def write_xi_integration_sacc(gg): + """Write the terminal integration-grid ξ± SACC part (``{version}_xi_integration.sacc``). This is a terminal product in its own right — COSEBIs and pure-E/B consume it. It carries a ``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim`` @@ -224,12 +224,12 @@ def write_xi_fine_sacc(gg): gg.meanr, gg.xip, gg.xim, - grid="fine", + grid="integration", theta_nom=gg.rnom, variances=np.concatenate([gg.varxip, gg.varxim]), ) - out_path = os.path.join(OUTPUT_DIR, f"{VERSION}_xi_fine.sacc") - sacc_io.save(s, out_path) + out_path = os.path.join(OUTPUT_DIR, f"{VERSION}_xi_integration.sacc") + sacc_io.save(s, out_path, type="data") log(f" Wrote {out_path}") @@ -402,7 +402,7 @@ def main(): gg.write(out_txt, write_patch_results=False, write_cov=False) log(f" Wrote {out_txt}") - write_xi_fine_sacc(gg) + write_xi_integration_sacc(gg) elapsed = time.time() - t0 log(f"Done! Total time: {elapsed / 3600:.1f}h ({elapsed:.0f}s)") From 3abb566ef76106f700dd68e4edf0973a7590c4e3 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 18 Jul 2026 11:20:09 +0200 Subject: [PATCH 013/160] sacc_io: guard merge() against inconsistent theta grids Consistency follows tagging semantics: the grid tag declares which binning a set of points lives on, so all same-length theta arrays under one tag value must be bitwise identical (sacc never validates angles, and grids diverging at floating-point level choke CosmoSIS downstream). Different lengths within a tag group pass (scale-cut subsets); grids under different tag values are unconstrained (reporting vs integration differ by design). The rho/tau/pure-EB writers now tag their points grid="reporting" by default (overridable via grid=), so they join xi's consistency group and no untagged group appears in our own files. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MN9VazXKHUHQg16kiG7Ufk --- src/sp_validation/sacc_io.py | 96 ++++++++++++++++++++++--- src/sp_validation/tests/test_sacc_io.py | 80 +++++++++++++++++++++ 2 files changed, 166 insertions(+), 10 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 21872d0e..26251450 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -156,10 +156,10 @@ def _check_ascending(name, values): ) -def _add_theta_series(s, dtype, tracers, theta, values): +def _add_theta_series(s, dtype, tracers, theta, values, **tags): """Insert one theta-tagged series, one point per (theta, value) pair.""" for th, value in zip(theta, values): - s.add_data_point(dtype, tracers, float(value), theta=float(th)) + s.add_data_point(dtype, tracers, float(value), theta=float(th), **tags) def add_xi( @@ -280,7 +280,9 @@ def add_cosebis(s, bins, En, Bn, scale_cut): ) -def add_pure_eb(s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb): +def add_pure_eb( + s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb, *, grid="reporting" +): """Add pure E/B-mode correlation functions for one tracer pair. Six blocks are inserted in ``PURE_KEYS`` order (xip_E, xim_E, xip_B, @@ -297,15 +299,18 @@ def add_pure_eb(s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb): Angular separations (arcmin), shared by all six blocks. xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb : array_like The six pure E/B / ambiguous mode arrays at ``theta``. + grid : str, optional + Stored as the ``grid`` tag on every point (default ``'reporting'``), + joining ξ's theta-consistency group in ``merge``'s guard. """ _check_ascending("theta", theta) tracers = _pair(bins) arrays = (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb) for dtype, arr in zip(PURE_TYPES.values(), arrays): - _add_theta_series(s, dtype, tracers, theta, arr) + _add_theta_series(s, dtype, tracers, theta, arr, grid=grid) -def add_rho(s, k, theta, rho_p, rho_m): +def add_rho(s, k, theta, rho_p, rho_m, *, grid="reporting"): """Add a ρ_k PSF statistic (ρ+ then ρ−) on the ``psf_stars`` tracer. Parameters @@ -318,14 +323,17 @@ def add_rho(s, k, theta, rho_p, rho_m): Angular separations (arcmin). rho_p, rho_m : array_like ρ_k+ and ρ_k− at ``theta``. + grid : str, optional + Stored as the ``grid`` tag on every point (default ``'reporting'``), + joining ξ's theta-consistency group in ``merge``'s guard. """ _check_ascending("theta", theta) tracers = (PSF_TRACER, PSF_TRACER) - _add_theta_series(s, RHO_PLUS.format(k=k), tracers, theta, rho_p) - _add_theta_series(s, RHO_MINUS.format(k=k), tracers, theta, rho_m) + _add_theta_series(s, RHO_PLUS.format(k=k), tracers, theta, rho_p, grid=grid) + _add_theta_series(s, RHO_MINUS.format(k=k), tracers, theta, rho_m, grid=grid) -def add_tau(s, bins, k, theta, tau_p, tau_m): +def add_tau(s, bins, k, theta, tau_p, tau_m, *, grid="reporting"): """Add a τ_k PSF-leakage statistic (τ+ then τ−). Parameters @@ -341,11 +349,14 @@ def add_tau(s, bins, k, theta, tau_p, tau_m): Angular separations (arcmin). tau_p, tau_m : array_like τ_k+ and τ_k− at ``theta``. + grid : str, optional + Stored as the ``grid`` tag on every point (default ``'reporting'``), + joining ξ's theta-consistency group in ``merge``'s guard. """ _check_ascending("theta", theta) tracers = (source_name(bins[0]), PSF_TRACER) - _add_theta_series(s, TAU_PLUS.format(k=k), tracers, theta, tau_p) - _add_theta_series(s, TAU_MINUS.format(k=k), tracers, theta, tau_m) + _add_theta_series(s, TAU_PLUS.format(k=k), tracers, theta, tau_p, grid=grid) + _add_theta_series(s, TAU_MINUS.format(k=k), tracers, theta, tau_m, grid=grid) def assemble_covariance(s, blocks): @@ -623,6 +634,15 @@ def merge(saccs): library's clash behaviour, which mangles clashing keys by appending labels. + Theta consistency follows tagging semantics: the ``grid`` tag declares + which binning a set of points lives on, so all same-length theta arrays + under one tag value must be bitwise identical — sacc itself never + validates angles across data types/tracers, and a grid that differs + only at floating-point level chokes CosmoSIS downstream instead of + failing loud here. Different lengths within a tag group pass (scale-cut + subsets are legitimate); grids under different tag values are + unconstrained (``reporting`` vs ``integration`` differ by design). + Parameters ---------- saccs : sequence of sacc.Sacc @@ -633,6 +653,13 @@ def merge(saccs): ------- sacc.Sacc The merged data set. + + Raises + ------ + ValueError + If metadata conflicts, a shared tracer differs across inputs, or two + same-length theta arrays under the same ``grid`` tag value are not + bitwise identical. """ saccs = list(saccs) metadata = {} @@ -670,9 +697,58 @@ def merge(saccs): merged = sacc.concatenate_data_sets(*stripped, same_tracers=same_tracers) for key, value in metadata.items(): merged.metadata[key] = value + _check_theta_consistency(merged) return merged +def _theta_groups(s): + """Nested map ``grid-tag-value -> (data_type, tracers) -> theta array``. + + One entry per ``(data_type, tracers)`` series carrying a ``theta`` tag, + in each series' own insertion order (never re-sorted), nested under the + ``grid`` tag value it lives on (``None`` for untagged series) — the + shape ``merge``'s consistency guard checks within each tag value. + """ + groups = {} + for point in s.data: + if "theta" not in point.tags: + continue + by_series = groups.setdefault(point.tags.get("grid"), {}) + by_series.setdefault((point.data_type, point.tracers), []).append( + point.tags["theta"] + ) + return { + tag: {key: np.asarray(theta) for key, theta in by_series.items()} + for tag, by_series in groups.items() + } + + +def _check_theta_consistency(s): + """Raise unless same-length theta arrays under one ``grid`` tag match. + + Consistency follows tagging semantics: the ``grid`` tag declares which + binning a series lives on, so all same-length theta arrays sharing a + tag value must be bitwise identical — sacc never validates angles + across data types/tracers, and a grid diverging at floating-point level + chokes CosmoSIS downstream instead of failing loud here. Different + lengths within a tag value pass (scale-cut subsets are legitimate); + series under different tag values are unconstrained (``reporting`` vs + ``integration`` differ by design). + """ + for tag, by_series in _theta_groups(s).items(): + series = list(by_series.items()) + for i, (key_a, theta_a) in enumerate(series): + for key_b, theta_b in series[i + 1 :]: + if len(theta_a) != len(theta_b) or np.array_equal(theta_a, theta_b): + continue + max_diff = np.max(np.abs(theta_a - theta_b)) + raise ValueError( + f"theta grids under the same grid tag ({tag!r}) differ; " + f"harmonize upstream — groups {key_a!r} and {key_b!r} " + f"(max abs diff {max_diff:.3e})" + ) + + def update_statistic(s, sub): """Overwrite the values of ``s``'s points that match ``sub``'s, in place. diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 11788fbb..10b8ea01 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -795,6 +795,86 @@ def test_merge_rejects_divergent_shared_tracer(): sio.merge([s_xi, s_co]) +# --------------------------------------------------------------------------- # +# 13b. merge(): theta-consistency guard across groups. +# --------------------------------------------------------------------------- # +def _rho_sacc(k=0, theta=None, scale=1.0, metadata=None): + s = sio.new_sacc({0: _nz(0)}, metadata=metadata) + theta = _theta() if theta is None else theta + sio.add_rho( + s, + k, + theta, + np.arange(len(theta)) * scale * 1e-6, + np.arange(len(theta)) * scale * 2e-6, + ) + return s + + +def test_merge_identical_theta_grids_passes(): + # xi and rho both default to grid='reporting' and share the grid bitwise + s_xi, s_rho = _xi_sacc(), _rho_sacc(theta=_theta()) + s_rho2 = _rho_sacc(k=1, theta=_theta()) + merged = sio.merge([s_xi, s_rho, s_rho2]) + assert len(merged.mean) == sum(len(s.mean) for s in (s_xi, s_rho, s_rho2)) + + +def test_merge_nearly_identical_theta_grids_raises(): + theta = _theta() + s_rho = _rho_sacc(theta=theta) + # same 'reporting' grid group, same length, ~1e-9 relative perturbation + s_rho2 = _rho_sacc(k=1, theta=theta * (1 + 1e-9)) + with pytest.raises(ValueError, match="theta grids under the same grid tag"): + sio.merge([s_rho, s_rho2]) + + +def test_merge_rho_off_xi_reporting_grid_raises(): + s_xi = _xi_sacc() # grid='reporting' + # rho defaults to 'reporting' too: a slightly-off grid must fail loud + s_rho = _rho_sacc(theta=_theta() * (1 + 1e-9)) + with pytest.raises(ValueError, match="theta grids under the same grid tag"): + sio.merge([s_xi, s_rho]) + + +def test_merge_clearly_different_theta_grids_same_tag_raises(): + s_rho = _rho_sacc(theta=_theta()) # theta in [1, 100] + # same 'reporting' group, same length, entirely different binning + s_rho2 = _rho_sacc(k=1, theta=np.geomspace(200.0, 400.0, 6)) + with pytest.raises(ValueError, match="theta grids under the same grid tag"): + sio.merge([s_rho, s_rho2]) + + +def test_merge_different_length_theta_grids_passes(): + s_rho = _rho_sacc(theta=_theta()) # 6-point theta + s_rho2 = _rho_sacc(k=1, theta=_theta(nbins=10)) # same group, subset OK + merged = sio.merge([s_rho, s_rho2]) + assert len(merged.mean) == len(s_rho.mean) + len(s_rho2.mean) + + +def test_merge_same_length_grids_under_different_tags_pass(): + # reporting vs integration differ by design — no cross-tag constraint + s = sio.new_sacc({0: _nz(0)}) + _add_xi(s, grid="reporting") # theta in [1, 100], 6 points + sio.add_xi( + s, + (0, 0), + np.geomspace(200.0, 400.0, 6), # same length, different values + np.arange(6) * 1e-5, + np.arange(6) * 2e-5, + grid="integration", + ) + merged = sio.merge([s, _rho_sacc(theta=_theta())]) + assert len(merged.mean) == len(s.mean) + 12 + + +def test_merge_same_tag_clearly_different_grids_raise(): + # same (untagged) group, same length, values far apart -> still an error + s_rho = _rho_sacc(theta=_theta()) # theta in [1, 100] + s_rho2 = _rho_sacc(k=1, theta=np.geomspace(200.0, 400.0, 6)) + with pytest.raises(ValueError, match="theta grids under the same grid tag"): + sio.merge([s_rho, s_rho2]) + + # --------------------------------------------------------------------------- # # 15. Unmatched selections fail loud — no silent-empty arrays anywhere. # --------------------------------------------------------------------------- # From 2da8c10dcdc42a59f0e603698c3efa36cbf08fd2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 18 Jul 2026 12:52:59 +0200 Subject: [PATCH 014/160] sacc_io: extend merge guard to ell grids and bandpower windows; tag pseudo-Cl grid="reporting" The theta consistency guard generalizes to both angular domains: theta and ell points are grouped separately by grid tag value, and within a tag value all same-length grids must be bitwise identical. Two ell series sharing a grid must also carry equal bandpower windows (window ells and weight matrix); series without windows skip that check. add_pseudo_cl now stamps grid="reporting" on every point by default (overridable), joining the merge guard's consistency groups; since sacc's add_ell_cl accepts no extra tags, its per-point insertion (ell + shared window + window_ind) is inlined. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MN9VazXKHUHQg16kiG7Ufk --- src/sp_validation/sacc_io.py | 142 +++++++++++++++--------- src/sp_validation/tests/test_sacc_io.py | 52 ++++++++- 2 files changed, 136 insertions(+), 58 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 26251450..3df6488a 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -220,6 +220,7 @@ def add_pseudo_cl( *, window_ells, window_weights, + grid="reporting", ): """Add pseudo-Cℓ (EE, BB, EB) with a shared bandpower window. @@ -239,14 +240,27 @@ def add_pseudo_cl( Window matrix ``W`` of shape ``(nell, nbp)`` — one column per bandpower — from NaMaster ``get_bandpower_windows``. One ``sacc.BandpowerWindow`` is built and shared across EE/BB/EB. + grid : str, optional + Stored as the ``grid`` tag on every point (default ``'reporting'``), + joining ``merge``'s ℓ-consistency group; variant ℓ binnings belong + under different tag values. """ _check_ascending("ell_eff", ell_eff) tracers = _pair(bins) window = sacc.BandpowerWindow(np.asarray(window_ells), np.asarray(window_weights)) + # add_ell_cl accepts no extra tags, so inline its per-point insertion + # (ell + shared window + window_ind column index) plus the grid tag. for dtype, cl in ((CL_EE, cl_ee), (CL_BB, cl_bb), (CL_EB, cl_eb)): - s.add_ell_cl( - dtype, *tracers, np.asarray(ell_eff), np.asarray(cl), window=window - ) + for n, (ell, value) in enumerate(zip(ell_eff, cl)): + s.add_data_point( + dtype, + tracers, + float(value), + ell=float(ell), + window=window, + window_ind=n, + grid=grid, + ) def add_cosebis(s, bins, En, Bn, scale_cut): @@ -634,14 +648,17 @@ def merge(saccs): library's clash behaviour, which mangles clashing keys by appending labels. - Theta consistency follows tagging semantics: the ``grid`` tag declares - which binning a set of points lives on, so all same-length theta arrays - under one tag value must be bitwise identical — sacc itself never - validates angles across data types/tracers, and a grid that differs - only at floating-point level chokes CosmoSIS downstream instead of - failing loud here. Different lengths within a tag group pass (scale-cut - subsets are legitimate); grids under different tag values are - unconstrained (``reporting`` vs ``integration`` differ by design). + Grid consistency follows tagging semantics: the ``grid`` tag declares + which binning a set of points lives on, so all same-length theta (or + ell) arrays under one tag value must be bitwise identical — sacc itself + never validates angles across data types/tracers, and a grid that + differs only at floating-point level chokes CosmoSIS downstream instead + of failing loud here. Different lengths within a tag group pass + (scale-cut subsets are legitimate); grids under different tag values + are unconstrained (``reporting`` vs ``integration`` differ by design); + θ and ℓ are separate domains, each checked against itself only. ℓ + series sharing a bitwise-equal grid must also share the bandpower + window (series without windows skip that check). Parameters ---------- @@ -657,9 +674,10 @@ def merge(saccs): Raises ------ ValueError - If metadata conflicts, a shared tracer differs across inputs, or two - same-length theta arrays under the same ``grid`` tag value are not - bitwise identical. + If metadata conflicts, a shared tracer differs across inputs, two + same-length theta or ell arrays under the same ``grid`` tag value + are not bitwise identical, or two ℓ series sharing a grid carry + different bandpower windows. """ saccs = list(saccs) metadata = {} @@ -697,56 +715,76 @@ def merge(saccs): merged = sacc.concatenate_data_sets(*stripped, same_tracers=same_tracers) for key, value in metadata.items(): merged.metadata[key] = value - _check_theta_consistency(merged) + _check_grid_consistency(merged, "theta") + _check_grid_consistency(merged, "ell") return merged -def _theta_groups(s): - """Nested map ``grid-tag-value -> (data_type, tracers) -> theta array``. +def _grid_groups(s, angle): + """Nested map ``grid-tag-value -> (data_type, tracers) -> point indices``. - One entry per ``(data_type, tracers)`` series carrying a ``theta`` tag, - in each series' own insertion order (never re-sorted), nested under the - ``grid`` tag value it lives on (``None`` for untagged series) — the - shape ``merge``'s consistency guard checks within each tag value. + One entry per ``(data_type, tracers)`` series carrying an ``angle`` + (``'theta'`` or ``'ell'``) tag, in each series' own insertion order + (never re-sorted), nested under the ``grid`` tag value it lives on + (``None`` for untagged series) — the shape ``merge``'s consistency + guard checks within each tag value. Indices (not angle values) are + kept so the ℓ check can also recover each series' bandpower window. """ groups = {} - for point in s.data: - if "theta" not in point.tags: - continue - by_series = groups.setdefault(point.tags.get("grid"), {}) - by_series.setdefault((point.data_type, point.tracers), []).append( - point.tags["theta"] - ) - return { - tag: {key: np.asarray(theta) for key, theta in by_series.items()} - for tag, by_series in groups.items() - } + for i, point in enumerate(s.data): + if angle in point.tags: + groups.setdefault(point.tags.get("grid"), {}).setdefault( + (point.data_type, point.tracers), [] + ).append(i) + return groups -def _check_theta_consistency(s): - """Raise unless same-length theta arrays under one ``grid`` tag match. +def _check_grid_consistency(s, angle): + """Raise unless same-length ``angle`` arrays under one ``grid`` tag match. Consistency follows tagging semantics: the ``grid`` tag declares which - binning a series lives on, so all same-length theta arrays sharing a - tag value must be bitwise identical — sacc never validates angles - across data types/tracers, and a grid diverging at floating-point level - chokes CosmoSIS downstream instead of failing loud here. Different - lengths within a tag value pass (scale-cut subsets are legitimate); - series under different tag values are unconstrained (``reporting`` vs - ``integration`` differ by design). + binning a series lives on, so all same-length theta (or ell) arrays + sharing a tag value must be bitwise identical — sacc never validates + angles across data types/tracers, and a grid diverging at + floating-point level chokes CosmoSIS downstream instead of failing + loud here. Different lengths within a tag value pass (scale-cut + subsets are legitimate); series under different tag values are + unconstrained (``reporting`` vs ``integration`` differ by design). + θ and ℓ are separate domains, each checked against itself only. For + ℓ, two series on a bitwise-equal grid must also share the bandpower + window (equal window ells and weight matrix); series without windows + (foreign files) skip the window check. """ - for tag, by_series in _theta_groups(s).items(): - series = list(by_series.items()) - for i, (key_a, theta_a) in enumerate(series): - for key_b, theta_b in series[i + 1 :]: - if len(theta_a) != len(theta_b) or np.array_equal(theta_a, theta_b): + for tag, by_series in _grid_groups(s, angle).items(): + series = [ + (key, np.array([s.data[i].tags[angle] for i in idx]), idx) + for key, idx in by_series.items() + ] + for i, (key_a, arr_a, idx_a) in enumerate(series): + for key_b, arr_b, idx_b in series[i + 1 :]: + if len(arr_a) != len(arr_b): continue - max_diff = np.max(np.abs(theta_a - theta_b)) - raise ValueError( - f"theta grids under the same grid tag ({tag!r}) differ; " - f"harmonize upstream — groups {key_a!r} and {key_b!r} " - f"(max abs diff {max_diff:.3e})" - ) + if not np.array_equal(arr_a, arr_b): + max_diff = np.max(np.abs(arr_a - arr_b)) + raise ValueError( + f"{angle} grids under the same grid tag ({tag!r}) " + f"differ; harmonize upstream — groups {key_a!r} and " + f"{key_b!r} (max abs diff {max_diff:.3e})" + ) + if angle != "ell" or any( + "window" not in s.data[idx[0]].tags for idx in (idx_a, idx_b) + ): + continue + win_a, win_b = map(s.get_bandpower_windows, (idx_a, idx_b)) + if not ( + np.array_equal(win_a.values, win_b.values) + and np.array_equal(win_a.weight, win_b.weight) + ): + raise ValueError( + f"bandpower windows differ between series sharing an " + f"ell grid under grid tag {tag!r}; harmonize upstream " + f"— groups {key_a!r} and {key_b!r}" + ) def update_statistic(s, sub): diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 10b8ea01..8cd3279e 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -867,12 +867,52 @@ def test_merge_same_length_grids_under_different_tags_pass(): assert len(merged.mean) == len(s.mean) + 12 -def test_merge_same_tag_clearly_different_grids_raise(): - # same (untagged) group, same length, values far apart -> still an error - s_rho = _rho_sacc(theta=_theta()) # theta in [1, 100] - s_rho2 = _rho_sacc(k=1, theta=np.geomspace(200.0, 400.0, 6)) - with pytest.raises(ValueError, match="theta grids under the same grid tag"): - sio.merge([s_rho, s_rho2]) +def _cl_sacc(bin=0, ell=None, W=None, grid="reporting"): + ell = np.array([30.0, 120.0, 210.0, 300.0]) if ell is None else ell + nell, nbp = 50, len(ell) + W = np.random.default_rng(5).uniform(size=(nell, nbp)) if W is None else W + s = sio.new_sacc({bin: _nz(bin)}) + sio.add_pseudo_cl( + s, + (bin, bin), + ell, + np.arange(nbp) * 1e-9, + np.arange(nbp) * 2e-9, + np.arange(nbp) * 3e-9, + window_ells=np.arange(2, 2 + nell).astype(float), + window_weights=W, + grid=grid, + ) + return s + + +def test_merge_identical_ell_grids_and_windows_pass(): + s_a, s_b = _cl_sacc(bin=0), _cl_sacc(bin=1) # same ell, same window + merged = sio.merge([s_a, s_b]) + assert len(merged.mean) == len(s_a.mean) + len(s_b.mean) + + +def test_merge_nearly_identical_ell_grids_raises(): + ell = np.array([30.0, 120.0, 210.0, 300.0]) + s_a = _cl_sacc(bin=0, ell=ell) + s_b = _cl_sacc(bin=1, ell=ell * (1 + 1e-9)) # same 'reporting' group + with pytest.raises(ValueError, match="ell grids under the same grid tag"): + sio.merge([s_a, s_b]) + + +def test_merge_shared_ell_grid_different_windows_raises(): + W = np.random.default_rng(5).uniform(size=(50, 4)) + s_a = _cl_sacc(bin=0, W=W) + s_b = _cl_sacc(bin=1, W=W * (1 + 1e-6)) # same ell, perturbed window + with pytest.raises(ValueError, match="bandpower windows differ"): + sio.merge([s_a, s_b]) + + +def test_merge_different_ell_grids_across_tags_pass(): + s_a = _cl_sacc(bin=0) # grid='reporting' + s_b = _cl_sacc(bin=1, ell=np.array([40.0, 130.0, 220.0, 310.0]), grid="finer") + merged = sio.merge([s_a, s_b]) # different tag values: unconstrained + assert len(merged.mean) == len(s_a.mean) + len(s_b.mean) # --------------------------------------------------------------------------- # From 01fa5eb712401546634e4586c23a5b151b52c9ff Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 13:41:20 +0200 Subject: [PATCH 015/160] Store covariance block-diagonally in assemble_covariance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit assemble_covariance now passes sacc.add_covariance a list of per-block matrices instead of a dense zero-filled N×N array. sacc.BaseCovariance.make turns a list into a BlockDiagonalCovariance (one FITS table per block, Σ block² on disk), so cross-blocks are zero and implicit rather than materialized. Validation (contiguous/ascending indices, no gap/overlap, square blocks matching their index span) is unchanged. Every existing consumer already read through the polymorphic .dense property, so only the type assertions needed updating (FullCovariance -> BlockDiagonalCovariance). Added a test that merging two files that already carry a BlockDiagonalCovariance (assembled via assemble_covariance) stays block-diagonal through merge and a save/load round-trip. Updated the module docstring's storage-cost discussion accordingly. Co-Authored-By: Claude Fable 5 --- src/sp_validation/sacc_io.py | 34 ++++++++++++++----------- src/sp_validation/tests/test_sacc_io.py | 30 ++++++++++++++++++++-- 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 3df6488a..28600e47 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -11,15 +11,16 @@ diagnostics, and the fine ξ± integration input for COSEBIs / pure-EB (``grid='integration'`` tagged points). The covariance is assembled block-diagonally from the - per-statistic covariances (zero cross-blocks): the analysis - blocks first, then a dense per-pair integration-ξ block (the - analytic integration-binning covariance when it exists — it - feeds derived-statistic error propagation — or the TreeCorr - ``varxip``/``varxim`` diagonal as degraded fallback). At the - production integration binning (1000 θ bins) a dense block is - ~32 MB per pair; extreme convergence-check grids (10k bins) - degrade to the diagonal fallback rather than forking the - layout. + per-statistic covariances (zero cross-blocks, never + materialized): the analysis blocks first, then a dense + per-pair integration-ξ block (the analytic integration-binning + covariance when it exists — it feeds derived-statistic error + propagation — or the TreeCorr ``varxip``/``varxim`` diagonal as + degraded fallback). ``assemble_covariance`` hands sacc a list + of blocks, which stores a ``BlockDiagonalCovariance`` (one + FITS table per block, Σ block² on disk rather than a dense + N²) — cost scales with the integration grid's size, a + parameter set by the caller, not baked into the layout. Insertion order is load-bearing. A Sacc is a flat list of data points in the order ``add_data_point`` was called, and row/column @@ -374,13 +375,16 @@ def add_tau(s, bins, k, theta, tau_p, tau_m, *, grid="reporting"): def assemble_covariance(s, blocks): - """Assemble a block-diagonal ``FullCovariance`` from per-statistic blocks. + """Assemble a ``BlockDiagonalCovariance`` from per-statistic blocks. Each block is validated against the current insertion order: its indices must be contiguous and ascending, the blocks must tile ``0…len(s.mean)`` exactly (no gap, no overlap), and each block must be square with a size matching its index span. Any violation raises ``ValueError`` naming the - mismatch. Cross-blocks are left zero. + mismatch. Cross-blocks are zero and implicit — never materialized — + because the blocks are passed to ``add_covariance`` as a list, which + ``sacc.BaseCovariance.make`` turns into a ``BlockDiagonalCovariance`` + (one FITS table per block, Σ block² on disk rather than a dense N² file). Parameters ---------- @@ -395,11 +399,11 @@ def assemble_covariance(s, blocks): Returns ------- sacc.Sacc - ``s``, with the assembled ``FullCovariance`` attached. + ``s``, with the assembled ``BlockDiagonalCovariance`` attached. """ items = blocks.items() if isinstance(blocks, dict) else blocks ntot = len(s.mean) - full = np.zeros((ntot, ntot)) + ordered_blocks = [] cursor = 0 for selector, cov in items: idx = _resolve_indices(s, selector) @@ -424,14 +428,14 @@ def assemble_covariance(s, blocks): f"covariance block {selector!r} has size {cov.shape[0]} but " f"spans {len(idx)} data points" ) - full[np.ix_(idx, idx)] = cov + ordered_blocks.append(cov) cursor = idx[-1] + 1 if cursor != ntot: raise ValueError( f"covariance blocks cover {cursor} of {ntot} data points — the " "blocks must tile the whole data vector" ) - s.add_covariance(full) + s.add_covariance(ordered_blocks) return s diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 8cd3279e..1ecf1322 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -232,7 +232,7 @@ def test_assemble_covariance_alignment(tmp_path): cov_xi, cov_cl, cov_co = _spd(len(xi), 1), _spd(len(cl), 2), _spd(len(co), 3) sio.assemble_covariance(s, [(xi, cov_xi), (cl, cov_cl), (co, cov_co)]) s2 = _roundtrip(s, tmp_path, "cov") - assert type(s2.covariance).__name__ == "FullCovariance" + assert type(s2.covariance).__name__ == "BlockDiagonalCovariance" dense = s2.covariance.dense # each block's sub-covariance is exactly what went in assert np.array_equal(dense[np.ix_(xi, xi)], cov_xi) @@ -258,7 +258,7 @@ def test_assemble_covariance_selector_tuples(): ((sio.COSEBI_BB, tr), _spd(len(s.indices(sio.COSEBI_BB, tr)), 4)), ], ) - assert type(s.covariance).__name__ == "FullCovariance" + assert type(s.covariance).__name__ == "BlockDiagonalCovariance" assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) @@ -734,6 +734,32 @@ def test_merge_covariance_block_diagonal(): assert np.all(dense[:n_xi, n_xi:] == 0) +def test_merge_block_diagonal_covariance_stays_block_diagonal(tmp_path): + """Merging two files that already carry a BlockDiagonalCovariance (e.g. + each assembled via ``assemble_covariance``) must not densify — the + result stays a ``BlockDiagonalCovariance``, on disk too.""" + s_xi, s_co = _xi_sacc(), _cosebi_sacc() + sio.assemble_covariance( + s_xi, [(np.arange(len(s_xi.mean)), _spd(len(s_xi.mean), 1))] + ) + sio.assemble_covariance( + s_co, [(np.arange(len(s_co.mean)), _spd(len(s_co.mean), 2))] + ) + assert type(s_xi.covariance).__name__ == "BlockDiagonalCovariance" + merged = sio.merge([s_xi, s_co]) + assert type(merged.covariance).__name__ == "BlockDiagonalCovariance" + sio.save(merged, str(tmp_path / "vBLK.sacc"), type="mock") + merged_rt = sio.load(str(tmp_path / "vBLK.sacc")) + assert type(merged_rt.covariance).__name__ == "BlockDiagonalCovariance" + n_xi = len(s_xi.mean) + assert np.array_equal( + merged_rt.covariance.dense[:n_xi, :n_xi], s_xi.covariance.dense + ) + assert np.array_equal( + merged_rt.covariance.dense[n_xi:, n_xi:], s_co.covariance.dense + ) + + def test_merge_mixed_covariance_fails(): s_xi, s_co = _xi_sacc(), _cosebi_sacc() s_xi.add_covariance(_spd(len(s_xi.mean), 1)) # s_co has none From b131ae7e60b033d1841976ea9a69bb1f55e7ced2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 13:41:31 +0200 Subject: [PATCH 016/160] Unify pure-EB integration-grid default to 1000 bins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pure_eb.py's calculate_pure_eb/plot_pure_eb defaulted nbins_int=100; cosebis.py's calculate_cosebis already defaulted to 1000, and every production config (papers/bmodes, papers/cosmo_val fiducial/pure_eb blocks) already overrides to 1000. This aligns the function-signature default with what every caller actually uses; config plumbing is untouched, so any explicit override still wins. The papers/cosmo_val/config/config.yaml cosebis.nbins_int (currently 2000, production numerics) is deliberately left unchanged — see report. Co-Authored-By: Claude Fable 5 --- src/sp_validation/cosmo_val/pure_eb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 7524074e..71f5d76a 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -27,7 +27,7 @@ def calculate_pure_eb( nbins=None, min_sep_int=0.08, max_sep_int=300, - nbins_int=100, + nbins_int=1000, npatch=256, var_method="jackknife", cov_path_int=None, @@ -57,7 +57,7 @@ def calculate_pure_eb( max_sep_int : float, optional Maximum separation for the integration binning. Defaults to 300. nbins_int : int, optional - Number of bins for the integration binning. Defaults to 100. + Number of bins for the integration binning. Defaults to 1000. npatch : int, optional Number of patches for the jackknife or bootstrap resampling. Defaults to the value in self.npatch if not provided. @@ -143,7 +143,7 @@ def plot_pure_eb( nbins=None, min_sep_int=0.08, max_sep_int=300, - nbins_int=100, + nbins_int=1000, npatch=None, var_method="jackknife", cov_path_int=None, @@ -175,7 +175,7 @@ def plot_pure_eb( Binning parameters for reporting scale. Uses treecorr_config if None. min_sep_int, max_sep_int, nbins_int : float, float, int Binning parameters for integration scale - (default: 0.08-300 arcmin, 100 bins) + (default: 0.08-300 arcmin, 1000 bins) npatch : int, optional Number of patches for jackknife covariance. Uses self.npatch if None. var_method : str From df28f426484b8108f5aef5f4087bc087bd22a856 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 14:09:21 +0200 Subject: [PATCH 017/160] sacc_io: make optional statistic components genuinely optional Writers no longer force components an analysis may not have computed: add_pseudo_cl's BB/EB, add_cosebis's Bn, and add_pure_eb's B/ambiguous blocks now default to None and are simply not written when omitted (add_pure_eb still requires each +/- pair together). Structural arguments (grids, tracer/bin identifiers, the value for a component you ARE adding) remain required with no default. Composite readers (get_pseudo_cl, get_cosebis, get_pure_eb) return None / omit the key for an absent optional component instead of raising, via a new _mean_optional helper; a selection naming a missing component explicitly (s.indices, _mean, extract) still fails loud, per the existing empty-selection guard. merge() and assemble_covariance work unchanged on files with only a subset of components. Documents the optionality contract in the module docstring, and adds partial-file round-trip, merge, and explicit-selection-fails-loud tests. Co-Authored-By: Claude Fable 5 --- src/sp_validation/sacc_io.py | 164 +++++++++++++++++----- src/sp_validation/tests/test_sacc_io.py | 173 +++++++++++++++++++++++- 2 files changed, 298 insertions(+), 39 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 28600e47..a16079f9 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -50,6 +50,26 @@ the covariance was built in. Converters that need a type-major layout (e.g. the DES 2pt-FITS convention) permute explicitly via ``s.indices`` rather than assuming global order. + +Optionality: a file's contents are flexible about which components of + a statistic it actually has. ``add_pseudo_cl`` requires EE (the + bandpower window's reference series) but BB and EB are optional + — an analysis that never computed EB simply omits it. + ``add_cosebis`` requires Eₙ but Bₙ is optional. ``add_pure_eb`` + requires xip_E/xim_E but the B and ambiguous-mode blocks are + each optional, independently (B and amb are unrelated + computations). Everything else a writer takes — θ/ℓ grids, + tracer/bin identifiers, the value array for a component you ARE + adding — is structurally necessary and has no default; + supplying it partially would desynchronise the covariance + layout, so it is refused rather than degraded. Readers mirror + this split: a composite reader (``get_pseudo_cl``, + ``get_cosebis``, ``get_pure_eb``) returns ``None`` (or omits the + key) for a component the file doesn't carry, but a selection + naming that component explicitly (``s.indices``, ``_mean``, + ``extract``) still fails loud on no match — silence is reserved + for "this file doesn't have that optional piece", never for + "you asked for something specific and it isn't there". """ import numpy as np @@ -216,14 +236,14 @@ def add_pseudo_cl( bins, ell_eff, cl_ee, - cl_bb, - cl_eb, + cl_bb=None, + cl_eb=None, *, window_ells, window_weights, grid="reporting", ): - """Add pseudo-Cℓ (EE, BB, EB) with a shared bandpower window. + """Add pseudo-Cℓ EE (required) plus whichever of BB/EB were computed. Parameters ---------- @@ -233,14 +253,19 @@ def add_pseudo_cl( Source bin pair ``(i, j)``. ell_eff : array_like Effective multipole of each bandpower. - cl_ee, cl_bb, cl_eb : array_like - EE, BB and EB bandpowers at ``ell_eff``. + cl_ee : array_like + EE bandpowers at ``ell_eff``. + cl_bb, cl_eb : array_like, optional + BB and/or EB bandpowers at ``ell_eff``. Each defaults to ``None`` and + is then simply not written — EB in particular is often not computed + at all. window_ells : array_like Multipoles spanned by the bandpower window matrix (shape ``(nell,)``). window_weights : array_like Window matrix ``W`` of shape ``(nell, nbp)`` — one column per bandpower — from NaMaster ``get_bandpower_windows``. One - ``sacc.BandpowerWindow`` is built and shared across EE/BB/EB. + ``sacc.BandpowerWindow`` is built and shared across every component + written. grid : str, optional Stored as the ``grid`` tag on every point (default ``'reporting'``), joining ``merge``'s ℓ-consistency group; variant ℓ binnings belong @@ -251,7 +276,11 @@ def add_pseudo_cl( window = sacc.BandpowerWindow(np.asarray(window_ells), np.asarray(window_weights)) # add_ell_cl accepts no extra tags, so inline its per-point insertion # (ell + shared window + window_ind column index) plus the grid tag. - for dtype, cl in ((CL_EE, cl_ee), (CL_BB, cl_bb), (CL_EB, cl_eb)): + components = [(CL_EE, cl_ee)] + components += [ + (dtype, cl) for dtype, cl in ((CL_BB, cl_bb), (CL_EB, cl_eb)) if cl is not None + ] + for dtype, cl in components: for n, (ell, value) in enumerate(zip(ell_eff, cl)): s.add_data_point( dtype, @@ -264,8 +293,8 @@ def add_pseudo_cl( ) -def add_cosebis(s, bins, En, Bn, scale_cut): - """Add COSEBIs (all Eₙ then all Bₙ) for one scale cut. +def add_cosebis(s, bins, En, scale_cut, Bn=None): + """Add COSEBIs Eₙ (required) and Bₙ (optional) for one scale cut. Parameters ---------- @@ -273,17 +302,23 @@ def add_cosebis(s, bins, En, Bn, scale_cut): Target, mutated in place. bins : tuple of int Source bin pair ``(i, j)``. - En, Bn : array_like - E- and B-mode COSEBI amplitudes, one per logarithmic mode ``n`` - (1-based). The ``[En; Bn]`` layout matches the COSEBI covariance. + En : array_like + E-mode COSEBI amplitudes, one per logarithmic mode ``n`` (1-based). scale_cut : tuple of float ``(theta_min, theta_max)`` in arcmin, stored on every point as the ``theta_min``/``theta_max`` tags; multiple cuts coexist in one file, told apart by these tags. + Bn : array_like, optional + B-mode COSEBI amplitudes at the same ``n``. Defaults to ``None`` and + is then simply not written. The ``[En; Bn]`` layout, when both are + present, matches the COSEBI covariance. """ tracers = _pair(bins) theta_min, theta_max = scale_cut - for dtype, modes in ((COSEBI_EE, En), (COSEBI_BB, Bn)): + components = [(COSEBI_EE, En)] + if Bn is not None: + components.append((COSEBI_BB, Bn)) + for dtype, modes in components: for n, value in enumerate(modes, start=1): s.add_data_point( dtype, @@ -296,13 +331,23 @@ def add_cosebis(s, bins, En, Bn, scale_cut): def add_pure_eb( - s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb, *, grid="reporting" + s, + bins, + theta, + xip_E, + xim_E, + xip_B=None, + xim_B=None, + xip_amb=None, + xim_amb=None, + *, + grid="reporting", ): - """Add pure E/B-mode correlation functions for one tracer pair. + """Add pure E-mode (required) plus whichever of B/ambiguous were computed. - Six blocks are inserted in ``PURE_KEYS`` order (xip_E, xim_E, xip_B, - xim_B, xip_amb, xim_amb), matching ``b_modes._EB_KEYS`` and the pure-EB - covariance layout. + Blocks are inserted in ``PURE_KEYS`` order (xip_E, xim_E, xip_B, xim_B, + xip_amb, xim_amb), matching ``b_modes._EB_KEYS`` and the pure-EB + covariance layout — whichever subset is present. Parameters ---------- @@ -311,18 +356,43 @@ def add_pure_eb( bins : tuple of int Source bin pair ``(i, j)``. theta : array_like - Angular separations (arcmin), shared by all six blocks. - xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb : array_like - The six pure E/B / ambiguous mode arrays at ``theta``. + Angular separations (arcmin), shared by every block written. + xip_E, xim_E : array_like + The pure E-mode correlation functions at ``theta``. + xip_B, xim_B : array_like, optional + The pure B-mode correlation functions. Both default to ``None``; the + pair is written together or not at all — supply both or neither. + xip_amb, xim_amb : array_like, optional + The ambiguous-mode correlation functions. Both default to ``None``; + same both-or-neither rule as B. grid : str, optional Stored as the ``grid`` tag on every point (default ``'reporting'``), joining ξ's theta-consistency group in ``merge``'s guard. """ _check_ascending("theta", theta) tracers = _pair(bins) - arrays = (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb) - for dtype, arr in zip(PURE_TYPES.values(), arrays): - _add_theta_series(s, dtype, tracers, theta, arr, grid=grid) + pairs = { + "B": (xip_B, xim_B), + "amb": (xip_amb, xim_amb), + } + for label, (p, m) in pairs.items(): + if (p is None) != (m is None): + raise ValueError( + f"add_pure_eb: xip_{label} and xim_{label} must both be " + "given or both omitted" + ) + values = { + "xip_E": xip_E, + "xim_E": xim_E, + "xip_B": xip_B, + "xim_B": xim_B, + "xip_amb": xip_amb, + "xim_amb": xim_amb, + } + for key in PURE_KEYS: + arr = values[key] + if arr is not None: + _add_theta_series(s, PURE_TYPES[key], tracers, theta, arr, grid=grid) def add_rho(s, k, theta, rho_p, rho_m, *, grid="reporting"): @@ -495,18 +565,20 @@ def get_xi(s, bins, *, grid): def get_pseudo_cl(s, bins): """Return ``(ell_eff, cl_ee, cl_bb, cl_eb, window)`` for one tracer pair. - ``window`` is the shared ``sacc.BandpowerWindow`` recovered via - ``get_bandpower_windows``; its columns are in the same insertion order as - the returned ``ell_eff``/``cl`` arrays, so window column ``j`` corresponds - to ``ell_eff[j]``. + ``cl_bb``/``cl_eb`` come back ``None`` if the file doesn't carry that + component (``add_pseudo_cl`` makes both optional). ``window`` is the + shared ``sacc.BandpowerWindow`` recovered via ``get_bandpower_windows``; + its columns are in the same insertion order as the returned + ``ell_eff``/``cl`` arrays, so window column ``j`` corresponds to + ``ell_eff[j]``. """ tracers = _pair(bins) window = s.get_bandpower_windows(s.indices(CL_EE, tracers)) return ( _tag(s, CL_EE, tracers, "ell"), _mean(s, CL_EE, tracers), - _mean(s, CL_BB, tracers), - _mean(s, CL_EB, tracers), + _mean_optional(s, CL_BB, tracers), + _mean_optional(s, CL_EB, tracers), window, ) @@ -514,6 +586,9 @@ def get_pseudo_cl(s, bins): def get_cosebis(s, bins, scale_cut=None): """Return ``(n, En, Bn)`` for one tracer pair. + ``Bn`` comes back ``None`` if the file doesn't carry it (``add_cosebis`` + makes it optional). + Parameters ---------- scale_cut : tuple of float, optional @@ -535,18 +610,26 @@ def get_cosebis(s, bins, scale_cut=None): return ( modes.astype(int), _mean(s, COSEBI_EE, tracers, **tags), - _mean(s, COSEBI_BB, tracers, **tags), + _mean_optional(s, COSEBI_BB, tracers, **tags), ) def get_pure_eb(s, bins): - """Return ``(theta, {key: array})`` for the six pure-EB blocks. + """Return ``(theta, {key: array})`` for whichever pure-EB blocks exist. - The dict is keyed by ``PURE_KEYS`` (xip_E, xim_E, …). + xip_E/xim_E are always present (``add_pure_eb`` requires them); the dict + holds whichever of the B and ambiguous-mode keys (out of ``PURE_KEYS``: + xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb) the file actually carries — + a key absent from the file is simply absent from the dict, not mapped to + ``None``. """ tracers = _pair(bins) theta = _tag(s, PURE_TYPES["xip_E"], tracers, "theta") - arrays = {key: _mean(s, PURE_TYPES[key], tracers) for key in PURE_KEYS} + arrays = {} + for key in PURE_KEYS: + values = _mean_optional(s, PURE_TYPES[key], tracers) + if values is not None: + arrays[key] = values return theta, arrays @@ -597,6 +680,19 @@ def _tag(s, data_type, tracers, tag, **tag_filters): return np.array([s.data[i].tags[tag] for i in idx]) +def _mean_optional(s, data_type, tracers, **tag_filters): + """Mean values for a selection, or ``None`` if the file has none. + + Used by composite readers (``get_pseudo_cl``, ``get_cosebis``, + ``get_pure_eb``) for the components a writer made optional (BB/EB, + COSEBI Bₙ, pure B/ambiguous): absence is a legitimate "this file doesn't + have that piece", not a typo to fail loud on — unlike ``_mean``/ + ``_indices``, used for selections that name a component explicitly. + """ + idx = np.asarray(s.indices(data_type, tracers, **tag_filters), dtype=int) + return s.mean[idx] if len(idx) else None + + def extract(s, data_type=None, tracers=None, **tag_filters): """Extract a sub-Sacc (points + aligned covariance sub-block). diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 1ecf1322..8f8564ae 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -137,7 +137,7 @@ def test_pseudo_cl_roundtrip(tmp_path): def test_cosebis_roundtrip(tmp_path): En, Bn = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 s = _base_sacc() - sio.add_cosebis(s, (0, 0), En, Bn, (1.0, 100.0)) + sio.add_cosebis(s, (0, 0), En, (1.0, 100.0), Bn=Bn) s2 = _roundtrip(s, tmp_path, "cosebi") n, E, B = sio.get_cosebis(s2, (0, 0)) assert np.array_equal(n, np.arange(1, 11)) @@ -219,7 +219,7 @@ def _multi_statistic_sacc(): window_weights=W, ) sio.add_cosebis( - s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) + s, (0, 0), np.arange(1, 6) * 1e-6, (1.0, 100.0), Bn=np.arange(1, 6) * 1e-7 ) return s @@ -445,7 +445,7 @@ def test_end_to_end_one_file_layout(tmp_path): s, (0, 0), theta_c, np.arange(20) * 1e-5, np.arange(20) * 2e-5, grid="reporting" ) sio.add_cosebis( - s, (0, 0), np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7, (1.0, 100.0) + s, (0, 0), np.arange(1, 11) * 1e-6, (1.0, 100.0), Bn=np.arange(1, 11) * 1e-7 ) sio.add_xi( s, @@ -696,7 +696,7 @@ def _xi_sacc(metadata=None): def _cosebi_sacc(metadata=None): s = sio.new_sacc({0: _nz(0)}, metadata=metadata) sio.add_cosebis( - s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) + s, (0, 0), np.arange(1, 6) * 1e-6, (1.0, 100.0), Bn=np.arange(1, 6) * 1e-7 ) return s @@ -962,7 +962,7 @@ def test_get_cosebis_raises_on_unmatched_scale_cut(): def test_get_cosebis_rejects_ambiguous_multi_cut_file(): s = _cosebi_sacc() sio.add_cosebis( - s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (2.0, 50.0) + s, (0, 0), np.arange(1, 6) * 1e-6, (2.0, 50.0), Bn=np.arange(1, 6) * 1e-7 ) with pytest.raises(ValueError, match="several COSEBIs scale cuts"): sio.get_cosebis(s, (0, 0)) @@ -1019,3 +1019,166 @@ def test_pseudo_cl_window_column_correspondence(tmp_path): col = s2.data[i].tags["window_ind"] assert col == pos # insertion order preserved => column j <-> ell[j] assert np.array_equal(window.weight[:, col], W[:, pos]) + + +# --------------------------------------------------------------------------- # +# 16. Optionality: writers omit optional components; readers tolerate their +# absence in composite reads but still fail loud on an explicit selection +# naming a component that isn't there. +# --------------------------------------------------------------------------- # +def test_pseudo_cl_ee_bb_only_no_eb(tmp_path): + """EB is often not even computed — add_pseudo_cl must not require it.""" + ell_eff = np.array([30.0, 120.0, 210.0]) + nell, nbp = 20, len(ell_eff) + window_ells = np.arange(2, 2 + nell).astype(float) + W = np.random.default_rng(1).uniform(size=(nell, nbp)) + ee, bb = np.arange(nbp) * 1e-9, np.arange(nbp) * 2e-9 + s = _base_sacc() + sio.add_pseudo_cl( + s, (0, 0), ell_eff, ee, bb, window_ells=window_ells, window_weights=W + ) + tr = ("source_0", "source_0") + assert len(s.indices(sio.CL_EB, tr)) == 0 + s2 = _roundtrip(s, tmp_path, "cl_ee_bb") + ell, cl_ee, cl_bb, cl_eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell, ell_eff) + assert np.array_equal(cl_ee, ee) + assert np.array_equal(cl_bb, bb) + assert cl_eb is None + + +def test_pseudo_cl_ee_only(tmp_path): + ell_eff = np.array([30.0, 120.0, 210.0]) + nell, nbp = 20, len(ell_eff) + W = np.random.default_rng(2).uniform(size=(nell, nbp)) + ee = np.arange(nbp) * 1e-9 + s = _base_sacc() + sio.add_pseudo_cl( + s, + (0, 0), + ell_eff, + ee, + window_ells=np.arange(2, 2 + nell).astype(float), + window_weights=W, + ) + s2 = _roundtrip(s, tmp_path, "cl_ee_only") + ell, cl_ee, cl_bb, cl_eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(cl_ee, ee) + assert cl_bb is None + assert cl_eb is None + + +def test_cosebis_en_only_no_bn(tmp_path): + En = np.arange(1, 8) * 1e-6 + s = _base_sacc() + sio.add_cosebis(s, (0, 0), En, (1.0, 100.0)) + tr = ("source_0", "source_0") + assert len(s.indices(sio.COSEBI_BB, tr)) == 0 + s2 = _roundtrip(s, tmp_path, "cosebi_e_only") + n, E, B = sio.get_cosebis(s2, (0, 0)) + assert np.array_equal(E, En) + assert B is None + + +def test_pure_eb_e_only_no_b_no_amb(tmp_path): + theta = _theta() + xip_E, xim_E = np.arange(6) * 1e-6, np.arange(6) * 2e-6 + s = _base_sacc() + sio.add_pure_eb(s, (0, 0), theta, xip_E, xim_E) + s2 = _roundtrip(s, tmp_path, "pureeb_e_only") + th, back = sio.get_pure_eb(s2, (0, 0)) + assert np.array_equal(th, theta) + assert set(back) == {"xip_E", "xim_E"} + assert np.array_equal(back["xip_E"], xip_E) + assert np.array_equal(back["xim_E"], xim_E) + + +def test_pure_eb_e_and_b_no_amb(tmp_path): + theta = _theta() + arrays = { + key: np.arange(6) * (i + 1) * 1e-6 + for i, key in enumerate(("xip_E", "xim_E", "xip_B", "xim_B")) + } + s = _base_sacc() + sio.add_pure_eb(s, (0, 0), theta, **arrays) + s2 = _roundtrip(s, tmp_path, "pureeb_e_b") + th, back = sio.get_pure_eb(s2, (0, 0)) + assert set(back) == {"xip_E", "xim_E", "xip_B", "xim_B"} + + +def test_pure_eb_rejects_half_a_pair(): + theta = _theta() + xip_E, xim_E = np.arange(6) * 1e-6, np.arange(6) * 2e-6 + s = _base_sacc() + with pytest.raises(ValueError, match="xip_B and xim_B"): + sio.add_pure_eb(s, (0, 0), theta, xip_E, xim_E, xip_B=np.arange(6) * 1e-6) + with pytest.raises(ValueError, match="xip_amb and xim_amb"): + sio.add_pure_eb(s, (0, 0), theta, xip_E, xim_E, xim_amb=np.arange(6) * 1e-6) + + +def test_xi_only_plus_covariance_file(tmp_path): + """A file with only xi +/- and a covariance — no Cl/COSEBIs at all.""" + theta = _theta() + s = _base_sacc() + xip, xim = _add_xi(s) + tr = ("source_0", "source_0") + xi_idx = _xi_block(s, tr) + sio.assemble_covariance(s, [(xi_idx, _spd(len(xi_idx), 7))]) + s2 = _roundtrip(s, tmp_path, "xi_only") + th, p, m = sio.get_xi(s2, (0, 0), grid="reporting") + assert np.array_equal(th, theta) + assert np.array_equal(p, xip) + assert np.array_equal(m, xim) + assert s2.covariance is not None + with pytest.raises(ValueError, match="matched no points"): + sio._indices(s2, sio.CL_EE, tr) + + +def test_reader_explicit_selection_fails_loud_on_missing_optional_component(): + """Absence is silent for composite readers, but a targeted selection + naming a missing optional component (e.g. CL_EB) still fails loud.""" + ell_eff = np.array([30.0, 120.0, 210.0]) + nell, nbp = 20, len(ell_eff) + W = np.random.default_rng(3).uniform(size=(nell, nbp)) + s = _base_sacc() + sio.add_pseudo_cl( + s, + (0, 0), + ell_eff, + np.arange(nbp) * 1e-9, + window_ells=np.arange(2, 2 + nell).astype(float), + window_weights=W, + ) + tr = ("source_0", "source_0") + with pytest.raises(ValueError, match="matched no points"): + sio._mean(s, sio.CL_EB, tr) + with pytest.raises(ValueError, match="no points"): + sio.extract(s, data_type=sio.CL_EB, tracers=tr) + + +def test_merge_partial_files(): + """merge() combines a Cl file missing EB with a COSEBIs file missing Bn.""" + s_cl = sio.new_sacc({0: _nz(0)}) + ell_eff = np.array([30.0, 120.0, 210.0]) + nell, nbp = 20, len(ell_eff) + W = np.random.default_rng(4).uniform(size=(nell, nbp)) + sio.add_pseudo_cl( + s_cl, + (0, 0), + ell_eff, + np.arange(nbp) * 1e-9, + np.arange(nbp) * 2e-9, # BB only, no EB + window_ells=np.arange(2, 2 + nell).astype(float), + window_weights=W, + ) + s_co = sio.new_sacc({0: _nz(0)}) + sio.add_cosebis(s_co, (0, 0), np.arange(1, 6) * 1e-6, (1.0, 100.0)) # En only + + merged = sio.merge([s_cl, s_co]) + tr = ("source_0", "source_0") + assert len(merged.indices(sio.CL_EB, tr)) == 0 + assert len(merged.indices(sio.COSEBI_BB, tr)) == 0 + ell, cl_ee, cl_bb, cl_eb, window = sio.get_pseudo_cl(merged, (0, 0)) + assert cl_bb is not None and cl_eb is None + n, E, B = sio.get_cosebis(merged, (0, 0)) + assert B is None From da9a90aeb8c107fca247f37dc78eabd4b5cf3ede Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 14:09:28 +0200 Subject: [PATCH 018/160] cosmo_val config: align COSEBIs integration grid with the 1000-bin default cosebis.nbins_int was 2000; every other integration-grid entry in this config (pure_eb, the fiducial block) is already 1000, and cosebis.py's own function defaults are 1000. Unify. Co-Authored-By: Claude Fable 5 --- papers/cosmo_val/config/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 945f2f8f..98409371 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -68,7 +68,7 @@ cosmo_val: cosebis: min_sep_int: 0.9 max_sep_int: 300 - nbins_int: 2000 + nbins_int: 1000 npatch: 100 nmodes: 20 scale_cuts: [ From aa2f474ee254e04327b5cce5b3c027a9bc900a7e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 14:38:42 +0200 Subject: [PATCH 019/160] feat(sacc): one terminal file + fail-closed assembly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fold the integration-grid ξ± into the single terminal {version}.sacc, and make part loading fail closed on unblinded real data. Integration ξ± (grid='integration') is no longer its own terminal product. The xi_highres part is now gathered by rule assemble_sacc into {version}.sacc as tagged points, next to the reporting ξ± block. It is fiducial-only (the 10k-bin MPI run emits only the fiducial part), so it joins the fiducial version's terminal file alone. assemble_sacc.py adds xi_integration to CANONICAL; its own DiagonalCovariance passes straight through. assemble_sacc.py no longer loads every part with allow_unblinded=True. The run type (data|mock, from config, default data) gates it: mock runs load freely, data runs fail closed unless a part carries the concealed=True stamp. This is the seam for PR #253's blind-at-birth — a concealed data part then assembles with allow_unblinded=False untouched. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD --- src/sp_validation/cosmo_val/sacc_writers.py | 14 +++--- workflow/rules/cosmo_val.smk | 29 +++++++++-- workflow/rules/twopoint.smk | 8 +-- workflow/scripts/assemble_sacc.py | 54 ++++++++++++++++----- workflow/scripts/run_2pcf_highres.py | 8 +-- 5 files changed, 86 insertions(+), 27 deletions(-) diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index 72594650..10d1137f 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -11,10 +11,11 @@ ``sacc.concatenate_data_sets``, whose ``BlockDiagonalCovariance`` output the contract rules out). -The integration-grid ``{version}_xi_integration.sacc`` is a terminal product in -its own right (:func:`xi_to_sacc` with ``grid="integration"`` and a -``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim``); COSEBIs and pure-E/B -consume it. +The integration-grid ``{version}_xi_integration.sacc`` is one more part +(:func:`xi_to_sacc` with ``grid="integration"`` and a ``DiagonalCovariance`` from +TreeCorr ``varxip``/``varxim``); COSEBIs and pure-E/B consume it, and +:func:`assemble_analysis_sacc` folds its integration-grid ξ± rows (tagged +``grid="integration"``) into the single terminal ``{version}.sacc``. Everything here is single-bin today (``bins=(0, 0)``); the interface is tomography-native so a future round supplies real bin pairs unchanged. @@ -51,7 +52,7 @@ def xi_to_sacc( """One ξ± part (``bins=(0, 0)``) on the reporting or integration grid. ``variances`` (the concatenated ``[varxip; varxim]``) attaches a - ``DiagonalCovariance`` — used for the terminal integration file, where npatch=1 + ``DiagonalCovariance`` — used for the integration-grid part, where npatch=1 leaves TreeCorr shot-noise variance as the only covariance estimate. """ s = sio.new_sacc(nz, metadata) @@ -221,7 +222,8 @@ def assemble_analysis_sacc(nz, metadata, parts): Each part is a single-statistic Sacc (from a ``*_to_sacc`` writer, loaded from disk) carrying its own covariance = its block. This re-adds every part's data points into one Sacc in the order the parts are given — which - must be the canonical order (ξ± reporting, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ) + must be the canonical order (ξ± reporting, ξ± integration, pseudo-Cℓ, + COSEBIs, pure-E/B, ρ, τ) — and assembles a single ``FullCovariance`` from the per-part covariance blocks. Point insertion order and block order therefore agree by construction, which ``sacc_io.assemble_covariance`` validates (contiguous, diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index e8eb332d..126517d4 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -169,6 +169,16 @@ def cv_xi_reporting_sacc(version): ) +def cv_xi_integration_sacc(version): + """Integration-grid ξ± SACC part the xi_highres rule (run_2pcf_highres.py) writes. + + A per-statistic part (grid='integration', its own DiagonalCovariance from + TreeCorr varxip/varxim), not a terminal file: assemble_sacc folds its rows + into {version}.sacc. The name matches xi_highres's fiducial-keyed output. + """ + return str(COSMO_VAL / f"{version}_xi_integration.sacc") + + def cv_analysis_sacc(version): """Terminal assembled analysis file {version}.sacc.""" return str(COSMO_VAL / f"{version}.sacc") @@ -442,9 +452,11 @@ rule cv_summarize_bmodes: # --------------------------------------------------------------------------- # Terminal analysis file: assemble the per-statistic SACC parts into {version}.sacc # --------------------------------------------------------------------------- -# The five born-as-SACC parts (xi_reporting, pseudo_cl, cosebis, pure_eb, rho_tau) -# are each written by their own rule carrying its own covariance block, except -# ξ± reporting and pseudo-Cℓ which are born cov-less by design. assemble_sacc.py +# The born-as-SACC parts (xi_reporting, xi_integration, pseudo_cl, cosebis, +# pure_eb, rho_tau) are each written by their own rule carrying its own covariance +# block, except ξ± reporting and pseudo-Cℓ which are born cov-less by design. The +# integration-grid ξ± is fiducial-only, so it joins the fiducial version's file +# alone (see cv_assemble_inputs). assemble_sacc.py # loads the parts in canonical order and rebuilds one {version}.sacc with a # single FullCovariance (point-insertion order = block order). # @@ -476,6 +488,12 @@ def cv_assemble_inputs(version): pure_eb=cv_pure_eb_sacc(version), rho_tau=cv_rho_tau_sacc(version), ) + # The integration-grid ξ± is a fiducial-only product (the 10k-bin MPI run in + # xi_highres emits only {fiducial}_xi_integration.sacc), so it folds into the + # fiducial version's terminal file alone; other versions' {version}.sacc omit + # the integration rows rather than trigger a job with no output to bind. + if version == config["fiducial"]["version"]: + parts["xi_integration"] = cv_xi_integration_sacc(version) if CV.get("include_pseudo_cl", False): parts["pseudo_cl"] = cv_pseudo_cl_analysis_sacc(version) parts["pseudo_cl_cov"] = cv_pseudo_cl_cov(version) @@ -490,6 +508,11 @@ rule assemble_sacc: sacc=cv_analysis_sacc("{version}"), params: version="{version}", + # Run type (data|mock) gates unblinded loading in assemble_sacc.py: a + # 'data' run fails closed on unblinded parts, a 'mock' run loads freely. + # Production runs on real catalogues, so the default is 'data'. PR #253's + # blind-at-birth conceals each data part, letting the 'data' run assemble. + type=CV.get("type", "data"), # Statistics this rule wired (same toggles as cv_assemble_inputs). The # script validates part_paths against this so a typo'd input keyword # can't silently drop a statistic from the terminal file. diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 8202ed66..7955b17b 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -33,9 +33,11 @@ rule xi: rule xi_highres: """High-resolution xi for COSEBIS integration. - Terminal born-as-SACC product: {version}_xi_integration.sacc (a DiagonalCovariance - from TreeCorr varxip/varxim). COSEBIs and pure-E/B consume it. The raw .txt - dump is kept as a convergence byproduct. + Born-as-SACC part: {version}_xi_integration.sacc (a DiagonalCovariance from + TreeCorr varxip/varxim). COSEBIs and pure-E/B consume it, and rule + assemble_sacc folds its integration-grid ξ± rows (grid='integration') into + the single terminal {version}.sacc. The raw .txt dump is kept as a + convergence byproduct. """ container: None output: diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index fb61a7c4..798bd48e 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -6,7 +6,8 @@ Each per-statistic ``*.sacc`` *part* (written born-as-SACC by the mixins and the run_2pcf / generate_pseudo_cl scripts) holds one statistic. The assembler loads -them in canonical order — ξ± reporting, pseudo-Cℓ, COSEBIs, pure-E/B, ρ/τ — and +them in canonical order — ξ± reporting, ξ± integration, pseudo-Cℓ, COSEBIs, +pure-E/B, ρ/τ — and calls :func:`sacc_writers.assemble_analysis_sacc`, which rebuilds one Sacc with a single block-diagonal ``FullCovariance`` (point-insertion order = block order, validated by ``sacc_io.assemble_covariance``). @@ -14,9 +15,10 @@ Covariance sourcing (the part-by-part decision) ----------------------------------------------- ``assemble_analysis_sacc`` REQUIRES every part to carry its own covariance block. -The COSEBIs, pure-E/B and ρ/τ parts already do (their writers attach it). The -ξ± reporting and pseudo-Cℓ parts are born cov-less by design; this script injects -their blocks before assembly: +The ξ± integration, COSEBIs, pure-E/B and ρ/τ parts already do (their writers +attach it — ξ± integration a ``DiagonalCovariance`` from TreeCorr varxip/varxim). +The ξ± reporting and pseudo-Cℓ parts are born cov-less by design; this script +injects their blocks before assembly: * **ξ± reporting** — the CosmoCov theory covariance ``.txt`` (``--xi-cov``). For the single-bin round it is already ``[ξ+; ξ−]``-ordered (CosmoCov / covdat_to_fits: @@ -50,8 +52,17 @@ _CL_ORDER = ("EE", "BB", "EB") # Canonical part order — the order assemble_analysis_sacc inserts points in, which -# must match the covariance block order. Missing parts are simply skipped. -CANONICAL = ("xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau") +# must match the covariance block order. Missing parts are simply skipped. The +# integration-grid ξ± part (grid='integration', its own DiagonalCovariance) sits +# next to the reporting ξ±: both are ξ± sections distinguished only by grid tag. +CANONICAL = ( + "xi_reporting", + "xi_integration", + "pseudo_cl", + "cosebis", + "pure_eb", + "rho_tau", +) def _pseudo_cl_cov_block(cov_fits, hdu): @@ -78,8 +89,8 @@ def _pseudo_cl_cov_block(cov_fits, hdu): def _attach_cov(part, name, xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var): """Ensure ``part`` carries a covariance, injecting the xi/pseudo-Cℓ block. - ``part`` is mutated in place. cosebis/pure_eb/rho_tau parts already carry - their covariance and pass straight through. Raises loudly if a required xi / + ``part`` is mutated in place. xi_integration/cosebis/pure_eb/rho_tau parts + already carry their covariance and pass straight through. Raises loudly if a required xi / pseudo-Cℓ block is missing and no placeholder was requested. """ if part.covariance is not None: @@ -112,6 +123,7 @@ def assemble_sacc( pseudo_cl_cov=None, pseudo_cl_cov_hdu="COVAR_FULL", placeholder_var=None, + allow_unblinded=False, ): """Assemble ``{version}.sacc`` from the per-statistic ``part_paths`` mapping. @@ -132,6 +144,10 @@ def assemble_sacc( rejected too (catches a typo in the expected list itself). xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var Covariance sourcing — see the module docstring. + allow_unblinded : bool, optional + Passed to :func:`sacc_io.load` for every part. Default ``False`` fails + closed on unblinded real data; the caller sets it ``True`` only for mock + runs. See the load loop for the PR #253 blind-at-birth seam. """ if expected is not None: unknown = [name for name in expected if name not in CANONICAL] @@ -153,10 +169,12 @@ def assemble_sacc( path = part_paths.get(name) if path is None: continue - # Assembly runs pre-blind on unblinded real-data parts (Smokescreen - # conceals the assembled analysis file downstream), so the fail-closed - # load is told this is a legitimate pre-blind consumer. - part = sacc_io.load(path, allow_unblinded=True) + # Fail closed on real data by default: a data-type part loads only when + # it already carries the concealed=True blinding stamp. allow_unblinded + # is set True only for mock runs (see the caller). This is the seam for + # PR #253's blind-at-birth: once each part is concealed at write time, a + # data run assembles with allow_unblinded=False untouched. + part = sacc_io.load(path, allow_unblinded=allow_unblinded) if nz is None: # The nz tracers + metadata are identical across parts (same version); # take them from the first loaded part for the assembled file. @@ -198,6 +216,9 @@ def _from_snakemake(smk): # typo in an input keyword drops the part from part_paths above, so validate # against this expected list rather than trusting the hasattr filter. expected = list(p["expected"]) + # Fail closed on real data: only a mock run may read unblinded parts. The + # run type comes from config (default 'data' — the production catalogues). + run_type = p.get("type", "data") assemble_sacc( version=p["version"], part_paths=part_paths, @@ -207,6 +228,7 @@ def _from_snakemake(smk): pseudo_cl_cov=getattr(inp, "pseudo_cl_cov", None), pseudo_cl_cov_hdu=p.get("pseudo_cl_cov_hdu", "COVAR_FULL"), placeholder_var=p.get("placeholder_var", None), + allow_unblinded=(run_type == "mock"), ) @@ -216,6 +238,13 @@ def _from_cli(argv=None): ) ap.add_argument("--version", required=True, help="Catalogue version") ap.add_argument("--out", required=True, help="Output {version}.sacc path") + ap.add_argument( + "--type", + choices=("data", "mock"), + default="data", + help="Run type. 'mock' reads parts freely; 'data' fails closed on " + "unblinded parts (only concealed/blinded parts load).", + ) for name in CANONICAL: ap.add_argument( f"--{name.replace('_', '-')}", default=None, help=f"{name} part" @@ -248,6 +277,7 @@ def _from_cli(argv=None): pseudo_cl_cov=a.pseudo_cl_cov, pseudo_cl_cov_hdu=a.pseudo_cl_cov_hdu, placeholder_var=a.allow_placeholder, + allow_unblinded=(a.type == "mock"), ) diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py index 689187ea..d76ae7b6 100644 --- a/workflow/scripts/run_2pcf_highres.py +++ b/workflow/scripts/run_2pcf_highres.py @@ -204,10 +204,12 @@ def compute_patch_centers(ra, dec): def write_xi_integration_sacc(gg): - """Write the terminal integration-grid ξ± SACC part (``{version}_xi_integration.sacc``). + """Write the integration-grid ξ± SACC part (``{version}_xi_integration.sacc``). - This is a terminal product in its own right — COSEBIs and pure-E/B consume - it. It carries a ``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim`` + This is a per-statistic part — COSEBIs and pure-E/B consume it, and + ``rule assemble_sacc`` folds its integration-grid ξ± rows into the single + terminal ``{version}.sacc``. It carries a ``DiagonalCovariance`` from TreeCorr + ``varxip``/``varxim`` (npatch=1 leaves shot-noise variance as the only covariance estimate). Both run paths land here: in-container this uses the full SACC stack; on the bare-host MPI run only ``sacc_io`` + the n(z) file are needed (no healpy). From 1dfff786d061084ae6e0145f987a6dfd4722d77d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 14:38:52 +0200 Subject: [PATCH 020/160] docs(workflow): mark glass-mock A/B/C 'blind' as distinct from Smokescreen The pseudo_cl / pseudo_cl_cov rules and the shared BLINDS list use a 'blind' wildcard that is the glass-mock multi-catalogue A/B/C variant, not Smokescreen blinding. Add prominent comments at the BLINDS definition, the wildcard constraint, and both rules so the two axes are not confused. A full rename is avoided: 'blind' is baked into on-disk filenames we do not own (external nz_{version}_{A|B|C}.txt) and into the covariance / inference path builders. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD --- workflow/common.py | 7 +++++++ workflow/rules/twopoint.smk | 17 +++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/workflow/common.py b/workflow/common.py index 17e60ce7..1a168041 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -28,6 +28,12 @@ ) ) CAT_CONFIG = "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" +# NB: "blind" here is the glass-mock multi-catalogue A/B/C variant convention +# (three mock realisations), NOT Smokescreen blinding. The name predates the +# blind-at-birth work and is kept because it is baked into on-disk filenames we +# do not own (e.g. sguerrini's nz_{version}_{A|B|C}.txt) and into the covariance +# / inference path builders below. Smokescreen concealment is a separate axis +# (the concealed=True SACC stamp), tracked by issues #241/#247. BLINDS = ["A", "B", "C"] BLOCK_PAIRS = [("++", "1"), ("--", "2"), ("+-", "3")] @@ -42,6 +48,7 @@ # silent failures. Apply with: wildcard_constraints: **WILDCARD_CONSTRAINTS WILDCARD_CONSTRAINTS = { "version": r"SP_v[\d.]+(_w_iv)?(_ecut\d+)?(_leak_corr)?", + # glass-mock A/B/C variant, not Smokescreen blinding — see BLINDS above. "blind": r"[ABC]", "nbins": r"\d+", "min_sep": r"[0-9.]+", diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 7955b17b..9eb90c64 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -109,11 +109,16 @@ wildcard_constraints: rule pseudo_cl: - """Generate pseudo-Cl data vector (born as SACC) with configurable binning.""" + """Generate pseudo-Cl data vector (born as SACC) with configurable binning. + + NB: the ``blind`` wildcard is the glass-mock A/B/C variant (three mock + catalogues), NOT Smokescreen blinding — see common.py BLINDS. The Smokescreen + concealed=True stamp is a separate axis on the SACC file. + """ output: pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.sacc"), wildcard_constraints: - blind="[ABC]", + blind="[ABC]", # glass-mock variant, not Smokescreen blinding params: version="{version}", blind="{blind}", @@ -133,11 +138,15 @@ rule pseudo_cl: rule pseudo_cl_cov: - """Generate pseudo-Cl covariance with configurable binning.""" + """Generate pseudo-Cl covariance with configurable binning. + + NB: ``blind`` is the glass-mock A/B/C variant, not Smokescreen blinding + (see common.py BLINDS). + """ output: pseudo_cl_cov=str(COSMO_VAL / "pseudo_cl_cov_{version}_blind={blind}_{binning}_nbins={nbins}.fits"), wildcard_constraints: - blind="[ABC]", + blind="[ABC]", # glass-mock variant, not Smokescreen blinding params: version="{version}", blind="{blind}", From 19f8b1ffe2adc57bfe4426e605f83ae744564ebb Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 14:51:20 +0200 Subject: [PATCH 021/160] fix(sacc): reconcile writers with the merged sacc_io API Resolve two silent semantic conflicts from the base merge (no textual conflict, but the tests broke). - add_cosebis reordered its params to (s, bins, En, scale_cut, Bn=None). cosebis_to_sacc still passed the old positional order; call it by keyword. - assemble_covariance now builds a BlockDiagonalCovariance (one FITS table per block, validated ordering), not a dense FullCovariance. Update the assembler docstrings, the smk comments, and the assembled-file test assertions to the block-diagonal type. The assembled .dense is unchanged (cross-blocks are zero either way); single-part writer covariances stay FullCovariance. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD --- src/sp_validation/cosmo_val/sacc_writers.py | 15 ++++++++------- src/sp_validation/tests/test_assemble_sacc.py | 6 +++--- src/sp_validation/tests/test_sacc_writers.py | 10 +++++----- workflow/rules/cosmo_val.smk | 4 ++-- workflow/scripts/assemble_sacc.py | 4 ++-- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index 10d1137f..b59db0c2 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -6,9 +6,10 @@ statistic into a single-statistic SACC — a *part* — carrying that statistic's own covariance as its one covariance block. The Snakemake DAG writes one part per rule; :func:`assemble_analysis_sacc` then loads the parts and rebuilds the -single ``{version}.sacc`` analysis file with a ``FullCovariance`` assembled -block-diagonally in canonical order (per the SACC layout contract — *not* -``sacc.concatenate_data_sets``, whose ``BlockDiagonalCovariance`` output the +single ``{version}.sacc`` analysis file with a ``BlockDiagonalCovariance`` +assembled from the per-part blocks in canonical order (per the SACC layout +contract, via the validated :func:`sp_validation.sacc_io.assemble_covariance` — +*not* ``sacc.concatenate_data_sets``, whose unvalidated block-diagonal the contract rules out). The integration-grid ``{version}_xi_integration.sacc`` is one more part @@ -103,11 +104,11 @@ def cosebis_to_sacc(nz, metadata, result, scale_cut): ``b_modes.calculate_cosebis`` — ``{"En", "Bn", "cov", ...}`` — where ``cov`` is the ``[En; Bn]``-ordered COSEBIs covariance. Non-fiducial scale cuts are a diagnostic (the PTE scan) and stay in the sidecar ``.npz``; only the - fiducial cut is a data product, because a ``FullCovariance`` must cover + fiducial cut is a data product, because the analysis covariance must cover every stored point and the cuts overlap in mode space. """ s = sio.new_sacc(nz, metadata) - sio.add_cosebis(s, BIN, result["En"], result["Bn"], scale_cut) + sio.add_cosebis(s, BIN, result["En"], scale_cut, Bn=result["Bn"]) s.add_covariance(np.asarray(result["cov"])) return s @@ -224,7 +225,7 @@ def assemble_analysis_sacc(nz, metadata, parts): part's data points into one Sacc in the order the parts are given — which must be the canonical order (ξ± reporting, ξ± integration, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ) - — and assembles a single ``FullCovariance`` from the per-part covariance + — and assembles a single ``BlockDiagonalCovariance`` from the per-part covariance blocks. Point insertion order and block order therefore agree by construction, which ``sacc_io.assemble_covariance`` validates (contiguous, tiling, square) and raises on if they don't. @@ -238,7 +239,7 @@ def assemble_analysis_sacc(nz, metadata, parts): Returns ------- sacc.Sacc - The analysis Sacc with a ``FullCovariance`` covering every point. + The analysis Sacc with a ``BlockDiagonalCovariance`` covering every point. """ s = sio.new_sacc(nz, metadata) blocks = [] diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py index 26ff5f15..323357f5 100644 --- a/src/sp_validation/tests/test_assemble_sacc.py +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -143,7 +143,7 @@ def test_assemble_sacc_placeholder_canonical_order(tmp_path): out = tmp_path / "vSYNTH.sacc" s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) assert out.exists() - assert type(s.covariance).__name__ == "FullCovariance" + assert type(s.covariance).__name__ == "BlockDiagonalCovariance" assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) # Canonical insertion order: the first data types are ξ+ then ξ−. @@ -236,9 +236,9 @@ def test_assemble_sacc_respects_pseudo_cl_toggle(tmp_path): s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) tr = ("source_0", "source_0") assert len(s.indices(sio.CL_EE, tr)) == 0 - # Round-trips as a valid FullCovariance over the remaining points. + # Round-trips as a valid BlockDiagonalCovariance over the remaining points. s2 = sio.load(str(out)) - assert type(s2.covariance).__name__ == "FullCovariance" + assert type(s2.covariance).__name__ == "BlockDiagonalCovariance" assert s2.covariance.dense.shape == (len(s2.mean), len(s2.mean)) diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index 26072c11..e3a96a5d 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -3,7 +3,7 @@ Synthetic and fast: each ``*_to_sacc`` writer is exercised with in-memory arrays, round-tripped through ``tmp_path``, and checked against the SACC layout contract (data types, tags, ordering, covariance alignment). The analysis-file -assembler is verified to produce a single ``FullCovariance`` covering every +assembler is verified to produce a single ``BlockDiagonalCovariance`` covering every point with each per-statistic block correctly placed. One real small-nside NaMaster round-trip proves the pseudo-Cℓ window survives the writer path. """ @@ -259,11 +259,11 @@ def get_bandpower_windows(self): return [xi, cl, co] -def test_assemble_analysis_sacc_full_covariance(tmp_path): +def test_assemble_analysis_sacc_block_diagonal_covariance(tmp_path): nz = {0: _nz()} parts = _make_parts(nz) s = sw.assemble_analysis_sacc(nz, META, parts) - assert type(s.covariance).__name__ == "FullCovariance" + assert type(s.covariance).__name__ == "BlockDiagonalCovariance" assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) # every point covered; blocks placed and cross-blocks zero tr = ("source_0", "source_0") @@ -284,7 +284,7 @@ def test_assemble_analysis_sacc_full_covariance(tmp_path): ) # round-trips s2 = _roundtrip(s, tmp_path, "analysis") - assert type(s2.covariance).__name__ == "FullCovariance" + assert type(s2.covariance).__name__ == "BlockDiagonalCovariance" assert np.allclose(s2.covariance.dense, s.covariance.dense) @@ -314,5 +314,5 @@ def test_assemble_from_reloaded_parts(tmp_path): sio.save(part, str(tmp_path / f"part{i}.sacc"), type="mock") reloaded.append(sio.load(str(tmp_path / f"part{i}.sacc"))) s = sw.assemble_analysis_sacc(nz, META, reloaded) - assert type(s.covariance).__name__ == "FullCovariance" + assert type(s.covariance).__name__ == "BlockDiagonalCovariance" assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 126517d4..73ec22c7 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -458,7 +458,7 @@ rule cv_summarize_bmodes: # integration-grid ξ± is fiducial-only, so it joins the fiducial version's file # alone (see cv_assemble_inputs). assemble_sacc.py # loads the parts in canonical order and rebuilds one {version}.sacc with a -# single FullCovariance (point-insertion order = block order). +# single BlockDiagonalCovariance (point-insertion order = block order). # # The pseudo-Cℓ part is the TAGGED, blinded inference product (blind=A, powspace, # nbins=32) — the same pseudo-Cℓ today's cosmosis_fitting.py consumes — so the @@ -470,7 +470,7 @@ rule cv_summarize_bmodes: # mask-keyed and lives deep in the inference tree, so wiring it couples cosmo_val # to the whole inference covariance DAG — that sourcing is PR-3's converter # territory. Until then a documented diagonal placeholder keeps the ξ block (and -# so the FullCovariance) structurally valid; it is a flagged stand-in, never a +# so the BlockDiagonalCovariance) structurally valid; it is a flagged stand-in, never a # science covariance, and plugs out via --xi-cov the moment PR 3 lands. diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 798bd48e..4eec725d 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -9,7 +9,7 @@ them in canonical order — ξ± reporting, ξ± integration, pseudo-Cℓ, COSEBIs, pure-E/B, ρ/τ — and calls :func:`sacc_writers.assemble_analysis_sacc`, which rebuilds one Sacc with a -single block-diagonal ``FullCovariance`` (point-insertion order = block order, +single ``BlockDiagonalCovariance`` (point-insertion order = block order, validated by ``sacc_io.assemble_covariance``). Covariance sourcing (the part-by-part decision) @@ -35,7 +35,7 @@ When a cov input is absent the assembly cannot proceed on a real product; pass ``--allow-placeholder`` to attach a documented diagonal placeholder (``placeholder_var`` on every point of the cov-less parts) so the DAG dry-run and -the fast test can still produce a structurally-valid ``FullCovariance``. The +the fast test can still produce a structurally-valid ``BlockDiagonalCovariance``. The placeholder is a flagged stand-in, never a science covariance. """ From f5eb4177963c973c9f97d8b95583e64ed575d613 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 15:12:01 +0200 Subject: [PATCH 022/160] chore(deps): restore develop's uv.lock SSOT; drop stale firecrown-era override machinery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The branch had accidentally superseded #266's reproducible-deps model (uv.lock as source of truth, `uv sync --frozen` in the Dockerfile) with an older firecrown/Smokescreen-blinding scheme: uv.lock deleted, dependency resolution done via `uv pip install --overrides uv-overrides.txt`, and a patch script to make pip-installed firecrown importable without NumCosmo. Firecrown has been dropped from the project, so all of that goes. Restores develop's pyproject.toml, uv.lock, Dockerfile, and the deploy-image blinding-stack smoke test wholesale; removes uv-overrides.txt and scripts/patch_firecrown.py. Keeps one genuine PR4-driven change: tightens the sacc constraint to >=2.4,<3, since sacc_io.py (this branch) uses concatenate_data_sets/BlockDiagonalCovariance, both from sacc's 2.x rewrite (develop's lock already resolves sacc to 2.4; this just makes the pyproject floor honest). Drops the unused numexpr addition — no code in the tree imports it. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD --- .github/workflows/deploy-image.yml | 6 - Dockerfile | 57 +- pyproject.toml | 97 +- scripts/patch_firecrown.py | 225 -- uv-overrides.txt | 21 - uv.lock | 4451 ++++++++++++++++++++++++++++ 6 files changed, 4538 insertions(+), 319 deletions(-) delete mode 100644 scripts/patch_firecrown.py delete mode 100644 uv-overrides.txt create mode 100644 uv.lock diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 13f0fafe..192e87a6 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -44,12 +44,6 @@ jobs: - name: Import smoke test run: docker run --rm ${{ steps.meta.outputs.tags }} python -c "import sp_validation" - # The fast suite doesn't import the blinding stack, so a broken - # firecrown/smokescreen install would otherwise ship green. Prove the - # image can actually load it (sacc + patched firecrown + smokescreen). - - name: Blinding-stack import smoke test - run: docker run --rm ${{ steps.meta.outputs.tags }} python -c "import sacc; import firecrown.likelihood; import smokescreen" - # Run the fast test suite against the freshly-built image *before* # pushing, so a failing suite blocks publication. The image carries the # full stack and the test files (COPY . + editable install), so this diff --git a/Dockerfile b/Dockerfile index 818db67d..ccc930bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Development image with more bells and whistles -FROM ghcr.io/cosmostat/shapepipe:develop +FROM ghcr.io/cosmostat/shapepipe:im_sims RUN apt-get update -y --quiet --fix-missing && \ apt-get dist-upgrade -y --quiet --fix-missing && \ @@ -12,41 +12,28 @@ RUN apt-get update -y --quiet --fix-missing && \ npm \ tmux -# The base image installs into the system interpreter (/usr/local); use `uv pip` -# so the heavy scientific stack and our deps land where `python` resolves. -RUN uv pip install --no-cache-dir \ - snakemake - -# The base shapepipe image ships cs_util 0.1.9, and `uv pip install -e` does NOT -# upgrade an already-satisfied dependency to meet a *new* lower bound (astral-sh/uv -# #8410). sp_validation now needs `cs_util.size` (cs_util>=0.2.1), so upgrade it -# explicitly here — otherwise the editable install silently keeps 0.1.9 and the -# galaxy import smoke test fails. shear_psf_leakage@develop allows cs-util<0.3, -# so 0.2.x satisfies the whole graph. -RUN uv pip install --no-cache-dir --upgrade 'cs_util>=0.2.1' +# The base shapepipe image provides a uv-managed venv at /app/.venv (exported as +# VIRTUAL_ENV); install sp_validation's deps into that same venv rather than +# spawning a second one under /sp_validation. +ENV UV_PROJECT_ENVIRONMENT=/app/.venv WORKDIR /sp_validation -COPY . /sp_validation -# Install with the test + glass + blinding extras so the image can run the unit -# suite in CI, the GLASS map-level mock test, *and* the SACC/Smokescreen blinding -# stack. `glass` (Generator for Large Scale Structure) ships `glass.ext.camb`; -# `cosmology` provides the `Cosmology` wrapper (`Cosmology.from_camb`) GLASS -# consumes. The `[blinding]` extra (firecrown + smokescreen) needs the override -# file: firecrown declares conda-forge-only / unused sampler connectors as hard -# deps — see uv-overrides.txt for the full story. -RUN uv pip install --no-cache-dir --overrides uv-overrides.txt -e '.[test,glass,blinding]' +# uv.lock is the SSOT: `uv sync --frozen` installs exactly what it pins, so an +# image build can never silently re-resolve and drift a base-image version (the +# numpy-past-numba drift this lockfile exists to prevent). `--inexact` keeps the +# base image's ShapePipe stack (shapepipe, ngmix, galsim, …) — packages not in +# our lock — instead of pruning them. Copy the lock + manifest first so this +# layer caches independently of source edits. Extras: test (CI unit suite), +# glass (GLASS map-level mock — pulls glass.ext.camb + the cosmology wrapper), +# workflow (Snakemake + mpi4py runners). cs_util 0.2.2 (with cs_util.size) and a +# numba-safe numpy 2.4.6 come straight from the lock, so the old ad-hoc snakemake +# and cs_util `--upgrade` layers are gone. +COPY pyproject.toml uv.lock /sp_validation/ +RUN uv sync --frozen --inexact --no-install-project \ + --extra test --extra glass --extra workflow -# Same uv gotcha as the cs_util upgrade above (astral-sh/uv #8410): if the base -# image already carries a numpy that violates the [blinding] extra's new -# `numpy<2.5` cap (firecrown 1.15.1 breaks on numpy 2.5 at import), the -# editable install won't move it. Request the bound explicitly so the image is -# deterministic either way; numpy 2.4.x is ABI-compatible with the compiled -# stack (verified: pyccl/camb/treecorr/healpy/pymaster + fast suite). -RUN uv pip install --no-cache-dir 'numpy>=2.2,<2.5' - -# firecrown is distributed for conda-forge (where NumCosmo always exists) and -# hits NumCosmo at import time in a pip env, on paths unrelated to our use. -# This patches the installed tree (surgical, pinned-version-checked, loud on -# mismatch) and verifies `import firecrown.likelihood; import smokescreen`. -RUN python scripts/patch_firecrown.py +# Install sp_validation itself (editable) into the same venv; deps are already +# satisfied by the sync above. +COPY . /sp_validation +RUN uv pip install --no-deps -e . diff --git a/pyproject.toml b/pyproject.toml index ecdec6f1..db04b681 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,9 +9,9 @@ authors = [ ] license = {text = "MIT"} readme = "README.md" -# 3.12 floor set by Smokescreen 1.5.6 (and firecrown v1.15); the container base -# (shapepipe:develop) is already python:3.12-slim-bookworm, so this aligns -# pyproject with the actual runtime. +# 3.12 floor: cosmo-numba (a hard dependency below) requires >=3.12, and the +# production container is Python 3.12 (shapepipe base image). Keeping the floor +# in sync with the container is what makes `uv lock` resolvable. requires-python = ">=3.12" classifiers = [ "License :: OSI Approved :: MIT License", @@ -28,13 +28,38 @@ dependencies = [ "camb>=1.6", "clmm", "colorama", + # Blinding closure (core, no extra): cryptography + sacc here, plus the + # Smokescreen fork pin below; pyccl (the theory backend) is already core. + # cryptography and sacc are declared explicitly so the core runtime + # closure is self-documenting and independent of fork-metadata drift. + "cryptography", # Track cs_util's develop branch directly (git dependency) rather than a # PyPI pin: the two repos are iterating together heavily and cs_util # releases are infrequent. This PR's cosmology repoint needs get_cosmo / # get_theo_c_ell / get_theo_xi / PLANCK18 in cs_util.cosmo, which land via # CosmoStat/cs_util#76 — so this goes green once #76 merges into develop. "cs_util @ git+https://github.com/CosmoStat/cs_util.git@develop", + # Fast numba B-mode kernels (Schneider et al. 2022): the Schneider E/B split + # and COSEBIS live here, imported in b_modes.py. Tracks aguinot/cosmo-numba + # main (not published on PyPI). main carries the numpy-2 FFT fix via its + # rocket-fft dependency (which teaches numba's nopython mode to handle + # np.fft), and declares numba/numpy/rocket-fft from its requirements.txt so + # those constraints reach the resolver. + "cosmo-numba @ git+https://github.com/aguinot/cosmo-numba.git@main", "emcee", + # numba is the load-bearing pin of this whole environment: its numpy ceiling + # (numba 0.66 -> numpy<2.5) is what keeps the resolver from drifting numpy + # forward and breaking numba/ngmix — the failure this lockfile exists to + # prevent. cosmo-numba above also carries this constraint, but we pin numba + # ourselves too: it makes the one critical constraint visible and resilient + # to cosmo-numba's dependency metadata (which has silently emptied out + # between refs before). We pin numba, never numpy directly. + "numba", + # Imported directly across src/ (calibration, plots) alongside seaborn — + # declared explicitly rather than leaned on as a seaborn transitive. + "matplotlib", + "pandas", + "pyyaml", # SHA-pinned snapshot of getdist branch `upper_triangle_whisker`. "getdist @ git+https://github.com/benabed/getdist.git@113cd22a9a0d013b6f72fe734be81f260f3d3be5", "h5py", @@ -47,7 +72,6 @@ dependencies = [ "jupytext>=1.15", "lenspack", "lmfit", - "numexpr", "numpy>=2.0", "opencv-python-headless", "pyccl", @@ -55,9 +79,9 @@ dependencies = [ "pymaster", "regions", "reproject", - # SACC (LSST DESC's data-vector container) is the standard format for all - # data products from the tomographic round on (PRD #241); sp_validation.sacc_io - # is core library code, so sacc is a core dependency. + # sp_validation.sacc_io (assembled on this branch) uses concatenate_data_sets + # and BlockDiagonalCovariance, both from sacc's 2.x rewrite; the lock already + # resolves to 2.4, this just makes the floor honest. "sacc>=2.4,<3", # scipy 1.18 ported FITPACK from Fortran to C, changing the return shape of # RectBivariateSpline(scalar, scalar, grid=False) from 0-d `array(x)` to @@ -76,6 +100,12 @@ dependencies = [ # getdist feature-branch below, which is an external fork we pin for repro.) "shear_psf_leakage @ git+https://github.com/CosmoStat/shear_psf_leakage.git@develop", "skyproj", + # UNIONS-WL fork of DESC Smokescreen, pinned by SHA on the fork's + # packaging branch: it declares pyccl and imports its theory backends + # lazily, so the install closure is CCL-only. Provisional pin — swapped to the + # fork's release tag once the fork packaging PRs merge. Git pin only; + # nothing is published to PyPI. + "smokescreen @ git+https://github.com/UNIONS-WL/Smokescreen@588a6b9b26560bd5ba3dd5ba342f3c40152644f9", "statsmodels", "treecorr>=5.0", "tqdm", @@ -85,6 +115,13 @@ dependencies = [ [project.urls] Homepage = "https://github.com/CosmoStat/sp_validation" +[tool.uv] +# The reproducibility target is the Linux container; scope the lock to Linux +# (mirrors shapepipe) so `uv lock` resolves the linux-centric stack (pymaster, +# mpi4py) without hunting for macOS/Windows wheels. macOS dev installs still +# work via `uv pip install -e .` (unlocked), just not `uv sync` from the lock. +environments = ["sys_platform == 'linux'"] + [project.optional-dependencies] test = [ "pytest", @@ -112,31 +149,20 @@ glass = [ "glass==2025.1", "glass.ext.camb==2023.6", "cosmology==2022.10.9", + # fitsio: make_unions_glass_sim.py writes the mock catalogue as FITS. + "fitsio", ] -# Data-vector blinding (PRD #241 §3-§5): Smokescreen applies the Muir et al. -# shift d → d + t(hidden) − t(fid), with firecrown + CCL as the theory engine -# (only compute_theory_vector is used; sampling stays with CosmoSIS). The blind -# must be exactly recomputable from the seed at unblinding time, so the whole -# theory stack is pinned exactly, as a set. Smokescreen 1.5.6 + firecrown v1.15 -# both set the python floor (>=3.12). -# -# firecrown is not on PyPI and declares conda-forge-only / unused sampler -# connectors as hard deps, so installing this extra requires the dependency -# override file: `uv pip install --overrides uv-overrides.txt -e '.[blinding]'` -# (see uv-overrides.txt; the Dockerfile does this for the container). -# After installing this extra, run `python scripts/patch_firecrown.py` — it -# makes pip-installed firecrown importable without NumCosmo (conda-forge-only); -# see that script's docstring for the full story. -blinding = [ - "firecrown @ git+https://github.com/LSSTDESC/firecrown.git@v1.15.1", - "smokescreen==1.5.6", - "pyccl==3.3.4", - # firecrown 1.15.1 subclasses npt.NDArray (DataVector); numpy 2.5 turned - # npt.NDArray into a non-subclassable typing alias, breaking firecrown at - # import. firecrown's own env caps numpy<2.4; 2.4.3 is verified against - # the full compiled stack (pyccl/camb/treecorr/healpy/pymaster) + the - # sp_validation fast suite. - "numpy>=2.2,<2.5", +# Cosmo-inference workflow runners (workflow/scripts/*). Kept optional: the core +# library resolves without them, but the container installs this extra so the +# Snakemake workflow and cross-validation runners are available. +workflow = [ + "snakemake", + # run_2pcf_highres.py drives the MPI convergence run; the container ships + # OpenMPI (/opt/ompi) so mpi4py builds against it. + "mpi4py", + # NOTE: workflow/scripts/cv_*.py also import `cv_runner`, which is not + # published or resolvable (no public repo found) — left undeclared pending + # its source. Same for `unions_wl` (scripts/check_footprint.py). ] develop = ["sp_validation[test,docs]"] @@ -147,7 +173,14 @@ addopts = [ "--cov=sp_validation", "--cov-report=term", "--cov-report=xml", - "--junitxml=pytest.xml" + "--junitxml=pytest.xml", + # The base ShapePipe image's test extra ships old pytest-pydocstyle / + # pytest-pycodestyle, whose pytest_collect_file hooks use the `path` arg the + # newer pytest our lock installs has removed — they crash collection. + # sp_validation lints with ruff, not these, so don't load them. Harmless if + # absent (`-p no:` just skips an unregistered plugin). + "-p", "no:pydocstyle", + "-p", "no:pycodestyle" ] markers = [ "fast: marks tests as fast (deselect with '-m \"not fast\"')", diff --git a/scripts/patch_firecrown.py b/scripts/patch_firecrown.py deleted file mode 100644 index 7b528da1..00000000 --- a/scripts/patch_firecrown.py +++ /dev/null @@ -1,225 +0,0 @@ -"""Make pip-installed firecrown importable without NumCosmo. - -Run *inside* the target environment, after installing the ``[blinding]`` extra: - - python scripts/patch_firecrown.py - -Why this exists (PRD #241, PR 1): firecrown is the theory engine for -Smokescreen blinding — only ``compute_theory_vector`` on the SACC-read -cosmic-shear path is used. Upstream distributes firecrown via conda-forge, -where NumCosmo (a GObject-introspection C library, absent from PyPI) is always -present; in a pip/uv environment, firecrown 1.15.1 hits NumCosmo at *import -time* through two paths that have nothing to do with cosmic shear: - -1. ``firecrown/generators/__init__.py`` eagerly re-exports the LSST Y1/Y10 - predefined n(z) bin constants, defeating the lazy ``__getattr__`` that - ``_inferred_galaxy_zdist`` already provides — and computing those constants - imports NumCosmo. -2. ``firecrown/likelihood/__init__.py`` eagerly imports the cluster - likelihoods, which import ``crow`` (lsstdesc-crow), which subclasses a - NumCosmo C class at module load (``class CountsIntegralND(Ncm.IntegralND)``). - -This script (a) restores laziness in ``generators``, (b) makes the cluster -imports optional, and (c) installs a *loud* ``numcosmo_py`` shim so that any -genuine NumCosmo use raises immediately instead of being silently faked. -Everything is exact-string surgery against the pinned firecrown v1.15.1: if a -target string is missing (e.g. after a version bump), the script fails loudly -so the pin and the patch get reviewed together. Idempotent — safe to re-run. - -The right long-term fix is upstream (guarded/lazy imports in firecrown); until -then this file is the entire cost of staying pip-installable. -""" - -import importlib.metadata -import importlib.util -import subprocess -import sys -from pathlib import Path - -EXPECTED_FIRECROWN = "1.15.1" - -GENERATORS_OLD = """\ - # Lazy-loaded bins (via __getattr__) - Y1_LENS_BINS, - Y1_SOURCE_BINS, - Y10_LENS_BINS, - Y10_SOURCE_BINS, - LSST_Y1_LENS_HARMONIC_BIN_COLLECTION, - LSST_Y1_SOURCE_HARMONIC_BIN_COLLECTION, - LSST_Y10_LENS_HARMONIC_BIN_COLLECTION, - LSST_Y10_SOURCE_HARMONIC_BIN_COLLECTION, -) -""" - -GENERATORS_NEW = """\ -) - -# NOTE (sp_validation patch, scripts/patch_firecrown.py): the LSST Y1/Y10 -# predefined bin constants are computed lazily in _inferred_galaxy_zdist via a -# module-level __getattr__ that imports NumCosmo. Importing them EAGERLY here -# forced NumCosmo at `import firecrown.generators` (hence at -# `import firecrown.likelihood`), which pip cannot satisfy. Re-expose them -# lazily instead; the SACC-read cosmic-shear path never touches them. -_LAZY_BIN_NAMES = frozenset( - { - "Y1_LENS_BINS", - "Y1_SOURCE_BINS", - "Y10_LENS_BINS", - "Y10_SOURCE_BINS", - "LSST_Y1_LENS_HARMONIC_BIN_COLLECTION", - "LSST_Y1_SOURCE_HARMONIC_BIN_COLLECTION", - "LSST_Y10_LENS_HARMONIC_BIN_COLLECTION", - "LSST_Y10_SOURCE_HARMONIC_BIN_COLLECTION", - } -) - - -def __getattr__(name): - if name in _LAZY_BIN_NAMES: - from . import _inferred_galaxy_zdist as _z - - return getattr(_z, name) - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - -""" - -LIKELIHOOD_OLD = """\ -# Cluster statistics -from firecrown.likelihood._binned_cluster import BinnedCluster -from firecrown.likelihood._binned_cluster_number_counts import ( - BinnedClusterNumberCounts, -) -from firecrown.likelihood._binned_cluster_number_counts_shear import ( - BinnedClusterShearProfile, -) -""" - -LIKELIHOOD_NEW = """\ -# Cluster statistics. -# NOTE (sp_validation patch, scripts/patch_firecrown.py): the cluster -# likelihoods import `crow` (lsstdesc-crow), which subclasses NumCosmo C -# classes at module load. NumCosmo is conda-forge-only, so in a pip/uv env -# these imports fail. They are NOT on the cosmic-shear (TwoPoint/WeakLensing) -# path, so they become optional: without NumCosmo the cluster classes are -# unavailable but everything else loads. -try: - from firecrown.likelihood._binned_cluster import BinnedCluster - from firecrown.likelihood._binned_cluster_number_counts import ( - BinnedClusterNumberCounts, - ) - from firecrown.likelihood._binned_cluster_number_counts_shear import ( - BinnedClusterShearProfile, - ) -except (ImportError, RuntimeError, TypeError): # pragma: no cover - BinnedCluster = None # type: ignore[assignment,misc] - BinnedClusterNumberCounts = None # type: ignore[assignment,misc] - BinnedClusterShearProfile = None # type: ignore[assignment,misc] -""" - -SHIM = '''\ -"""Minimal loud shim for numcosmo_py (installed by sp_validation). - -NumCosmo is a GObject-introspection C library available only via conda-forge. -With the companion patches to firecrown (scripts/patch_firecrown.py), the -SACC-read cosmic-shear likelihood path never imports it; this shim provides -the import-time names so the patched package loads, and any genuine numerical -use of NumCosmo raises loudly rather than being silently faked. -""" - - -class _Missing: - def __init__(self, path="numcosmo_py"): - self._p = path - - def __getattr__(self, name): - return _Missing(f"{self._p}.{name}") - - def __call__(self, *a, **k): - raise RuntimeError( - f"{self._p} was called, but NumCosmo is not installed (conda-forge " - "only, not on PyPI). It is not needed for the SACC-read " - "cosmic-shear likelihood path." - ) - - def __getitem__(self, item): - return _Missing(f"{self._p}[...]") - - -Ncm = _Missing("numcosmo_py.Ncm") -Nc = _Missing("numcosmo_py.Nc") -GObject = _Missing("numcosmo_py.GObject") - - -def dict_to_var_dict(*a, **k): - raise RuntimeError("numcosmo_py.dict_to_var_dict unavailable (no NumCosmo)") - - -def var_dict_to_dict(*a, **k): - raise RuntimeError("numcosmo_py.var_dict_to_dict unavailable (no NumCosmo)") -''' - - -def patch_file(path: Path, old: str, new: str) -> str: - text = path.read_text() - if new in text: - return "already patched" - if old not in text: - sys.exit( - f"FATAL: expected text not found in {path}.\n" - "firecrown has probably been bumped past the pinned version this " - "patch targets — review scripts/patch_firecrown.py together with " - "the [blinding] pin in pyproject.toml." - ) - path.write_text(text.replace(old, new, 1)) - return "patched" - - -def main() -> None: - spec = importlib.util.find_spec("firecrown") - if spec is None or spec.origin is None: - sys.exit("FATAL: firecrown is not installed in this environment.") - pkg = Path(spec.origin).parent - - # Metadata, not `import firecrown` — pre-patch, importing is what's broken. - version = importlib.metadata.version("firecrown") - if version != EXPECTED_FIRECROWN: - sys.exit( - f"FATAL: firecrown {version} != expected {EXPECTED_FIRECROWN}; " - "review this patch against the new version before bumping " - "EXPECTED_FIRECROWN." - ) - - print( - "generators/__init__.py:", - patch_file(pkg / "generators" / "__init__.py", GENERATORS_OLD, GENERATORS_NEW), - ) - print( - "likelihood/__init__.py:", - patch_file(pkg / "likelihood" / "__init__.py", LIKELIHOOD_OLD, LIKELIHOOD_NEW), - ) - - # Loud numcosmo_py shim — only when no real NumCosmo is present. - if importlib.util.find_spec("numcosmo_py") is None: - shim_dir = pkg.parent / "numcosmo_py" - shim_dir.mkdir(exist_ok=True) - (shim_dir / "__init__.py").write_text(SHIM) - print("numcosmo_py shim: installed") - else: - print("numcosmo_py shim: skipped (numcosmo_py importable)") - - check = subprocess.run( - [ - sys.executable, - "-c", - "import firecrown.likelihood; import smokescreen", - ], - capture_output=True, - text=True, - ) - if check.returncode != 0: - sys.exit(f"FATAL: post-patch import check failed:\n{check.stderr}") - print("post-patch import check: firecrown.likelihood + smokescreen OK") - - -if __name__ == "__main__": - main() diff --git a/uv-overrides.txt b/uv-overrides.txt deleted file mode 100644 index 813ad055..00000000 --- a/uv-overrides.txt +++ /dev/null @@ -1,21 +0,0 @@ -# uv dependency overrides — pass via `--overrides uv-overrides.txt` (or -# UV_OVERRIDE=uv-overrides.txt) to every `uv pip install` against this project. -# -# Why this file exists: firecrown declares its sampler *connectors* as hard -# dependencies, but we use firecrown only as the theory engine for Smokescreen -# blinding (`compute_theory_vector`); sampling stays with CosmoSIS in -# cosmo_inference. Of the three connector deps: -# -# - numcosmo-py exists only on conda-forge, so pip/uv resolution of firecrown -# is *impossible* without an override; -# - cosmosis ships sdist-only (full Fortran/C build with gsl/cfitsio) — a -# heavy, fragile compile in every CI image build, for a connector we never -# import; -# - cobaya is wheel-clean but equally unused. -# -# Each line below replaces the package's requirement (wherever it appears in -# the graph) with one gated on an always-false marker, dropping it from -# resolution. firecrown's likelihood/CCL core imports none of them. -numcosmo-py; python_version < "3" -cosmosis; python_version < "3" -cobaya; python_version < "3" diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..bef8072e --- /dev/null +++ b/uv.lock @@ -0,0 +1,4451 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version < '3.14' and sys_platform == 'linux'", +] +supported-markers = [ + "sys_platform == 'linux'", +] + +[[package]] +name = "adjusttext" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/5c/496e506ad3313664df24a79f801719cadcd62af5999fcb299c9b08ff0d4b/adjusttext-1.4.0.tar.gz", hash = "sha256:1f73860ced8cccce3f85ee6989ca133c2579b67a7453f63dbeb38f39bf123154", size = 15852, upload-time = "2026-06-08T16:48:32.726Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/2c/897bdd17b05724c894a5b831c6b2e9853adcc2d07a70d6246c0cd5cd3912/adjusttext-1.4.0-py3-none-any.whl", hash = "sha256:6febd6484c0d45c39a22f44b2c1f4a8cd01ef58fada565cab4b629c771df79b5", size = 13262, upload-time = "2026-06-08T16:48:31.765Z" }, +] + +[[package]] +name = "aenum" +version = "3.1.17" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/07/e9/8b283567c1fef7c24d1f390b37daede8b61593d8cdaffb8e95d571699e83/aenum-3.1.17.tar.gz", hash = "sha256:a969a4516b194895de72c875ece355f17c0d272146f7fda346ef74f93cf4d5ba", size = 137648, upload-time = "2026-03-20T20:43:29.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/8d/1fe30c6fd8999b9d462547c4a1bb6690bda24af38f2913c4bec7decb81f2/aenum-3.1.17-py3-none-any.whl", hash = "sha256:8b883a37a04e74cc838ac442bdd28c266eae5bbf13e1342c7ef123ed25230139", size = 165560, upload-time = "2026-03-20T20:43:27.681Z" }, +] + +[[package]] +name = "alabaster" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz", hash = "sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e", size = 24210, upload-time = "2024-07-26T18:15:03.762Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "anyio" +version = "4.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/72/5562aabb8dd7181e8e860622a38bea08d17842b99ecd4c91f84ac95251b0/anyio-4.14.1.tar.gz", hash = "sha256:8d648a3544c1a700e3ff78615cd679e4c5c3f149904287e73687b2596963629e", size = 254831, upload-time = "2026-06-24T20:56:06.017Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/7b/90df4a0a816d98d6ea26f559d87836d494a2cf1fcf063be67df50a7bcc30/anyio-4.14.1-py3-none-any.whl", hash = "sha256:4e5533c5b8ff0a24f5d7a176cbe6877129cd183893f66b537f8f227d10527d72", size = 124875, upload-time = "2026-06-24T20:56:04.413Z" }, +] + +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441, upload-time = "2025-07-30T10:02:05.147Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/d3/942aa10782b2697eee7af5e12eeff5ebb325ccfb86dd8abda54174e377e4/argon2_cffi_bindings-25.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1c70058c6ab1e352304ac7e3b52554daadacd8d453c1752e547c76e9c99ac44", size = 86558, upload-time = "2025-07-30T10:01:43.943Z" }, + { url = "https://files.pythonhosted.org/packages/0d/82/b484f702fec5536e71836fc2dbc8c5267b3f6e78d2d539b4eaa6f0db8bf8/argon2_cffi_bindings-25.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2fd3bfbff3c5d74fef31a722f729bf93500910db650c925c2d6ef879a7e51cb", size = 92364, upload-time = "2025-07-30T10:01:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/c9/c1/a606ff83b3f1735f3759ad0f2cd9e038a0ad11a3de3b6c673aa41c24bb7b/argon2_cffi_bindings-25.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c4f9665de60b1b0e99bcd6be4f17d90339698ce954cfd8d9cf4f91c995165a92", size = 85637, upload-time = "2025-07-30T10:01:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/44/b4/678503f12aceb0262f84fa201f6027ed77d71c5019ae03b399b97caa2f19/argon2_cffi_bindings-25.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ba92837e4a9aa6a508c8d2d7883ed5a8f6c308c89a4790e1e447a220deb79a85", size = 91934, upload-time = "2025-07-30T10:01:47.203Z" }, + { url = "https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e021e87faa76ae0d413b619fe2b65ab9a037f24c60a1e6cc43457ae20de6dc6", size = 81246, upload-time = "2025-07-30T10:01:54.145Z" }, + { url = "https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e924cfc503018a714f94a49a149fdc0b644eaead5d1f089330399134fa028a", size = 87126, upload-time = "2025-07-30T10:01:55.074Z" }, + { url = "https://files.pythonhosted.org/packages/72/70/7a2993a12b0ffa2a9271259b79cc616e2389ed1a4d93842fac5a1f923ffd/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87b72589133f0346a1cb8d5ecca4b933e3c9b64656c9d175270a000e73b288d", size = 80343, upload-time = "2025-07-30T10:01:56.007Z" }, + { url = "https://files.pythonhosted.org/packages/78/9a/4e5157d893ffc712b74dbd868c7f62365618266982b64accab26bab01edc/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99", size = 86777, upload-time = "2025-07-30T10:01:56.943Z" }, +] + +[[package]] +name = "argparse-dataclass" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/ff/a2e4e328075ddef2ac3c9431eb12247e4ba707a70324894f1e6b4f43c286/argparse_dataclass-2.0.0.tar.gz", hash = "sha256:09ab641c914a2f12882337b9c3e5086196dbf2ee6bf0ef67895c74002cc9297f", size = 6395, upload-time = "2023-06-11T20:32:54.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/66/e6c0a808950ba5a4042e2fcedd577fc7401536c7db063de4d7c36be06f84/argparse_dataclass-2.0.0-py3-none-any.whl", hash = "sha256:3ffc8852a88d9d98d1364b4441a712491320afb91fb56049afd8a51d74bb52d2", size = 8762, upload-time = "2023-06-11T20:32:52.724Z" }, +] + +[[package]] +name = "array-api-compat" +version = "1.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/12/65d297d3e425cb7fe5247b7ec14b4ed83539e9b5e89b65c15ea7ee274182/array_api_compat-1.15.0.tar.gz", hash = "sha256:53c5f922491bf15f62847afafc4e39eedfae57d218988fefb8cce39c2a9b3dea", size = 129305, upload-time = "2026-06-07T20:53:24.964Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/16/1a8fd2b19544b84575cf84ef7aa3ad4c173b756d5f087c91f85d1b295777/array_api_compat-1.15.0-py3-none-any.whl", hash = "sha256:7b1b9c53269061403fd5f45a8de349f16e7887653328bfa0c5f2d45299ff0a8e", size = 79113, upload-time = "2026-06-07T20:53:23.621Z" }, +] + +[[package]] +name = "arrow" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil", marker = "sys_platform == 'linux'" }, + { name = "tzdata", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/33/032cdc44182491aa708d06a68b62434140d8c50820a087fac7af37703357/arrow-1.4.0.tar.gz", hash = "sha256:ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7", size = 152931, upload-time = "2025-10-18T17:46:46.761Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl", hash = "sha256:749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205", size = 68797, upload-time = "2025-10-18T17:46:45.663Z" }, +] + +[[package]] +name = "asteval" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/71/bec6758ab41ada3b6870cb194ea16d2e76bb530562c8a8cd7d2167012bea/asteval-1.0.9.tar.gz", hash = "sha256:4f7323d63d9fbf89d7e1d5f8bb59b6c093970c15bb5926e6e67d0195858aa230", size = 55237, upload-time = "2026-06-11T18:40:36.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/d7/a71b12b625c44cfcb95d41397752edccc0a731903081609cffd6720f671d/asteval-1.0.9-py3-none-any.whl", hash = "sha256:60513ea5d608d27bdb0cf3785dc993a6c3adeaee45f1bded0953a09e904e461f", size = 23320, upload-time = "2026-06-11T18:40:35.068Z" }, +] + +[[package]] +name = "astropy" +version = "8.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astropy-iers-data", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "pyerfa", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/c4/21be4313ddfde5f60e0607fd307f367b9e0f0bf153a89b10cbd036dd8cfd/astropy-8.0.1.tar.gz", hash = "sha256:45ca31d5b91fa294cd590a4791a32db94de7f9c8a343155f4d5877baa82351da", size = 7152500, upload-time = "2026-07-05T07:24:48.482Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/3b/d29035da0a08e3b0b03d63abe9ce69b900d8ef7863dc557a2e34ebf56a4e/astropy-8.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9a4044396c15969bb029b648521f7c25d294e982dbd23e7c2c7dd328178b8b98", size = 10414634, upload-time = "2026-07-05T07:24:36.129Z" }, + { url = "https://files.pythonhosted.org/packages/fa/14/6f4427419f8a02e1d0a885fb64d8d4454c5d443627aa7cb1a3a36e16abb7/astropy-8.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fa11d56855e10107ea2231a6b6a33dbf1edbea6890adf34634c1f1d8f25c5a5a", size = 10452095, upload-time = "2026-07-05T07:24:38.424Z" }, + { url = "https://files.pythonhosted.org/packages/b2/ef/a5cf36a402c4511a776405f12d0b35196f55f4312ce407012a2fbcf1e4e0/astropy-8.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5b482bc6c57c966e6c6234410a1d9afbcf92bcac858cf287812f8c99ddc3fafc", size = 10407732, upload-time = "2026-07-05T07:24:40.774Z" }, +] + +[[package]] +name = "astropy-healpix" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astropy", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/bb/fb3dcca212b1d57ed26b33eabb9e1cc18fcadbfc2f2976b20530b7d80b90/astropy_healpix-2.0.0.tar.gz", hash = "sha256:6ce7646dc5b1c09a7ed24b1f23d3e2fce8e3570387220d26a24487ab001b3b0b", size = 112064, upload-time = "2026-06-29T20:02:46.407Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/d3/6055bf7486e0fc980a2c842fc757918f79b012c3181d6f03b3fe230a2f65/astropy_healpix-2.0.0-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6bf86b2cdd9615de0031bc13b33fb72bcf085c5a2c78a6cf27c7a02b9ec05be7", size = 191108, upload-time = "2026-06-29T20:02:34.106Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9e/0bf5fdb204527abb6ee1ae8f77ce867b1150203864a47c19d0e5bd3ae63a/astropy_healpix-2.0.0-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd3614d685ec2cd629f4e2bbb0e1db625fe878dde778b335ec07ab4754809d78", size = 193204, upload-time = "2026-06-29T20:02:35.261Z" }, + { url = "https://files.pythonhosted.org/packages/4a/1a/5b8680a2a68df29a38baebd9659c4c4b42f5885c7fcfeb03dc336afb0053/astropy_healpix-2.0.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8ffc56532822414fc6ba9b5e80b7897f98c96762d4ae69f14f5797e18946c26e", size = 188492, upload-time = "2026-06-29T20:02:36.721Z" }, + { url = "https://files.pythonhosted.org/packages/7e/43/174bf249286926518e77e29a52179a5c971544fa29affe8dee6ab33bcd5b/astropy_healpix-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:727588aebabb0bdf7f27aeb793a721090e9c62ab44c5dfe083c5a5ec4ad91df8", size = 198639, upload-time = "2026-06-29T20:02:41.547Z" }, + { url = "https://files.pythonhosted.org/packages/95/e1/2e57926e5c1e6a7b648b980656a3efd94ee58909fda577f7bf4fd701dca5/astropy_healpix-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31c4f998f83e7be4245fa75a48767d7749b2f233ee61e134573106dfbe53c73c", size = 201015, upload-time = "2026-06-29T20:02:42.721Z" }, + { url = "https://files.pythonhosted.org/packages/93/96/ce466e4f53b59b8e0a89fd09824cce2532c5b9146279b7397d844f78cadf/astropy_healpix-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3eca64fb1640dcb6b5633db73a33b2d208f64c5339a454ca38f538be3cdc0a4b", size = 195935, upload-time = "2026-06-29T20:02:43.868Z" }, +] + +[[package]] +name = "astropy-iers-data" +version = "0.2026.7.6.1.1.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7f/e3/d32d2401c53fa1aaf118c963c51c4eb9cd10d074a72a52430904dc8ef46e/astropy_iers_data-0.2026.7.6.1.1.20.tar.gz", hash = "sha256:bc38de235802c9f006c8a04693706e2adcc9e7ac1cbf415dc042b5f0ac14f1f3", size = 1939850, upload-time = "2026-07-06T01:02:02.379Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/a3/1d816863674c610234afef9a444d4eb562689a3fa1a872ac5687b4e0eac4/astropy_iers_data-0.2026.7.6.1.1.20-py3-none-any.whl", hash = "sha256:7aa50fa7f6d13258357d1e17fa97f3a8c2f04b9a7429d056e986a91cc46576a0", size = 1994649, upload-time = "2026-07-06T01:02:00.977Z" }, +] + +[[package]] +name = "asttokens" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/a5/8e3f9b6771b0b408517c82d97aed8f2036509bc247d46114925e32fe33f0/asttokens-3.0.1.tar.gz", hash = "sha256:71a4ee5de0bde6a31d64f6b13f2293ac190344478f081c3d1bccfcf5eacb0cb7", size = 62308, upload-time = "2025-11-15T16:43:48.578Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl", hash = "sha256:15a3ebc0f43c2d0a50eeafea25e19046c68398e487b9f1f5b517f7c0f40f976a", size = 27047, upload-time = "2025-11-15T16:43:16.109Z" }, +] + +[[package]] +name = "async-lru" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/1f/989ecfef8e64109a489fff357450cb73fa73a865a92bd8c272170a6922c2/async_lru-2.3.0.tar.gz", hash = "sha256:89bdb258a0140d7313cf8f4031d816a042202faa61d0ab310a0a538baa1c24b6", size = 16332, upload-time = "2026-03-19T01:04:32.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/e2/c2e3abf398f80732e58b03be77bde9022550d221dd8781bf586bd4d97cc1/async_lru-2.3.0-py3-none-any.whl", hash = "sha256:eea27b01841909316f2cc739807acea1c623df2be8c5cfad7583286397bb8315", size = 8403, upload-time = "2026-03-19T01:04:30.883Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "babel" +version = "2.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, +] + +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, +] + +[[package]] +name = "bleach" +version = "6.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "webencodings", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/3c/e12ac860709702bd5ebeb9b56a4fe334f1001246ee1b8f2b7ee28912df7d/bleach-6.4.0.tar.gz", hash = "sha256:4202482733d85cedd04e59fcb2f89f4e4c7c385a78d3c3c23c30446843a37452", size = 204857, upload-time = "2026-06-05T13:01:13.734Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/9d/40b6267367182187139a4000b82a3b287d84d745bccd808e75d916920e9d/bleach-6.4.0-py3-none-any.whl", hash = "sha256:4b6b6a54fff2e69a3dde9d21cc6301220bee3c3cb792187d11403fd795031081", size = 165109, upload-time = "2026-06-05T13:01:12.504Z" }, +] + +[package.optional-dependencies] +css = [ + { name = "tinycss2", marker = "sys_platform == 'linux'" }, +] + +[[package]] +name = "blosc2" +version = "4.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx", extra = ["http2"], marker = "sys_platform == 'linux'" }, + { name = "msgpack", marker = "sys_platform == 'linux'" }, + { name = "ndindex", marker = "sys_platform == 'linux'" }, + { name = "numexpr", marker = "platform_machine != 'wasm32' and sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "pydantic", marker = "sys_platform == 'linux'" }, + { name = "rich", marker = "sys_platform == 'linux'" }, + { name = "threadpoolctl", marker = "platform_machine != 'wasm32' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/26/c5b32a553f002e91886213ec91ff56a2dd8555e5e78eb4c289addb208642/blosc2-4.8.0.tar.gz", hash = "sha256:9789c3248052be59e63d6debafec4cd59d0eb4a6fd1cbaced36865d5d40ffa63", size = 5681856, upload-time = "2026-07-10T09:38:33.592Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/79/54ea19add4199547d684f3a90c3cc0568b6d915403b15c3e4bc73d4bbf3b/blosc2-4.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:95b04eb8a769e98de9640915f9c5c0a3074714762958896440e5a479a96bf83e", size = 6704345, upload-time = "2026-07-10T09:37:56.942Z" }, + { url = "https://files.pythonhosted.org/packages/e3/b0/aa277d5952ac4c673924454277de0800c780bcdb15917e303dac8ea3840f/blosc2-4.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:12668fe516f18f783877457ddcc5072d80edf4939914e27acf900ef65e970e69", size = 7208868, upload-time = "2026-07-10T09:37:58.649Z" }, + { url = "https://files.pythonhosted.org/packages/ab/1b/765d2d38609d11e727f63c44f30abacccd59132e74f6e082b3741656f108/blosc2-4.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90b1a86b9044b3b242d78d5ac1d62abb8af7b1da355eb8eb45978539f9df2eac", size = 6708353, upload-time = "2026-07-10T09:38:05.91Z" }, + { url = "https://files.pythonhosted.org/packages/ee/aa/c72f0ff0c1cfa7e3a06530453f9b5388e4bb6266b73608f58232f181d68c/blosc2-4.8.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c17ec006d2770a4710552b0f4f8ceb40654fda08fe5047b27ccb7b0f396c3c6e", size = 7214012, upload-time = "2026-07-10T09:38:07.685Z" }, + { url = "https://files.pythonhosted.org/packages/6e/54/d7b8a01b43751840d75908d57601f03711a7cf260d46140f4c09dff5f9d6/blosc2-4.8.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:2af9162e7712323b21ad4d6502326c21459b6851b758b0421aa0be2e8b9dc0c1", size = 2300187, upload-time = "2026-07-10T09:38:09.407Z" }, + { url = "https://files.pythonhosted.org/packages/2a/2d/0c9961009c422f366555574af4653f3c13ba73fe075efb31e7af42151af5/blosc2-4.8.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0bb7d94bffdef8304454bbb373b8e9bf41966f8ddb58c449782c649d4d6a395c", size = 6724604, upload-time = "2026-07-10T09:38:16.688Z" }, + { url = "https://files.pythonhosted.org/packages/6d/8e/cb14d6e21fa44255739e2dac8d66ec7fb3d7aee348f198cc943e45d84d76/blosc2-4.8.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec3fc9ad1b2528b471670233e9cf631be104b8526cf4b958a74099f6756c98f6", size = 7217824, upload-time = "2026-07-10T09:38:18.951Z" }, + { url = "https://files.pythonhosted.org/packages/e0/1f/ea276e70d09ed406aba4012a47c7595ed772b25fc096fba8b9eac3494db9/blosc2-4.8.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:da018d11be4ac995aeb7b1a406c5cafce8c9f8a3ca8a540fe872a482de198b0d", size = 2297787, upload-time = "2026-07-10T09:38:20.871Z" }, + { url = "https://files.pythonhosted.org/packages/2f/b7/cba879eed1f510b3f034fbeea7db81cb535c822fe96ac62b5e47012c0b9b/blosc2-4.8.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8574afd38d20f953e9114d7362535ef0019a4c85f2a672e6f3b36dcd00a77f32", size = 6669949, upload-time = "2026-07-10T09:38:27.542Z" }, + { url = "https://files.pythonhosted.org/packages/07/5a/012f58a518bebf0d11931a5a12ba08955aa959cca59764ad38a8d82c9a1b/blosc2-4.8.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f8b459e22059a3a794480df2c72ff008ab6c95b9799fa2979c1dd7830cf44e2", size = 7186485, upload-time = "2026-07-10T09:38:29.411Z" }, +] + +[[package]] +name = "cadcutils" +version = "1.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distro", marker = "sys_platform == 'linux'" }, + { name = "lxml", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "pyopenssl", marker = "sys_platform == 'linux'" }, + { name = "requests", marker = "sys_platform == 'linux'" }, + { name = "setuptools", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/44/4964823c72c9e6a93e26de0118cead37fa3c3600b4677f0c06788c903c80/cadcutils-1.6.2.tar.gz", hash = "sha256:6e2d7822756d48a363bc5f857cd717f80aab760554df86518b957f8826a7a906", size = 94673, upload-time = "2026-06-22T17:35:47.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/a4/963d9e7f8a2c385a42b77b2791a67f1bff32aa9c48299af2e6cafb8f3a5a/cadcutils-1.6.2-py3-none-any.whl", hash = "sha256:a95323d422731b41f9d24c2f06785ea2231cf69d1cc6ccacc4175df72b988877", size = 164697, upload-time = "2026-06-22T17:35:45.6Z" }, +] + +[[package]] +name = "camb" +version = "1.6.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "sympy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/ad/f2e6446fbd94f7adc1474e1f4e40d2e7fa7e5e7143640cb8557d492f0a73/camb-1.6.6.tar.gz", hash = "sha256:9856202a5c05570256e52377b20431891c7b08b2e9c334e141fd08d2a085516f", size = 799344, upload-time = "2026-03-11T16:46:55.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/f6/9131f388386c50982b1b2992424deca857e7c019f4014dab0770de690a84/camb-1.6.6-py3-none-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eded2dd7f19226ed4717535c07a029fc958b7c581549250da33a768e12c87b13", size = 1592029, upload-time = "2026-03-11T16:46:44.925Z" }, + { url = "https://files.pythonhosted.org/packages/41/53/9d0b3cf1a2a07a882adac9565074ac6974fb8b031c31da438d407db22c89/camb-1.6.6-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bb8e0c85b3e33d78ca0676c84b0b60b54dfa72d4e520a805166cab0458e81094", size = 2258716, upload-time = "2026-03-11T16:46:46.458Z" }, +] + +[[package]] +name = "certifi" +version = "2026.6.17" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/c7/424b75da314c1045981bd9777432fad05a9e0c69daa4ed7e308bbaffe405/certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", size = 134594, upload-time = "2026-06-17T10:31:07.894Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/c0/d1ec30ffb370f748f2fb54425972bfef9871e0132e82fb589c46b6676049/cffi-2.1.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d", size = 214815, upload-time = "2026-07-06T21:32:48.557Z" }, + { url = "https://files.pythonhosted.org/packages/1b/dc/5620cf930688be01f2d673804291de757a934c90b946dbdc3d84130c2ea4/cffi-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7", size = 222429, upload-time = "2026-07-06T21:32:49.848Z" }, + { url = "https://files.pythonhosted.org/packages/4b/a4/77b53abbf7a1e0beb9637edbef2a94d15f9c822f591e85d439ffd91519a6/cffi-2.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b", size = 210315, upload-time = "2026-07-06T21:32:51.221Z" }, + { url = "https://files.pythonhosted.org/packages/58/0c/f528df19cc94b675087324d4760d9e6d5bfae97d6217aa4fac43de4f5fcc/cffi-2.1.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7", size = 208859, upload-time = "2026-07-06T21:32:52.512Z" }, + { url = "https://files.pythonhosted.org/packages/62/f2/c9522a81c32132799a1972c39f5c5f8b4c8b9f00488a23feaa6c06f07741/cffi-2.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66", size = 221844, upload-time = "2026-07-06T21:32:53.704Z" }, + { url = "https://files.pythonhosted.org/packages/6e/28/bd53988b9833e8f8ad539d26f4c07a6b3f6bcb1e9e02e7ca038250b3428d/cffi-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe", size = 225287, upload-time = "2026-07-06T21:32:54.907Z" }, + { url = "https://files.pythonhosted.org/packages/79/99/0d0fd37f055224085f42bbb2c022d002e17dde4a97972822327b07d84101/cffi-2.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b", size = 223681, upload-time = "2026-07-06T21:32:56.329Z" }, + { url = "https://files.pythonhosted.org/packages/96/88/a996879e2eeccb815f6e3a5967b12a308257412acec882039d386bd2aa7b/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda", size = 194331, upload-time = "2026-07-06T21:33:03.697Z" }, + { url = "https://files.pythonhosted.org/packages/58/85/7ae00d5c8dd6266f4e944c3db630f3c5c9a98b61d469c714d848b1d8138a/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b", size = 196966, upload-time = "2026-07-06T21:33:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1c/4ed5a0e5bdca6cbc275556de3328dd1b76fd0c11cc13c88fe66d1d8715f2/cffi-2.1.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db", size = 214747, upload-time = "2026-07-06T21:33:09.671Z" }, + { url = "https://files.pythonhosted.org/packages/3a/a6/e879bb68cc23a2bc9ba8f4b7d8019f0c2694bad2ab6c4a3701d429439f58/cffi-2.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f", size = 222392, upload-time = "2026-07-06T21:33:10.896Z" }, + { url = "https://files.pythonhosted.org/packages/88/f6/01890cfd63c08f8eb96a8319b0443690197d240a8bd6346048cf7bde9190/cffi-2.1.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d", size = 210285, upload-time = "2026-07-06T21:33:12.251Z" }, + { url = "https://files.pythonhosted.org/packages/a6/cf/2b684132056f438567b61e19d690dd31cd0921ace051e0a458be6074369e/cffi-2.1.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:47ff3a8bfd8cb9da1af7524b965127095055654c177fcfc7578debcb015eecd0", size = 208801, upload-time = "2026-07-06T21:33:13.617Z" }, + { url = "https://files.pythonhosted.org/packages/6f/08/f2e7d62c460faae0926f2d6e423694aa409ced3bc1fe2927a0a6e5f05416/cffi-2.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:799416bae98336e400981ff6e532d67d5c709cfb30afb79865a1315f94b0e224", size = 221808, upload-time = "2026-07-06T21:33:15.466Z" }, + { url = "https://files.pythonhosted.org/packages/38/37/04f54b8e63a02f3d908332c9effbf8c366167c6f733ed8a3d4f79b7e2a1e/cffi-2.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:961be50688f7fba2fa65f63712d3b9b341a22311f5253460ce933f52f0de1c8c", size = 225241, upload-time = "2026-07-06T21:33:16.869Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d6/c72eecca433cd3e681c65ed313ab4835d9d4a379704d0f628a6a05f51c2e/cffi-2.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bf5c6cf48238b0eb4c086978c492ad1cbc22373fc5b2d7353b3a598ce6db887a", size = 223588, upload-time = "2026-07-06T21:33:18.239Z" }, + { url = "https://files.pythonhosted.org/packages/d8/f0/81478e482afa03f6d18dc8f2afb5edc45b3080853b634b5ed91961be0998/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a", size = 194142, upload-time = "2026-07-06T21:33:23.657Z" }, + { url = "https://files.pythonhosted.org/packages/7d/95/8de304305cd9204974b0ca051b86d307cafca13aa575a0ef1b44d92c0d8c/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3", size = 196819, upload-time = "2026-07-06T21:33:25.007Z" }, + { url = "https://files.pythonhosted.org/packages/2e/d2/065fcae1c73979fac8e054462478d0ff8a29c40cdc2ed7ea5676a061df53/cffi-2.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6", size = 222353, upload-time = "2026-07-06T21:33:29.178Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a5/e8bbb1ce5b3ac2f53ad6a10bde44318a5a8d99d4f4a000d44a6e39aeb3e4/cffi-2.1.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913", size = 210051, upload-time = "2026-07-06T21:33:30.534Z" }, + { url = "https://files.pythonhosted.org/packages/28/ed/c127d3ac36e899c965e3361357c3befacd6578c03f40125183e41c3b219e/cffi-2.1.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d", size = 208630, upload-time = "2026-07-06T21:33:31.753Z" }, + { url = "https://files.pythonhosted.org/packages/cc/d7/97d3136f81db489ec8d1d67748c110d6c994268fd7528014aa9f2b085e4e/cffi-2.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d53d10f7da99ae46f7373b9150393e9c5eab9b224909982b43832668de4779f5", size = 221593, upload-time = "2026-07-06T21:33:33.044Z" }, + { url = "https://files.pythonhosted.org/packages/d3/27/93195977168ee63aed233a1a0993a2178798654d1f4bddcdd321d6fd3b21/cffi-2.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c351efb95e832a853a29361675f33a7ce53de1a109cd73fd47af0712213aa4ce", size = 225146, upload-time = "2026-07-06T21:33:34.224Z" }, + { url = "https://files.pythonhosted.org/packages/b3/c1/6dbd291ee2ae5a50a034aa057207081f545923bbf15dad4511e985aafff5/cffi-2.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:dbf7c7a88e2bac086f06d14577332760bdeecc42bdec8ac4077f6260557d9326", size = 223240, upload-time = "2026-07-06T21:33:35.57Z" }, + { url = "https://files.pythonhosted.org/packages/14/d0/117dcd9209255ad8571fbc8c92ef32593a1d294dcec91ddc4e4db50606f2/cffi-2.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98", size = 223899, upload-time = "2026-07-06T21:33:39.514Z" }, + { url = "https://files.pythonhosted.org/packages/b6/3d/f20f8b886b254e3ad10e15cd4186d3aed49f3e6a35ab37aab9f8f25f7c03/cffi-2.1.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13", size = 211652, upload-time = "2026-07-06T21:33:40.851Z" }, + { url = "https://files.pythonhosted.org/packages/28/3b/fad54de07260b93ddeef4b96d0131d57ea900675df1d410ae1deee52d7a6/cffi-2.1.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d", size = 210755, upload-time = "2026-07-06T21:33:42.183Z" }, + { url = "https://files.pythonhosted.org/packages/cc/82/3d5c705acb7abbba9bbd7d79b8e62e0f25b6120eb7ae6ac49f1b721722fe/cffi-2.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ac0f1a2d0cfa7eea3f2aaf006ab6e70e8feeb16b75d65b7e5939982ca2f11056", size = 223933, upload-time = "2026-07-06T21:33:43.603Z" }, + { url = "https://files.pythonhosted.org/packages/6c/d0/47e338384ab6b1004241002fa616301020cea4fc95f283506565d252f276/cffi-2.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c16914df9fb7f500e440e6875fa23ff5e0b31db01fa9c06af98d59a91f0dc2e4", size = 226749, upload-time = "2026-07-06T21:33:45.046Z" }, + { url = "https://files.pythonhosted.org/packages/70/25/65bd5b58ea4bfdfc15cde02cb5365f89ef8ab8b2adfb8fe5c4bd4233382f/cffi-2.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5ecbd0499275d57506d397eebe1981cee87b47fcd9ef5c22cab7ed7644a39a94", size = 225703, upload-time = "2026-07-06T21:33:46.374Z" }, + { url = "https://files.pythonhosted.org/packages/55/c7/8c8c50cb11c6750051daf12164098a9a6f027ac4356967fd4d800a07f242/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c", size = 194121, upload-time = "2026-07-06T21:33:56.109Z" }, + { url = "https://files.pythonhosted.org/packages/99/e2/67680bf19a6b60d2bb7ff83baefa2a4c3d2d7dc0f3277034b802e1fc504c/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001", size = 196820, upload-time = "2026-07-06T21:33:57.288Z" }, + { url = "https://files.pythonhosted.org/packages/ef/c3/ad299dc38f3583f8d916b299f028af418a9ec98bc695fcbebeae7420691c/cffi-2.1.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699", size = 222342, upload-time = "2026-07-06T21:34:01.814Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d8/df4543cc087245044ed02ef3ad8e0a26619d0075ac7a77a12dc81177851b/cffi-2.1.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022", size = 210073, upload-time = "2026-07-06T21:34:03.255Z" }, + { url = "https://files.pythonhosted.org/packages/2c/0e/fac738d73728c6cea2a88a2883dca54892496cbba88a1dc1f2909cb8a6f5/cffi-2.1.0-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0", size = 208551, upload-time = "2026-07-06T21:34:04.433Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3f/0b04a700dd64f465c93020253a793a82c9b4dff9961f48facd0df945d9b8/cffi-2.1.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d3538f9c0e50670f4deb93dbb696576e60590369cae2faf7de681e597a8a1f1", size = 221649, upload-time = "2026-07-06T21:34:06.157Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7c/b7379a5704c79eda57ce075869ba70a0368d1c850f803b3c0d078d39dcaf/cffi-2.1.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:8f9ec95b8a043d3dfbc74d9abc6f7baf524dd27a8dc160b0a32ff9cdab650c28", size = 225203, upload-time = "2026-07-06T21:34:07.489Z" }, + { url = "https://files.pythonhosted.org/packages/5a/02/d5e6c43ea85c41bda2a184a3418f195fe7cf602967a8d2b94e085b83deef/cffi-2.1.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:af5e2915d41fe6c961694d7bfdc8562942638200f3ce2765dfb8b745cf997629", size = 223263, upload-time = "2026-07-06T21:34:08.712Z" }, + { url = "https://files.pythonhosted.org/packages/e0/27/1d0b408497e41a74795af122d7b603c418c5fed0171450f899afd04e594f/cffi-2.1.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d", size = 223904, upload-time = "2026-07-06T21:34:12.606Z" }, + { url = "https://files.pythonhosted.org/packages/8b/31/e115c985105dd7ffb32444505f18ceb874bb42d992af05d5dced7ecf1980/cffi-2.1.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8", size = 211554, upload-time = "2026-07-06T21:34:13.987Z" }, + { url = "https://files.pythonhosted.org/packages/5a/67/9e6e09409336d9e515c58367e7cfcf4f89df06ad25252675595a58eb59d5/cffi-2.1.0-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd", size = 210795, upload-time = "2026-07-06T21:34:15.972Z" }, + { url = "https://files.pythonhosted.org/packages/19/e5/d3cc82a4a0be7902af279c04181ad038449c096734464a5ae1de3e1401bd/cffi-2.1.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0611e7ebf90573a535ebdc33ae9da222d037853983e13359f580fab781ca017f", size = 223843, upload-time = "2026-07-06T21:34:17.509Z" }, + { url = "https://files.pythonhosted.org/packages/b9/65/b434abc97ce7cecc2c640fde160507c0ecc7e21544b483ba3325d2e2ea17/cffi-2.1.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:86cf8755a791f72c85dc287128cc62d4f24d392e3f1e15837245623f4a33cccc", size = 226773, upload-time = "2026-07-06T21:34:19.05Z" }, + { url = "https://files.pythonhosted.org/packages/b5/9f/d4dc66ca651eb1145a133314cda721abf13cfac3d28c4a0402263ae6ad75/cffi-2.1.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:ba00f661f8ba35d075c937174e27c2c421cec3942fd2e0ea3e66996757c0fdd9", size = 225719, upload-time = "2026-07-06T21:34:20.576Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, + { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, + { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, + { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, + { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, + { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, + { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, + { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" }, + { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" }, + { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" }, + { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" }, + { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" }, + { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" }, + { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" }, + { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" }, + { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" }, + { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" }, + { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" }, + { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" }, + { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, +] + +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + +[[package]] +name = "clmm" +version = "1.16.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astropy", marker = "sys_platform == 'linux'" }, + { name = "healpy", marker = "sys_platform == 'linux'" }, + { name = "matplotlib", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "qp-prob", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/73/7ff76d487cdbcddb9c80ec03313db1b11f48fe4fcb5e1609b6eca5a73159/clmm-1.16.10.tar.gz", hash = "sha256:48273784419c038a89619b798a140153ad1e451390ff56074f3b6825c91ebe32", size = 72475, upload-time = "2025-12-04T17:30:11.47Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/a6/2a730fd1b23bb67c31c83449b12e74d6a3369681e63a098f02e353bdb77f/clmm-1.16.10-py3-none-any.whl", hash = "sha256:97e70d1bb7f45f994ab19c07b405688d984bd28399d6e31187012e8a0bf2feb6", size = 89434, upload-time = "2025-12-04T17:30:10.009Z" }, +] + +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "comm" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/13/7d740c5849255756bc17888787313b61fd38a0a8304fc4f073dfc46122aa/comm-0.2.3.tar.gz", hash = "sha256:2dc8048c10962d55d7ad693be1e7045d891b7ce8d999c97963a5e3e99c055971", size = 6319, upload-time = "2025-07-25T14:02:04.452Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, +] + +[[package]] +name = "conda-inject" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/a8/8dc86113c65c949cc72d651461d6e4c544b3302a85ed14a5298829e6a419/conda_inject-1.3.2.tar.gz", hash = "sha256:0b8cde8c47998c118d8ff285a04977a3abcf734caf579c520fca469df1cd0aac", size = 3635, upload-time = "2024-05-27T12:20:58.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/4c/fc30b69fb4062aee57e3ab7ff493647c4220144908f0839c619f912045bf/conda_inject-1.3.2-py3-none-any.whl", hash = "sha256:6e641b408980c2814e3e527008c30749117909a21ff47392f07ef807da93a564", size = 4133, upload-time = "2024-05-27T12:20:57.332Z" }, +] + +[[package]] +name = "configargparse" +version = "1.7.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/0b/30328302903c55218ffc5199646d0e9d28348ff26c02ba77b2ffc58d294a/configargparse-1.7.5.tar.gz", hash = "sha256:e3f9a7bb6be34d66b2e3c4a2f58e3045f8dfae47b0dc039f87bcfaa0f193fb0f", size = 53548, upload-time = "2026-03-11T02:19:38.144Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/19/3ba5e1b0bcc7b91aeab6c258afd70e4907d220fed3972febe38feb40db30/configargparse-1.7.5-py3-none-any.whl", hash = "sha256:1e63fdffedf94da9cd435fc13a1cd24777e76879dd2343912c1f871d4ac8c592", size = 27692, upload-time = "2026-03-11T02:19:36.442Z" }, +] + +[[package]] +name = "connection-pool" +version = "0.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/df/c9b4e25dce00f6349fd28aadba7b6c3f7431cc8bd4308a158fbe57b6a22e/connection_pool-0.0.3.tar.gz", hash = "sha256:bf429e7aef65921c69b4ed48f3d48d3eac1383b05d2df91884705842d974d0dc", size = 3795, upload-time = "2020-09-17T02:48:28.824Z" } + +[[package]] +name = "contourpy" +version = "1.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/1c/a12359b9b2ca3a845e8f7f9ac08bdf776114eb931392fcad91743e2ea17b/contourpy-1.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7", size = 332653, upload-time = "2025-07-26T12:01:24.155Z" }, + { url = "https://files.pythonhosted.org/packages/63/12/897aeebfb475b7748ea67b61e045accdfcf0d971f8a588b67108ed7f5512/contourpy-1.3.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8", size = 379536, upload-time = "2025-07-26T12:01:25.91Z" }, + { url = "https://files.pythonhosted.org/packages/43/8a/a8c584b82deb248930ce069e71576fc09bd7174bbd35183b7943fb1064fd/contourpy-1.3.3-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea", size = 384397, upload-time = "2025-07-26T12:01:27.152Z" }, + { url = "https://files.pythonhosted.org/packages/cc/8f/ec6289987824b29529d0dfda0d74a07cec60e54b9c92f3c9da4c0ac732de/contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1", size = 362601, upload-time = "2025-07-26T12:01:28.808Z" }, + { url = "https://files.pythonhosted.org/packages/05/0a/a3fe3be3ee2dceb3e615ebb4df97ae6f3828aa915d3e10549ce016302bd1/contourpy-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7", size = 1331288, upload-time = "2025-07-26T12:01:31.198Z" }, + { url = "https://files.pythonhosted.org/packages/33/1d/acad9bd4e97f13f3e2b18a3977fe1b4a37ecf3d38d815333980c6c72e963/contourpy-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411", size = 1403386, upload-time = "2025-07-26T12:01:33.947Z" }, + { url = "https://files.pythonhosted.org/packages/73/23/90e31ceeed1de63058a02cb04b12f2de4b40e3bef5e082a7c18d9c8ae281/contourpy-1.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:348ac1f5d4f1d66d3322420f01d42e43122f43616e0f194fc1c9f5d830c5b286", size = 334672, upload-time = "2025-07-26T12:01:41.942Z" }, + { url = "https://files.pythonhosted.org/packages/ed/93/b43d8acbe67392e659e1d984700e79eb67e2acb2bd7f62012b583a7f1b55/contourpy-1.3.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:655456777ff65c2c548b7c454af9c6f33f16c8884f11083244b5819cc214f1b5", size = 381234, upload-time = "2025-07-26T12:01:43.499Z" }, + { url = "https://files.pythonhosted.org/packages/46/3b/bec82a3ea06f66711520f75a40c8fc0b113b2a75edb36aa633eb11c4f50f/contourpy-1.3.3-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:644a6853d15b2512d67881586bd03f462c7ab755db95f16f14d7e238f2852c67", size = 385169, upload-time = "2025-07-26T12:01:45.219Z" }, + { url = "https://files.pythonhosted.org/packages/4b/32/e0f13a1c5b0f8572d0ec6ae2f6c677b7991fafd95da523159c19eff0696a/contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4debd64f124ca62069f313a9cb86656ff087786016d76927ae2cf37846b006c9", size = 362859, upload-time = "2025-07-26T12:01:46.519Z" }, + { url = "https://files.pythonhosted.org/packages/33/71/e2a7945b7de4e58af42d708a219f3b2f4cff7386e6b6ab0a0fa0033c49a9/contourpy-1.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a15459b0f4615b00bbd1e91f1b9e19b7e63aea7483d03d804186f278c0af2659", size = 1332062, upload-time = "2025-07-26T12:01:48.964Z" }, + { url = "https://files.pythonhosted.org/packages/12/fc/4e87ac754220ccc0e807284f88e943d6d43b43843614f0a8afa469801db0/contourpy-1.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca0fdcd73925568ca027e0b17ab07aad764be4706d0a925b89227e447d9737b7", size = 1403932, upload-time = "2025-07-26T12:01:51.979Z" }, + { url = "https://files.pythonhosted.org/packages/ae/15/e59f5f3ffdd6f3d4daa3e47114c53daabcb18574a26c21f03dc9e4e42ff0/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e4e6b05a45525357e382909a4c1600444e2a45b4795163d3b22669285591c1ae", size = 326751, upload-time = "2025-07-26T12:02:00.343Z" }, + { url = "https://files.pythonhosted.org/packages/0f/81/03b45cfad088e4770b1dcf72ea78d3802d04200009fb364d18a493857210/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab3074b48c4e2cf1a960e6bbeb7f04566bf36b1861d5c9d4d8ac04b82e38ba20", size = 375486, upload-time = "2025-07-26T12:02:02.128Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ba/49923366492ffbdd4486e970d421b289a670ae8cf539c1ea9a09822b371a/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c3d53c796f8647d6deb1abe867daeb66dcc8a97e8455efa729516b997b8ed99", size = 388106, upload-time = "2025-07-26T12:02:03.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/52/5b00ea89525f8f143651f9f03a0df371d3cbd2fccd21ca9b768c7a6500c2/contourpy-1.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50ed930df7289ff2a8d7afeb9603f8289e5704755c7e5c3bbd929c90c817164b", size = 352548, upload-time = "2025-07-26T12:02:05.165Z" }, + { url = "https://files.pythonhosted.org/packages/32/1d/a209ec1a3a3452d490f6b14dd92e72280c99ae3d1e73da74f8277d4ee08f/contourpy-1.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4feffb6537d64b84877da813a5c30f1422ea5739566abf0bd18065ac040e120a", size = 1322297, upload-time = "2025-07-26T12:02:07.379Z" }, + { url = "https://files.pythonhosted.org/packages/bc/9e/46f0e8ebdd884ca0e8877e46a3f4e633f6c9c8c4f3f6e72be3fe075994aa/contourpy-1.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2b7e9480ffe2b0cd2e787e4df64270e3a0440d9db8dc823312e2c940c167df7e", size = 1391023, upload-time = "2025-07-26T12:02:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/b1/71/f93e1e9471d189f79d0ce2497007731c1e6bf9ef6d1d61b911430c3db4e5/contourpy-1.3.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22e9b1bd7a9b1d652cd77388465dc358dafcd2e217d35552424aa4f996f524f5", size = 335810, upload-time = "2025-07-26T12:02:18.9Z" }, + { url = "https://files.pythonhosted.org/packages/91/f9/e35f4c1c93f9275d4e38681a80506b5510e9327350c51f8d4a5a724d178c/contourpy-1.3.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a22738912262aa3e254e4f3cb079a95a67132fc5a063890e224393596902f5a4", size = 382871, upload-time = "2025-07-26T12:02:20.418Z" }, + { url = "https://files.pythonhosted.org/packages/b5/71/47b512f936f66a0a900d81c396a7e60d73419868fba959c61efed7a8ab46/contourpy-1.3.3-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:afe5a512f31ee6bd7d0dda52ec9864c984ca3d66664444f2d72e0dc4eb832e36", size = 386264, upload-time = "2025-07-26T12:02:21.916Z" }, + { url = "https://files.pythonhosted.org/packages/04/5f/9ff93450ba96b09c7c2b3f81c94de31c89f92292f1380261bd7195bea4ea/contourpy-1.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f64836de09927cba6f79dcd00fdd7d5329f3fccc633468507079c829ca4db4e3", size = 363819, upload-time = "2025-07-26T12:02:23.759Z" }, + { url = "https://files.pythonhosted.org/packages/3e/a6/0b185d4cc480ee494945cde102cb0149ae830b5fa17bf855b95f2e70ad13/contourpy-1.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1fd43c3be4c8e5fd6e4f2baeae35ae18176cf2e5cced681cca908addf1cdd53b", size = 1333650, upload-time = "2025-07-26T12:02:26.181Z" }, + { url = "https://files.pythonhosted.org/packages/43/d7/afdc95580ca56f30fbcd3060250f66cedbde69b4547028863abd8aa3b47e/contourpy-1.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6afc576f7b33cf00996e5c1102dc2a8f7cc89e39c0b55df93a0b78c1bd992b36", size = 1404833, upload-time = "2025-07-26T12:02:28.782Z" }, + { url = "https://files.pythonhosted.org/packages/66/06/8a475c8ab718ebfd7925661747dbb3c3ee9c82ac834ccb3570be49d129f4/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d304906ecc71672e9c89e87c4675dc5c2645e1f4269a5063b99b0bb29f232d13", size = 326748, upload-time = "2025-07-26T12:02:37.193Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a3/c5ca9f010a44c223f098fccd8b158bb1cb287378a31ac141f04730dc49be/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ca658cd1a680a5c9ea96dc61cdbae1e85c8f25849843aa799dfd3cb370ad4fbe", size = 375554, upload-time = "2025-07-26T12:02:38.894Z" }, + { url = "https://files.pythonhosted.org/packages/80/5b/68bd33ae63fac658a4145088c1e894405e07584a316738710b636c6d0333/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ab2fd90904c503739a75b7c8c5c01160130ba67944a7b77bbf36ef8054576e7f", size = 388118, upload-time = "2025-07-26T12:02:40.642Z" }, + { url = "https://files.pythonhosted.org/packages/40/52/4c285a6435940ae25d7410a6c36bda5145839bc3f0beb20c707cda18b9d2/contourpy-1.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7301b89040075c30e5768810bc96a8e8d78085b47d8be6e4c3f5a0b4ed478a0", size = 352555, upload-time = "2025-07-26T12:02:42.25Z" }, + { url = "https://files.pythonhosted.org/packages/24/ee/3e81e1dd174f5c7fefe50e85d0892de05ca4e26ef1c9a59c2a57e43b865a/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2a2a8b627d5cc6b7c41a4beff6c5ad5eb848c88255fda4a8745f7e901b32d8e4", size = 1322295, upload-time = "2025-07-26T12:02:44.668Z" }, + { url = "https://files.pythonhosted.org/packages/3c/b2/6d913d4d04e14379de429057cd169e5e00f6c2af3bb13e1710bcbdb5da12/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fd6ec6be509c787f1caf6b247f0b1ca598bef13f4ddeaa126b7658215529ba0f", size = 1391027, upload-time = "2025-07-26T12:02:47.09Z" }, +] + +[[package]] +name = "cosmo-numba" +version = "1.0.1.dev6+g188d272c6" +source = { git = "https://github.com/aguinot/cosmo-numba.git?rev=main#188d272c67d6d699d7cff7a7a53ded8dec759bb0" } +dependencies = [ + { name = "mpmath", marker = "sys_platform == 'linux'" }, + { name = "numba", marker = "sys_platform == 'linux'" }, + { name = "numbaquadpack", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "rocket-fft", marker = "sys_platform == 'linux'" }, + { name = "sympy", marker = "sys_platform == 'linux'" }, +] + +[[package]] +name = "cosmology" +version = "2022.10.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/38/04099bb2b6a626bd69a3f117afc60f2c47c08899e0730e0e48c898bf4745/cosmology-2022.10.9.tar.gz", hash = "sha256:0c2857c9bf1fdd09f1f11ab5765df0389a4101f3a900fa11251c3f37696f02d4", size = 8488, upload-time = "2022-10-10T10:18:16.229Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/2c/c80db37593ad593e3f6c60705fbdf690b33f22c89edd5507d2731346cd87/cosmology-2022.10.9-py3-none-any.whl", hash = "sha256:3903658c2474177a1a1c75771ae14458d93200c516f8fc6c3f4d776f3287b288", size = 9341, upload-time = "2022-10-10T10:18:14.928Z" }, +] + +[[package]] +name = "coverage" +version = "7.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/8b/adeb62ea8951f13c4c7fef2e7a85e1a06b499c8d8237ea589d496029e53f/coverage-7.15.0.tar.gz", hash = "sha256:9ac3fe7a1435986463eaa8ee253ae2f2a268709ba4ae5c7dd1f52a05391ad78f", size = 925362, upload-time = "2026-07-02T13:10:50.535Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/17/99fa688541ae1d6e84543a0e544f83de0c944815b63e9e7b1ed411d15036/coverage-7.15.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e4d0bb73455bf97ab243a8f12c37c686ccf1c13bb614b7b85f1d062f06f42b2c", size = 252705, upload-time = "2026-07-02T13:09:05.059Z" }, + { url = "https://files.pythonhosted.org/packages/fb/02/6a95a5cd83b74839017ef9cf48d2d8c9ae60af919e17a3f336e6f9f1b7bd/coverage-7.15.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:20d9ccc4ebd0edc434d86dfd2a1dd2a8efa6b6b3073d0485a394fee86459ebb4", size = 255441, upload-time = "2026-07-02T13:09:06.559Z" }, + { url = "https://files.pythonhosted.org/packages/67/f2/406f6c57d600f68185942422c4c00f1a3255d60aee6e5fd961425cd9987e/coverage-7.15.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20c8a976c365c8cb12f0cbd099508772ea41fb5fa80657a8506df0e11bd278c5", size = 256556, upload-time = "2026-07-02T13:09:08.197Z" }, + { url = "https://files.pythonhosted.org/packages/74/8e/d3fa48489c15ecdec1ba48fd61f68798555dddd2f6716f9ad42adeb1a2a9/coverage-7.15.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f948fd5ba1b9cbca91f0ae08b4c1ce2b139509149a435e2585d056d57d70bf01", size = 258815, upload-time = "2026-07-02T13:09:09.691Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/2d40ddd110462c6a2769677cf7f1c119a52b45f568978fc6c98e4cc0dd0f/coverage-7.15.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f58185f06edf6ad68ec9fb155d63ef650c82f3fbd7e1770e2867751fb13158f4", size = 253117, upload-time = "2026-07-02T13:09:11.212Z" }, + { url = "https://files.pythonhosted.org/packages/51/c0/310782f0d7c3cb2b5ac05ba8d205fe91f24a36f6bf3256098f1782181c38/coverage-7.15.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:02adc79a920c73c647c5d117f55747df7f2de94571884758ce8bc58e04f0a796", size = 254475, upload-time = "2026-07-02T13:09:13.029Z" }, + { url = "https://files.pythonhosted.org/packages/86/f7/702da6c275f8ae6ade423d2877243122932c9b27f5403003b9ef8c927d12/coverage-7.15.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6eb7c300fbed667fd6e3588eba71c1904cdb06110ca6fdf908c26bdd88b8e382", size = 252619, upload-time = "2026-07-02T13:09:14.699Z" }, + { url = "https://files.pythonhosted.org/packages/fb/84/c5b15a7e5ecba4e56218d772d99fe80a63e63f8d11f12783723a6005ab45/coverage-7.15.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:b5fb23fa2de9dce1f5c36c09066d8fcda16cd96e8e26686caa2d7cb9b567d65c", size = 256689, upload-time = "2026-07-02T13:09:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/95/2f/c8b07559b57701230c61b23a953858c052890c12ef568d81780c6c46e92e/coverage-7.15.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cec79341dbe6281484024979976d0c7f22beae08b4a254655decd25d42cbe766", size = 252189, upload-time = "2026-07-02T13:09:17.828Z" }, + { url = "https://files.pythonhosted.org/packages/6b/80/6d2f049dd3fd3dbfd60b62ba6b2162a04009e2c002ce70b24cf3878dec7a/coverage-7.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6c664c5444b1d970b1b2a450e21fb19ee5c9cfdf151ded2dda37260031cca0da", size = 254059, upload-time = "2026-07-02T13:09:19.304Z" }, + { url = "https://files.pythonhosted.org/packages/7b/19/3a80b97d3b2a5c77a01ae359c6bed20c13738fe3d9380f08616d4fec0281/coverage-7.15.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bbcbb317c2e5ded5b21104af81c29f391be2af98d065693ffbe8d23949b948e5", size = 252227, upload-time = "2026-07-02T13:09:28.543Z" }, + { url = "https://files.pythonhosted.org/packages/a1/fa/b70062750686bd7da454da27927622f48bbac6990ac7a4c4a4653e7b0036/coverage-7.15.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:27f31ecb458da3f859aab3f15ada871eb7a7768807d88df4a9f186bb17737970", size = 254823, upload-time = "2026-07-02T13:09:30.177Z" }, + { url = "https://files.pythonhosted.org/packages/a9/09/dad6a75a2e561b9dc5086a8c5257a7591d584246f67e23e70d2995b89ab6/coverage-7.15.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fb759be317fdc62e0f56bffdf61cfcb45c7761ad6b71e3e583e71a67ae753c", size = 256059, upload-time = "2026-07-02T13:09:31.979Z" }, + { url = "https://files.pythonhosted.org/packages/e6/e7/b5d2941fa9564573d44b693a871ff3156f0c42cbefe977a09fa7fdc59971/coverage-7.15.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5cf007add5ab4bb8fa9f4c77e3732127c9e6cad501d7db43355fbfafca0be84", size = 258190, upload-time = "2026-07-02T13:09:34.035Z" }, + { url = "https://files.pythonhosted.org/packages/7c/1d/8e895bcde3c57ccd46d896dda5f2b3d5df761a1b0c6c9d450d175dedc632/coverage-7.15.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc78d9843bd576fbe2118248258d485e968dc535f95ed504a7b0867ba9b51389", size = 252456, upload-time = "2026-07-02T13:09:35.765Z" }, + { url = "https://files.pythonhosted.org/packages/14/4c/f6997da343ddeb959be82c3b05322793f92c071ad45f7cb8a96336e2dd5f/coverage-7.15.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a263060f1de0b4b74b4e089c2a70b8003b3781c733329a9c8fd54995328f9950", size = 254192, upload-time = "2026-07-02T13:09:37.445Z" }, + { url = "https://files.pythonhosted.org/packages/17/27/a0bc09d032267b9da89d95a2d874cfbef2a5aebbf0e87cf7aba221d79a99/coverage-7.15.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c48decf16e0dfd5b049c7d5e82200c23c08126719142998d4f172444e3d0529e", size = 252153, upload-time = "2026-07-02T13:09:39.422Z" }, + { url = "https://files.pythonhosted.org/packages/54/c0/77fc233d9fba07b244c40948c53fe27308b8f21732fb3417f87fbd6fd992/coverage-7.15.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:08fb028000ed0aaa0a4cbdfbb98be7cb42f370db973fbbb469733505ab20e13e", size = 256310, upload-time = "2026-07-02T13:09:41.006Z" }, + { url = "https://files.pythonhosted.org/packages/d5/24/601cecfb5825becacb8d45219a018a3b55b9dbaec624efdb0ea249d08be2/coverage-7.15.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb7dc0c3b7d8a1077abea0b8546ebc5e26d6ef6ecefc2f0f5ad2b8a53bdad837", size = 251974, upload-time = "2026-07-02T13:09:42.733Z" }, + { url = "https://files.pythonhosted.org/packages/47/1e/6f45e5a5b3d5484318d368702af6716b5ab8913b0428bec981a562fcf296/coverage-7.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6cb3602054ccbe9f0d8c2dc04bbeba90d5719236e2cd06e042ddd6d3fc7b6e37", size = 253745, upload-time = "2026-07-02T13:09:44.376Z" }, + { url = "https://files.pythonhosted.org/packages/a7/ce/22bae91e0b75445f68d365c7643ed0aa4880bbf77450ee74ca65bdae53a7/coverage-7.15.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:769e8ece11a596315ebf5aa7ec383aeeed016c091d2bf6363ffb996d41529092", size = 252286, upload-time = "2026-07-02T13:09:54.996Z" }, + { url = "https://files.pythonhosted.org/packages/dd/1e/bec5e32aa508615d9d7a2790effb25fb4dc28606e995816afe400b25ece3/coverage-7.15.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:65a6b6164ee5c39e2f3803f314292d6c61a607ba7fee253d1e03c42dc3903502", size = 254789, upload-time = "2026-07-02T13:09:56.678Z" }, + { url = "https://files.pythonhosted.org/packages/17/29/0e865435b4354e4a7c03b1b7920046d31d0a273d55decefea27e011cb9bf/coverage-7.15.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:75128817f95a5c45bb01d65fd2d8b9cb54bbe03d81608fb70e3e14b437ad56c2", size = 256135, upload-time = "2026-07-02T13:09:58.343Z" }, + { url = "https://files.pythonhosted.org/packages/84/ff/33a870b58a13325d62fc0a6c8f01fa0ff667cef60c7498e2382a147dfa18/coverage-7.15.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9887bb428fe2d4cd4bee89bac1a6c9932f484afd5b36fbd4ff6ea5f825bb1f5e", size = 258449, upload-time = "2026-07-02T13:10:00.057Z" }, + { url = "https://files.pythonhosted.org/packages/18/7b/6fffe596bf3ddba8462758d02c5dad730fd91055a6634aa2e4226229181a/coverage-7.15.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0bfc0be1f702042207a93a00523b1065ee1fe951e96edf311581c0bbc2e34888", size = 252313, upload-time = "2026-07-02T13:10:01.946Z" }, + { url = "https://files.pythonhosted.org/packages/58/1b/11468dd6c1676ab831a70cb9a8d4e198e8607fa0b7220ab918b73fe9bfbd/coverage-7.15.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f64627d55def5a43282d70e08396672692f77e4da610a5bb8bb4060b432b6859", size = 254142, upload-time = "2026-07-02T13:10:04.065Z" }, + { url = "https://files.pythonhosted.org/packages/79/41/29328e21d16b1b95092c30dd700e08cf915bd3734f836df8f3bdb0e8fa9f/coverage-7.15.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:2c6f0fa473003905c6d5bac328ee4eba9fbea654f15bc24b8a3274b23363fa99", size = 252108, upload-time = "2026-07-02T13:10:06.11Z" }, + { url = "https://files.pythonhosted.org/packages/9b/de/05ccfb990439655b35afbfd8e0d13fe66677565a7d4eb38c3f5ef2635e1c/coverage-7.15.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2bcf9afaf064172c6ec3c58a325a9957ad1178c05dd934e25f253321776e0676", size = 256385, upload-time = "2026-07-02T13:10:08.141Z" }, + { url = "https://files.pythonhosted.org/packages/51/0e/486828a3d2695ea7a2609f17ff572f6b01905e608379440a11da4b8dffbe/coverage-7.15.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:baf06bc987115d6fb938d403f7eab684a057766c490367999a2b71a6883110c6", size = 251923, upload-time = "2026-07-02T13:10:10.179Z" }, + { url = "https://files.pythonhosted.org/packages/18/c7/03582b6715f078e5e558354c87616d945b9894cda2dace8e4009b17035e4/coverage-7.15.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0405f2ff97b1c4c0e782cb32e02f32369bcf2e6b618b591d67e1ea754575dfe", size = 253580, upload-time = "2026-07-02T13:10:12.052Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b4/c0ca3028f42c9a08e51feb4561ef1192e5de99797cd1db5b04590c215bda/coverage-7.15.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:94c9686bfe8a9a6810297aecbd99beaa3445f9e8dc2f80b1382cca0d86b64461", size = 263267, upload-time = "2026-07-02T13:10:23.261Z" }, + { url = "https://files.pythonhosted.org/packages/5f/aa/a375e3846e5d3c013dc600b2a3231089055c73d77f5393dd2192a8d64da6/coverage-7.15.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9bd671c25f9d85f09d7ec481d0e43d5139f486c06a37139847a7ce569788af72", size = 265390, upload-time = "2026-07-02T13:10:25.152Z" }, + { url = "https://files.pythonhosted.org/packages/92/e1/5783cdabb797305e1c9e4809fea496d31834c51fa772514f73dc148bcfc9/coverage-7.15.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:110cbdf8d2e216577312cf06ccf85539c0e5a5420ef747e4a4719b5e483c88cd", size = 267811, upload-time = "2026-07-02T13:10:27.249Z" }, + { url = "https://files.pythonhosted.org/packages/85/31/96d8bbf58b8e9193bc8389574a91a0db48355ee98feb66aa6bf8d1b32eea/coverage-7.15.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c5d4619214f1d9993e7b00a8600d14614b7e9d84e89507460b126aa5e6559e5", size = 268928, upload-time = "2026-07-02T13:10:29.242Z" }, + { url = "https://files.pythonhosted.org/packages/5e/7a/5294567e811a1cb7eda93140c628fa050d66189da28da320f93d1d815c73/coverage-7.15.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:781a704516e2d8346fbbd5be6c6f3412dd824785146528b3a01816f26c081007", size = 262378, upload-time = "2026-07-02T13:10:31.107Z" }, + { url = "https://files.pythonhosted.org/packages/69/3f/3f48538421f899f28946f90a3d272136a4686e1abf461cc9249a783ee0f3/coverage-7.15.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd4a1b44bcb65ee29e947ac92bbee04956df3a6bfc6143641bb6cae7ede00fc9", size = 265263, upload-time = "2026-07-02T13:10:32.942Z" }, + { url = "https://files.pythonhosted.org/packages/ce/d3/092df15efcab8a9c1467ee960eb8019bbad3f9300d115d89ea6195f369ff/coverage-7.15.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0e4950c9d6d3e39c64c991814ff315e2d0b9cb8152363594212c9e55208c0a8f", size = 262866, upload-time = "2026-07-02T13:10:35.104Z" }, + { url = "https://files.pythonhosted.org/packages/e5/ab/0254d2b88665efb2c57ad368cc77ab5de3435bd8d5add4729c1b0e79431e/coverage-7.15.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:fe9c87ff42e5472d80d21704972e1f96e104a0a599d77c5e35db5a3c562e2571", size = 266599, upload-time = "2026-07-02T13:10:37.05Z" }, + { url = "https://files.pythonhosted.org/packages/a8/79/1cfa4023e489ce6fbc7be4a5d442dbc375edb4f4fda39a352cedb53263c2/coverage-7.15.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f00d5ae1dd2fe13fb8186e3e7d37bcbd8b25c0d764ff7d1b32cef9be058510a8", size = 261714, upload-time = "2026-07-02T13:10:38.966Z" }, + { url = "https://files.pythonhosted.org/packages/b7/eb/fee5c8665656be63f497418d410484637c438172568688e8ac92e06574e7/coverage-7.15.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:363ab38cc78b615f11c9cac3cf1d7eef950c18b9fdedfb9066f59461dcf84d68", size = 264025, upload-time = "2026-07-02T13:10:40.789Z" }, + { url = "https://files.pythonhosted.org/packages/52/30/21b2ad45959cd50e909e02ebac1e30b4ceb7162e91c11d4c570223a458b7/coverage-7.15.0-py3-none-any.whl", hash = "sha256:56da6a4cbe8f7e9e80bd072ca9cefe67d7106a440a7ec06519ec6507ac94ad19", size = 212632, upload-time = "2026-07-02T13:10:48.641Z" }, +] + +[[package]] +name = "cryptography" +version = "49.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/99/d1c90d6041656cc6ee229dc99cd67fd0cd5aec3c5f7d72fffc27cc750054/cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493", size = 854345, upload-time = "2026-06-12T20:02:30.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/41/3797cfaf69cae04a13ee78ebd83f0678d9c02b4779d21ce24445326f1a69/cryptography-49.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db", size = 4692978, upload-time = "2026-06-12T20:01:21.305Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8b/43011f7ebe515a8aa20d61f290a326cd890c2e738e16e59eaff8d9c3a412/cryptography-49.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325", size = 4716422, upload-time = "2026-06-12T20:01:48.566Z" }, + { url = "https://files.pythonhosted.org/packages/4a/91/01ce7303a4579e6d3a6abef01bd322848e9ea7a219adcabc5048b9033571/cryptography-49.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2", size = 4700503, upload-time = "2026-06-12T20:02:47.091Z" }, + { url = "https://files.pythonhosted.org/packages/62/99/a2c95cf8293f07491e9e27c20cc4dcd18176d944e674679adeb1d0173fd6/cryptography-49.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b", size = 5309779, upload-time = "2026-06-12T20:02:08.987Z" }, + { url = "https://files.pythonhosted.org/packages/20/2c/0622f20ff02b2ef32558733443805dc82fd4c275be01b2d19d14676f3a1b/cryptography-49.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6", size = 4749683, upload-time = "2026-06-12T20:02:03.335Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5b/c5246635d5fd3b64e0d45ae10e99fd32fe9676a79915ccfe5a61ba9af1a5/cryptography-49.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c", size = 4337874, upload-time = "2026-06-12T20:02:54.323Z" }, + { url = "https://files.pythonhosted.org/packages/6d/88/05563c7fe2e914e87d1a536d06fe83e66b4e1d95cb593e05aea375531da8/cryptography-49.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7", size = 4700283, upload-time = "2026-06-12T20:01:34.822Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b6/d7696e4e890d6ae1469935164c9e5215c557671cb78d6e3f458ccceaa632/cryptography-49.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68", size = 5265844, upload-time = "2026-06-12T20:01:24.09Z" }, + { url = "https://files.pythonhosted.org/packages/a9/3c/f3ad17eecc1a57b0ba236dc01f90e783c51f4a2f35f64777cc4f47a184b2/cryptography-49.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9", size = 4749290, upload-time = "2026-06-12T20:01:30.848Z" }, + { url = "https://files.pythonhosted.org/packages/4f/01/339573cf1023163a400b0b5d16f6d507de413b9f60be6fd1b77feeaf6737/cryptography-49.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f", size = 4834612, upload-time = "2026-06-12T20:01:29.246Z" }, + { url = "https://files.pythonhosted.org/packages/71/fd/577302e213a1be9468f92d1afef66fcf1ef83d516819d9992ca547f592bd/cryptography-49.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459", size = 4980804, upload-time = "2026-06-12T20:01:42.853Z" }, + { url = "https://files.pythonhosted.org/packages/86/12/c48a424f38db03027be9f7ed5c7dc5de9933dbee992865f98b13727a009d/cryptography-49.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d", size = 4678835, upload-time = "2026-06-12T20:02:48.743Z" }, + { url = "https://files.pythonhosted.org/packages/68/28/8a3ad4653662c93fc44dc4e5d8fd374c25c42e07b34bbfbadf49cf57a5a8/cryptography-49.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa", size = 4697239, upload-time = "2026-06-12T20:02:56.03Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b2/2193fc74f81aee4f9b62733133b73b5176718932ed8f2e4b03fa040480a6/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb", size = 4685593, upload-time = "2026-06-12T20:02:50.666Z" }, + { url = "https://files.pythonhosted.org/packages/47/f1/1d3eaa243bfc5de4a187b22aa8c048b3e4980bfbe830ac46e6bac2e66947/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d", size = 5289961, upload-time = "2026-06-12T20:01:46.468Z" }, + { url = "https://files.pythonhosted.org/packages/58/39/2d51306721330c486495853eda1c567880ff036de15a14c4b74f399934af/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561", size = 4731145, upload-time = "2026-06-12T20:02:16.832Z" }, + { url = "https://files.pythonhosted.org/packages/17/50/983e838c7fd0d87fd8c969bcdd328edaf5f756e38df5281637424c155873/cryptography-49.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122", size = 4321719, upload-time = "2026-06-12T20:02:52.611Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f5/8f571d7e27c55bce9f76f026143bcb1e040a4233149ecca0bea5fa5dd5f7/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505", size = 4685209, upload-time = "2026-06-12T20:02:07.282Z" }, + { url = "https://files.pythonhosted.org/packages/e7/84/0e27016a6fc5a0886f797018b26aa42f40c09a82332bff77822a451deaaa/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866", size = 5246285, upload-time = "2026-06-12T20:01:32.439Z" }, + { url = "https://files.pythonhosted.org/packages/11/2d/5e1fb307cb5931881516b464c98774b3f2c36b5d4bb9a2830253cf553cad/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8", size = 4730441, upload-time = "2026-06-12T20:02:01.469Z" }, + { url = "https://files.pythonhosted.org/packages/e4/c0/bff5a02ee731d207d6a1ed51732549d8c53d2bc8da1d10ec6f2844201d68/cryptography-49.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3", size = 4815869, upload-time = "2026-06-12T20:01:36.574Z" }, + { url = "https://files.pythonhosted.org/packages/b9/26/814681d14248d95d73d5c3eea0c39a94eb8302df966f670a2c60de90974b/cryptography-49.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27", size = 4960948, upload-time = "2026-06-12T20:02:18.688Z" }, + { url = "https://files.pythonhosted.org/packages/3d/df/40577043ca124e17012f408ddddaeb213b856336ac82ddb3bc915f39e29f/cryptography-49.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4", size = 4692429, upload-time = "2026-06-12T20:01:53.628Z" }, + { url = "https://files.pythonhosted.org/packages/2c/99/2d13299eb3dd27b02dcfaafcc91d6b5cb3329f7cbd6d8f51921acd566c1a/cryptography-49.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18", size = 4700968, upload-time = "2026-06-12T20:02:45.383Z" }, + { url = "https://files.pythonhosted.org/packages/a5/4d/9c0cd02f95e2602dd5e563da149ee0830abef3537be8b34dc56281ebe27a/cryptography-49.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69", size = 4697758, upload-time = "2026-06-12T20:01:41.13Z" }, + { url = "https://files.pythonhosted.org/packages/24/01/186c825898477d77e2324d5360fefe622ff1d8d1963ec0554e2cada8ec77/cryptography-49.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64", size = 5298863, upload-time = "2026-06-12T20:02:24.579Z" }, + { url = "https://files.pythonhosted.org/packages/b8/7b/62cbbab75d0659865bf0273790031544a0b16c8072d258f9428dcd8190dc/cryptography-49.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21", size = 4735983, upload-time = "2026-06-12T20:01:50.14Z" }, + { url = "https://files.pythonhosted.org/packages/6c/72/3e798c064bc39e471008075d0f9bc9daf77a80879c092e4a8e170c585ed4/cryptography-49.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9", size = 4334173, upload-time = "2026-06-12T20:01:44.743Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ee/6fca21d1ac73e06f8bef71940abfd4d2f6472b4bca284d770f32bd4086f6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc", size = 4697298, upload-time = "2026-06-12T20:02:20.918Z" }, + { url = "https://files.pythonhosted.org/packages/67/d0/a5fcd3515f0bae49a7b6d0413cc1bdccdcc1fc0047037a0d480642cdc5d6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8", size = 5254338, upload-time = "2026-06-12T20:02:22.737Z" }, + { url = "https://files.pythonhosted.org/packages/a0/84/84fe36f19caf857d61cb7fc9c63035a47ffabd84ea12d1d393148efa3615/cryptography-49.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36", size = 4735650, upload-time = "2026-06-12T20:02:41.389Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a0/db537264e234f7273a73ec020873d6d6b39dfd8a53db78b550ca8320440e/cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e", size = 4834820, upload-time = "2026-06-12T20:01:51.847Z" }, + { url = "https://files.pythonhosted.org/packages/93/77/8df9eb486495979bccecd1062e2eaf435250e84437040295b57d09048b0b/cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b", size = 4967968, upload-time = "2026-06-12T20:02:12.524Z" }, +] + +[[package]] +name = "cs-util" +version = "0.2.2" +source = { git = "https://github.com/CosmoStat/cs_util.git?rev=develop#bedcdbf50ac57fc4d0b72a62b251373cd1807824" } +dependencies = [ + { name = "astropy", marker = "sys_platform == 'linux'" }, + { name = "camb", marker = "sys_platform == 'linux'" }, + { name = "datetime", marker = "sys_platform == 'linux'" }, + { name = "healpy", marker = "sys_platform == 'linux'" }, + { name = "healsparse", marker = "sys_platform == 'linux'" }, + { name = "keyring", marker = "sys_platform == 'linux'" }, + { name = "matplotlib", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "pyccl", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "swig", marker = "sys_platform == 'linux'" }, + { name = "vos", marker = "sys_platform == 'linux'" }, +] + +[[package]] +name = "cycler" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/95/a3dbbb5028f35eafb79008e7522a75244477d2838f38cbb722248dabc2a8/cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c", size = 7615, upload-time = "2023-10-07T05:32:18.335Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, +] + +[[package]] +name = "dask" +version = "2026.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click", marker = "sys_platform == 'linux'" }, + { name = "cloudpickle", marker = "sys_platform == 'linux'" }, + { name = "fsspec", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "partd", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "toolz", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/b5/aa50877159f1efd65a744107e0662a15d2b80fa0c427d980313367f49493/dask-2026.7.0.tar.gz", hash = "sha256:b039970642ce97a063070bae2763dada8aae27e3cbd8ff6d894072f920a1e252", size = 11548914, upload-time = "2026-07-06T16:58:36.496Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/aa/ceeb81d9dc886d445416310f0eff5c3978f20224b4143e96e36b7e85b011/dask-2026.7.0-py3-none-any.whl", hash = "sha256:68def8b467529ab8425ef9fcc6735dcaefaaf715d5e19ae219485f99a394dbef", size = 1496878, upload-time = "2026-07-06T16:58:34.597Z" }, +] + +[package.optional-dependencies] +array = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] + +[[package]] +name = "dask-image" +version = "2026.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dask", extra = ["array"], marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "pims", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "tifffile", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/49/e592a13a5e1efdcdb8f1faab7c4e309c61648792e276f9ced5fb79381b33/dask_image-2026.5.0.tar.gz", hash = "sha256:ed6b462277e691b2c12b0890ba801a0f9a00cc1894b0aa71b195a7a1419b2b00", size = 80457, upload-time = "2026-05-27T14:05:57.383Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/5b/15d6d6ff8697b188787609be059fe4f07f99fc00f43f68e9e1540fa8733e/dask_image-2026.5.0-py3-none-any.whl", hash = "sha256:acf86cd7f0f1e97804198d30b7cc931efd29f9dec86c65ec004b50405c3f5227", size = 43814, upload-time = "2026-05-27T14:05:56.237Z" }, +] + +[[package]] +name = "datetime" +version = "6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytz", marker = "sys_platform == 'linux'" }, + { name = "zope-interface", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/77/32/decbfd165e9985ba9d8c2d34a39afe5aeba2fc3fe390eb6e9ef1aab98fa8/datetime-6.0.tar.gz", hash = "sha256:c1514936d2f901e10c8e08d83bf04e6c9dbd7ca4f244da94fec980980a3bc4d5", size = 64167, upload-time = "2025-11-25T08:00:34.586Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/7a/ea0f3e3ea74be36fc7cf54f966cde732a3de72697983cdb5646b0a4dacde/datetime-6.0-py3-none-any.whl", hash = "sha256:d19988f0657a4e72c9438344157254a8dcad6aea8cd5ae70a5d1b5a75e5dc930", size = 52637, upload-time = "2025-11-25T08:00:33.077Z" }, +] + +[[package]] +name = "debugpy" +version = "1.8.21" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/aa/12037145b7a56eaa5b29b41872f7a21b538e807e13f32c4d3c46e59be084/debugpy-1.8.21.tar.gz", hash = "sha256:a3c53278e84c94e11bd87c53970ec391d1a67396c8b22609fcac576520e611a6", size = 1697577, upload-time = "2026-06-01T19:30:35.156Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/09/59324b903599031ff9faaec1758292409f6561a0ec2492fe4b703327705a/debugpy-1.8.21-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:c193d474f0a211191f2b4449d2d06157c689013035bd952f3b617e0ef422b176", size = 3968900, upload-time = "2026-06-01T19:30:52.341Z" }, + { url = "https://files.pythonhosted.org/packages/48/57/412421516afc3055fa577516f00beec3d663f9b0ab330639547ae6c57720/debugpy-1.8.21-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:ecbd158386c31ffe71d46f72d44d56e66331ab9b16cad649156d514368f23ab2", size = 3962096, upload-time = "2026-06-01T19:30:59.235Z" }, + { url = "https://files.pythonhosted.org/packages/8c/2d/6e7ec524984a1702777868de49a4c53202bddac2a432a76a093469587750/debugpy-1.8.21-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:3d6922439bf33fd38a3e2c447869ebc7b97da5cd3d329ff1ef9bc06c4903437e", size = 3927115, upload-time = "2026-06-01T19:31:05.863Z" }, + { url = "https://files.pythonhosted.org/packages/95/51/67e7cf11a53e40694f720457d5b3a1cdaaa3d5a9a633e482f225456b93ff/debugpy-1.8.21-py2.py3-none-any.whl", hash = "sha256:b1e37d333663c8851516a47364ef473da127f9caebe4417e6df6f5825a7e9a92", size = 5352888, upload-time = "2026-06-01T19:31:25.186Z" }, +] + +[[package]] +name = "decorator" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/60/8b/32f9823da46cde7df2087faa08cd98d01b908f8dcab982cdba9c84e85355/decorator-5.3.1.tar.gz", hash = "sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82", size = 58084, upload-time = "2026-05-18T06:03:28.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365, upload-time = "2026-05-18T06:03:26.517Z" }, +] + +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + +[[package]] +name = "deprecated" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, +] + +[[package]] +name = "dill" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, +] + +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + +[[package]] +name = "docstring-parser" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, +] + +[[package]] +name = "docutils" +version = "0.22.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/b6/03bb70946330e88ffec97aefd3ea75ba575cb2e762061e0e62a213befee8/docutils-0.22.4.tar.gz", hash = "sha256:4db53b1fde9abecbb74d91230d32ab626d94f6badfc575d6db9194a49df29968", size = 2291750, upload-time = "2025-12-18T19:00:26.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, +] + +[[package]] +name = "donfig" +version = "0.8.1.post1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, +] + +[[package]] +name = "dpath" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/ce/e1fd64d36e4a5717bd5e6b2ad188f5eaa2e902fde871ea73a79875793fc9/dpath-2.2.0.tar.gz", hash = "sha256:34f7e630dc55ea3f219e555726f5da4b4b25f2200319c8e6902c394258dd6a3e", size = 28266, upload-time = "2024-06-12T22:08:03.686Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/d1/8952806fbf9583004ab479d8f58a9496c3d35f6b6009ddd458bdd9978eaf/dpath-2.2.0-py3-none-any.whl", hash = "sha256:b330a375ded0a0d2ed404440f6c6a715deae5313af40bbb01c8a41d891900576", size = 17618, upload-time = "2024-06-12T22:08:01.881Z" }, +] + +[[package]] +name = "ducc0" +version = "0.41.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cf/e7/849dd1a079b994184fa53905d47278f86d0dd3dcfb6ff2b0e3eb7d6a1c1b/ducc0-0.41.0.tar.gz", hash = "sha256:bac084745bbdb243482a4aec3ecc857bdd46faec298c67e48bc0bb90350dbabd", size = 353129, upload-time = "2026-03-26T18:56:04.669Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/8e/3a28ebdef748b9b59c22d9a801116a9ea073d0ce66d9a738bf5db6c29e0c/ducc0-0.41.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94718e321194313232ae6ba73c528a3847d5eae212a9fc1ef67471e7113e6492", size = 4563683, upload-time = "2026-03-26T18:54:28.643Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/5132178a63fbbbc8868c10b07cbacee138f248ac6f319040537e5a45b3a9/ducc0-0.41.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c17565f332745f08dd012334e07a4dcbb591c9591c2b14f51a1d57ad40018f53", size = 5601347, upload-time = "2026-03-26T18:54:34.054Z" }, + { url = "https://files.pythonhosted.org/packages/17/97/191420d245383304f20f96254774f14b96239e52a0eac44cab93c870941e/ducc0-0.41.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ca66a1806983545460c5447f053f15f6d9f8037b16b128b2a925f0410bcee06d", size = 4563741, upload-time = "2026-03-26T18:54:47.684Z" }, + { url = "https://files.pythonhosted.org/packages/1f/e3/a8783330d7225def7cafe33ed9a2fd7e29eff8f91e7312306037048199a2/ducc0-0.41.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d058b71274f9d60d6edae70cc31d6f5a72e29b0ecc7229725905682566d1e605", size = 5602698, upload-time = "2026-03-26T18:54:53.816Z" }, + { url = "https://files.pythonhosted.org/packages/67/ca/fce0c5395faca711cd4b7203bc001e820288608e6957f81ee183218509ed/ducc0-0.41.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e8c2c49fbfae0c6f7f4ec759a46b936e00c0d4398eeba8f6cfafd562b7dbe55", size = 4563768, upload-time = "2026-03-26T18:55:05.598Z" }, + { url = "https://files.pythonhosted.org/packages/47/a4/70b5a521697c7482ceca8b00f4406bef5a111d37142a719b2fa4942252f6/ducc0-0.41.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:13f98aefcce65b3416e86fa998a0a81ee0ebd9f0feef2588f44c6694b2a50bf0", size = 5604100, upload-time = "2026-03-26T18:55:11.133Z" }, + { url = "https://files.pythonhosted.org/packages/0d/88/022d053b65c9acd69b2de2a9400afd3916ec3a409f7c5776dc906f86b228/ducc0-0.41.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:583d59d5eb4f23c3eb7c6f8a2b6f32da72e3868e9aa711598978a76a25425f96", size = 4579841, upload-time = "2026-03-26T18:55:19.689Z" }, + { url = "https://files.pythonhosted.org/packages/9a/91/428eca0cc8a1142b9952bef02c06bfe54526d730dd070de873424ab8c0bf/ducc0-0.41.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a3a71fd23661ddd6a7f6b44433e778d35a25b3f1492c9239217cdfedbe8178b1", size = 5613706, upload-time = "2026-03-26T18:55:24.971Z" }, +] + +[[package]] +name = "emcee" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/53/1045ee878cb24281387079f8ee4f0ade1622c6aae1ed1fd91a53e4fa5b19/emcee-3.1.6.tar.gz", hash = "sha256:11af4daf6ab8f9ca69681e3c29054665db7bbd87fd4eb8e437d2c3a1248c637d", size = 2871117, upload-time = "2024-04-19T10:03:19.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/ef/2196b9bf88ffa1bde45853c72df021fbd07a8fa91a0f59a22d14a050dc04/emcee-3.1.6-py2.py3-none-any.whl", hash = "sha256:f2d63752023bdccf744461450e512a5b417ae7d28f18e12acd76a33de87580cb", size = 47351, upload-time = "2024-04-19T10:03:17.522Z" }, +] + +[[package]] +name = "executing" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/28/c14e053b6762b1044f34a13aab6859bbf40456d37d23aa286ac24cfd9a5d/executing-2.2.1.tar.gz", hash = "sha256:3632cc370565f6648cc328b32435bd120a1e4ebb20c77e3fdde9a13cd1e533c4", size = 1129488, upload-time = "2025-09-01T09:48:10.866Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, +] + +[[package]] +name = "fastjsonschema" +version = "2.21.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/b5/23b216d9d985a956623b6bd12d4086b60f0059b27799f23016af04a74ea1/fastjsonschema-2.21.2.tar.gz", hash = "sha256:b1eb43748041c880796cd077f1a07c3d94e93ae84bba5ed36800a33554ae05de", size = 374130, upload-time = "2025-08-14T18:49:36.666Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/a8/20d0723294217e47de6d9e2e40fd4a9d2f7c4b6ef974babd482a59743694/fastjsonschema-2.21.2-py3-none-any.whl", hash = "sha256:1c797122d0a86c5cace2e54bf4e819c36223b552017172f32c5c024a6b77e463", size = 24024, upload-time = "2025-08-14T18:49:34.776Z" }, +] + +[[package]] +name = "fitsio" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/6b/5658778e070faaa194f1a1133527cedfa948d173daed29d61f25c5487ab6/fitsio-1.4.1.tar.gz", hash = "sha256:2b2bf0bfd37790114b44283931c59a14b1fec25a653bf65c4fc555b8d3b24de9", size = 4148217, upload-time = "2026-07-10T21:21:31.585Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/a0/86e2bde46aa5559b7b9544208def3045675d055f0d132b235ff325279a9b/fitsio-1.4.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:de7ac1df97ce68626dac1da5014e6b396cff2789d1c617c36df03d6bc6d98a2d", size = 881494, upload-time = "2026-07-10T21:21:09.496Z" }, + { url = "https://files.pythonhosted.org/packages/a5/53/f7f644b812e572c57015ef5215f982922d9cdac8d239549c136ba48f2850/fitsio-1.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:34df9129decab3ae85377601668b82442159ed704800bd1ed047428bc532c96d", size = 810043, upload-time = "2026-07-10T21:21:11.254Z" }, + { url = "https://files.pythonhosted.org/packages/83/5d/d71993140df9f939e81d2a5531f26d15cbd700855348a756921b0ad24235/fitsio-1.4.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:161806de2bb9971e271379558616d2cef795ce9590923270ec3bbdadea0d0b2f", size = 913505, upload-time = "2026-07-10T21:21:16.695Z" }, + { url = "https://files.pythonhosted.org/packages/ef/ae/e89778256e0aee3e5b51a620cdf9d91ac89f348504045461bee28006275d/fitsio-1.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5ed27e5b8aaf1ae51718c6624d00fec1d6054fddde8d602eaea3bb6686bcfb0a", size = 815241, upload-time = "2026-07-10T21:21:18.018Z" }, + { url = "https://files.pythonhosted.org/packages/7c/47/7f64a60d883a4b69a0a15c1ec60ba9313d8e774af83ac2b00f57d511b76d/fitsio-1.4.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:1ab4eadb13caf362630d45bce7cf7e57e407d3c971f361f639ca9c316f460bb2", size = 913536, upload-time = "2026-07-10T21:21:22.556Z" }, + { url = "https://files.pythonhosted.org/packages/de/46/9f45b3c8bd371057f1333aec8f92879cb786fe9ed05b6b0c2643fd228558/fitsio-1.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2627eeec3dff4b692402babb88aec3cccbf73a3ee22d19b4accfe76491bd67a8", size = 815239, upload-time = "2026-07-10T21:21:23.61Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3b/9a6d92637db9a603a18681f60689c30208f2485016ed91280e42b927df57/fitsio-1.4.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:5f499149738fdce6afa3324a76fd65a762082647d8b1d30a356dc002dd4a9178", size = 928341, upload-time = "2026-07-10T21:21:28.075Z" }, + { url = "https://files.pythonhosted.org/packages/0a/fa/eeb997744bb1c2c9308dab9b492bcdb1843e4eb3dc5d1697d2f0a312c8b7/fitsio-1.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c2a0dba7a868b4b1716d7468431fda51bc89ee8d8f2a643ac28f9200d99d2e2b", size = 815548, upload-time = "2026-07-10T21:21:29.202Z" }, +] + +[[package]] +name = "flt" +version = "2026.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/66/538256582e373d81a944c0ee3a12fd0e4dd0f39d10136353c4ce914d7061/flt-2026.1.tar.gz", hash = "sha256:4bf2818d4a9137f5f715fd2ff13acfbf0175639373155cd89ea2b788edf00093", size = 12817, upload-time = "2026-03-20T22:47:29.754Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/0f/9911ee41dbb2eb043d6252aa8f9bade8995f7b7ee01406e049088c96acf1/flt-2026.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2962fd252df62d77bb2861859173ffc7cc436921a234a52c17eab7839bcce588", size = 479324, upload-time = "2026-03-20T22:46:55.38Z" }, + { url = "https://files.pythonhosted.org/packages/7c/86/c525fef761b2bf9cd961fa6e06a4e255ea9eca529d37e22098d8baa900f0/flt-2026.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b73c4aac4874808dc66e2f7a3cbcdd20a65d0ef14d6f2d378a79930ff3d0bad", size = 478225, upload-time = "2026-03-20T22:46:56.902Z" }, + { url = "https://files.pythonhosted.org/packages/1e/14/9beb3e8a64536741b6f06f62324305171391f9b6d84b804c42e0519f472e/flt-2026.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e02e75d2c8d6441cd7de2d7a13e62bb8feb687cee2d70c4bd57c43c5c78dc630", size = 483678, upload-time = "2026-03-20T22:47:02.783Z" }, + { url = "https://files.pythonhosted.org/packages/95/05/086fd21633043fe19dbe972cab79a680199286c57ba66a88f899c1f51dc7/flt-2026.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ef91c6dd6e453062b5f5d3cbb0c489bd8b3ee1440ad9e5c940d22921664bb69a", size = 483418, upload-time = "2026-03-20T22:47:04.284Z" }, + { url = "https://files.pythonhosted.org/packages/f1/93/a24b0937fa882fc513d399713d6780262ac5beacdef16a3638370669c960/flt-2026.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28c43c464fffa6edbee4bcaaf07565fdd8046ef649764d5647b0de9b2faa19a7", size = 479350, upload-time = "2026-03-20T22:47:10.738Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/54d75cabcacbbfa79b3c54515167f5a3b2dcfcb7af1ec23e8df46bc59e52/flt-2026.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f011917cb43d3590c5fdb8e622a9641b5a42e5519d6fea27b4553aa8fc6c8f32", size = 480496, upload-time = "2026-03-20T22:47:11.894Z" }, + { url = "https://files.pythonhosted.org/packages/60/94/a12bc6dd3136b187dacbd5b820537c0e5507e8f63f5c7db10019eeb54394/flt-2026.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:999f1ffbea353fd77b1797ecd8b69a7bae7bb098723ae0dca51a05fc6e4df6dc", size = 480306, upload-time = "2026-03-20T22:47:17.933Z" }, + { url = "https://files.pythonhosted.org/packages/c0/7b/700fc7cfc5033090b27e14e69d9678b1973642f684cdfeffdcf08e561a98/flt-2026.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:10575dc192697bd024ab22f7948fa7f850c40ca6a036e80fe2e96afec6e5c703", size = 473443, upload-time = "2026-03-20T22:47:19.19Z" }, +] + +[[package]] +name = "fonttools" +version = "4.63.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/84/69/c97f2c18e0db87d2c7b15da1974dace76ae938f1cfa22e2727a648b7ed43/fonttools-4.63.0.tar.gz", hash = "sha256:caeb583deeb5168e694b65cda8b4ee62abedfa66cf88488734466f2366b9c4e0", size = 3597189, upload-time = "2026-05-14T12:04:30.958Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/04/0b91d8e916e92ad1fac9e4624760baf0fd5ff2ead614c2f68fb21373f03f/fonttools-4.63.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef3048ef05dbb552b89817713d9cac912e00d0fde4a3105c00d29e52e10c89af", size = 5044298, upload-time = "2026-05-14T12:03:18.085Z" }, + { url = "https://files.pythonhosted.org/packages/77/c7/2342da9830e3e9d4870305ca5d2091d2a83284f2953079b7bdd3b5e029d8/fonttools-4.63.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:58dc6bb86a78d782f00f9190ca02c119cf5bbe2807536e361e18d42019f877d8", size = 4999800, upload-time = "2026-05-14T12:03:20.161Z" }, + { url = "https://files.pythonhosted.org/packages/e6/6d/67fe16c48d7ce050979b33f47e0d28a318f02da030602e944c34f7a16ef3/fonttools-4.63.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee08ebfa58f6e1aeff5697ab9582105bb620008c1caafb681e4c557e7483027b", size = 4982666, upload-time = "2026-05-14T12:03:22.87Z" }, + { url = "https://files.pythonhosted.org/packages/f2/00/3bbab338c07c71fa56269953845e92c951a61457bbbb0f1022551ea266d9/fonttools-4.63.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:27fdc65af8da6f88b9c6121c47a464cbe359fcfff7ff6fc2d37a1f395d755b78", size = 5133598, upload-time = "2026-05-14T12:03:25.168Z" }, + { url = "https://files.pythonhosted.org/packages/8e/40/e76320afa1df918e146155ef239b1719ee266092e96f5423bfd075affba1/fonttools-4.63.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e874792a8212b44583ea02189d9e693906b2f78b261f372f95d6c563210ac1d", size = 5024840, upload-time = "2026-05-14T12:03:36.745Z" }, + { url = "https://files.pythonhosted.org/packages/ce/36/0b805d8c485f872f65a509cbe3b58a5d0d17bee855333b54a150c79d3061/fonttools-4.63.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:22135da48a348785c5e2d5d2d9d6bec5ed44adacbaeb9db12d9493bf6c6bfa68", size = 4975801, upload-time = "2026-05-14T12:03:38.833Z" }, + { url = "https://files.pythonhosted.org/packages/c8/26/2cee03d0aa083ab022da5c07aff9ed3f689da1defb81ad6917c9627896da/fonttools-4.63.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ccf41f2efdf56994d22d73bef4ced1052161958169428d06ba9724ea9e9a64be", size = 4965009, upload-time = "2026-05-14T12:03:41.494Z" }, + { url = "https://files.pythonhosted.org/packages/7e/48/cc4b66d9058c0d0982c833fad10127c4b0e9324606aafa41382295ca4102/fonttools-4.63.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9ced0bd02ac751dd6319b0da88aaef24414e3b0dbc32bb4f24944821a3741a27", size = 5105892, upload-time = "2026-05-14T12:03:43.525Z" }, + { url = "https://files.pythonhosted.org/packages/dd/87/64cfa18a7a1621d17b7f4502b2b0ed8a135a90c3db51ea590ee99043e76b/fonttools-4.63.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b2248c5decb223562f7902ff6325077a073f608ee8e33e88ad88db734eb9f49", size = 5010526, upload-time = "2026-05-14T12:03:54.647Z" }, + { url = "https://files.pythonhosted.org/packages/36/e1/a8933a72c45a87177fbde2696e0d0755c8c9062f8c077a961c6215fa27b1/fonttools-4.63.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:308f957cdeaf8abe4e5f2f124902ef405448af92c90f80e302a3b771c2e6116b", size = 4923946, upload-time = "2026-05-14T12:03:56.984Z" }, + { url = "https://files.pythonhosted.org/packages/27/60/872e6e233b8c5e8b41413796ff18b7fe479661bd40147e071b450dfad7a1/fonttools-4.63.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bf00f21eb5fb721dbaf73d1e9da6d02a1af7768f2ebcf9798be98beab8ba90f6", size = 4962489, upload-time = "2026-05-14T12:03:59.443Z" }, + { url = "https://files.pythonhosted.org/packages/30/c4/83c24f2ec38b90cfda84bf4b1a1f49df80e84a1db4e7ac6e0d41bf23bc39/fonttools-4.63.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c1aaa4b9c75798400ac043ce04d74e7830376c85095a5a6ed7cba2f17a266bf4", size = 5071870, upload-time = "2026-05-14T12:04:02.122Z" }, + { url = "https://files.pythonhosted.org/packages/ea/2b/0aa8db70f18cf52e49b4ed5ecec68547f981160bf5ded3b5aed6faa0a6f9/fonttools-4.63.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0eac00b9118c3c2f87d272e45341871c5b3066baa3c86897fa634a7c3fb59096", size = 5148649, upload-time = "2026-05-14T12:04:12.747Z" }, + { url = "https://files.pythonhosted.org/packages/7f/63/18e4369c25043096f1048e0c9915951adc4f842bd81c6b18155824d6fa99/fonttools-4.63.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51394295f1a51de8b5f30bdb1e1b9a4231536c7064ef5c6e211eec19fa36036f", size = 4932147, upload-time = "2026-05-14T12:04:14.806Z" }, + { url = "https://files.pythonhosted.org/packages/a1/3f/67f3eac2ffd8a98446c5022f8ed3864eac878a5ff7af8df4c8286dba16cc/fonttools-4.63.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9e12f105d2b6342c559c298afb674006bb2893afc7102dcf8a1b55b0486b4e40", size = 5027237, upload-time = "2026-05-14T12:04:17.675Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ba/4e6214cb38a7b04779e97bb7636de9a5c7f20af7018d03dee0b64c08510a/fonttools-4.63.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:796f27556dbe094c4824f75ca85267e4df776c79036c8441469a4df37038c196", size = 5053933, upload-time = "2026-05-14T12:04:20.818Z" }, + { url = "https://files.pythonhosted.org/packages/2c/47/c99d5268f354002ce80f8d029cd9d7d872969da1de8b93d32de4dc56d6f4/fonttools-4.63.0-py3-none-any.whl", hash = "sha256:445af2eab030a16b9171ea8bdda7ebf7d96bda2df88ee182a464252f6e05e20d", size = 1164562, upload-time = "2026-05-14T12:04:29.092Z" }, +] + +[[package]] +name = "fqdn" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/3e/a80a8c077fd798951169626cde3e239adeba7dab75deb3555716415bd9b0/fqdn-1.5.1.tar.gz", hash = "sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f", size = 6015, upload-time = "2021-03-11T07:16:29.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl", hash = "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014", size = 9121, upload-time = "2021-03-11T07:16:28.351Z" }, +] + +[[package]] +name = "fsspec" +version = "2026.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/10/a1/ae4e3e5003468d6391d2c77b6fa1cd73bd5d13511d81c642d7b28ac90ed4/fsspec-2026.6.0.tar.gz", hash = "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a", size = 313646, upload-time = "2026-06-16T01:57:28.105Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" }, +] + +[[package]] +name = "future" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/b2/4140c69c6a66432916b26158687e821ba631a4c9273c474343badf84d3ba/future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05", size = 1228490, upload-time = "2024-02-21T11:52:38.461Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/71/ae30dadffc90b9006d77af76b393cb9dfbfc9629f339fc1574a1c52e6806/future-1.0.0-py3-none-any.whl", hash = "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216", size = 491326, upload-time = "2024-02-21T11:52:35.956Z" }, +] + +[[package]] +name = "getdist" +version = "1.6.1" +source = { git = "https://github.com/benabed/getdist.git?rev=113cd22a9a0d013b6f72fe734be81f260f3d3be5#113cd22a9a0d013b6f72fe734be81f260f3d3be5" } +dependencies = [ + { name = "matplotlib", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, +] + +[[package]] +name = "gitdb" +version = "4.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smmap", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, +] + +[[package]] +name = "gitpython" +version = "3.1.50" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gitdb", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, +] + +[[package]] +name = "glass" +version = "2025.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cosmology", marker = "sys_platform == 'linux'" }, + { name = "healpix", marker = "sys_platform == 'linux'" }, + { name = "healpy", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "transformcl", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a5/d6/26cd58e75834671259f26b9287a584d726e6023da35ddf7ca3b7f2c393fb/glass-2025.1.tar.gz", hash = "sha256:7b1aa2394e16010f7f1b4243f49e7e12d7a4dd28fcbf3e3f7cf25ce4905a8615", size = 48533, upload-time = "2025-02-21T18:43:48.046Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/e7/3e89bc5147a3e84245c783098afffbbacbcdf5678ba1aacc2c9c035434da/glass-2025.1-py3-none-any.whl", hash = "sha256:d7919a7d19e05ab8da4e52dbfbcfb5bd1ed2b72e0cc3afde39ec94ced22883c5", size = 47185, upload-time = "2025-02-21T18:43:46.649Z" }, +] + +[[package]] +name = "glass-ext-camb" +version = "2023.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "camb", marker = "sys_platform == 'linux'" }, + { name = "glass", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b7/1a/18bb47c0f7cc267c2f8c95273055b82b1975e4dccbfea6051e82a4d1f513/glass.ext.camb-2023.6.tar.gz", hash = "sha256:f7b6e521b2d45d3b68400353ce6ab1747c772631832a257567de028aa2060509", size = 2989, upload-time = "2023-06-30T22:27:31.511Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/50/e92d9db0d26966bad6c4c441822c345784af592767747ee563a65ee24450/glass.ext.camb-2023.6-py3-none-any.whl", hash = "sha256:f46015ecedf87cffabf008d21ac06be5d7a55557027f12c54e258ee11b85c44a", size = 3634, upload-time = "2023-06-30T22:27:30.171Z" }, +] + +[[package]] +name = "google-crc32c" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, + { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, + { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, + { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, + { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, +] + +[[package]] +name = "greenlet" +version = "3.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e2/f1/fbbfef6af0bad0548f09bc28948ea3c275b4edb19e17fc5ca9900a6a634d/greenlet-3.5.3.tar.gz", hash = "sha256:a61efc018fd3eb317eeca31aba90ee9e7f26f22884a79b6c6ec715bf71bb62f1", size = 200270, upload-time = "2026-06-26T19:28:24.832Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/73/815dd90131c1b71ebdf53dbc7c276cafec2a1173b97559f97aba72724a87/greenlet-3.5.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efa9f765dd09f9d0cdac651ffdf631ee59ec5dc6ee7a73e0c012ba9c52fbdf5b", size = 604761, upload-time = "2026-06-26T19:07:10.114Z" }, + { url = "https://files.pythonhosted.org/packages/9f/57/079cfe76bcef36b153b25607ee91c6fcb58f17f8b23c86bbbeabe0c88d72/greenlet-3.5.3-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7faba15ac005376e02a0384504e0243be3370ce010296a44a820feb342b505ab", size = 617044, upload-time = "2026-06-26T19:10:07.25Z" }, + { url = "https://files.pythonhosted.org/packages/37/87/b4d095775a3fb1bcafbb483fc206b27ebb785724c83051447737085dc54e/greenlet-3.5.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:87142215824be6ac05e2e8e2786eec307ccbc27c36723c3881959df654af6861", size = 614244, upload-time = "2026-06-26T18:32:17.594Z" }, + { url = "https://files.pythonhosted.org/packages/8a/70/7559b609683650fa2b95b8ab84b4ab0b26556a635d19675e12aa832d826d/greenlet-3.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:215275b1b49320987352e6c1b054acca0064f965a2c66992bed9a6f7d913f149", size = 1574210, upload-time = "2026-06-26T19:09:03.077Z" }, + { url = "https://files.pythonhosted.org/packages/ae/73/be55392074c60fc37655ca40fa6022457bfbf6718e9e342a7b0b41f96dd2/greenlet-3.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6b1b0eed82364b0e32c4ea0f221452d33e6bb17ae094d9f72aed9851812747ea", size = 1638627, upload-time = "2026-06-26T18:31:44.748Z" }, + { url = "https://files.pythonhosted.org/packages/d6/fa/5401ac78021c826a25b6dde0c705e0a8f29b617509f9185a31dac15fbe1b/greenlet-3.5.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2d185dd1621757e70c3861cceffd5317ab4e7ed7eb09c82994828468527ade5", size = 607435, upload-time = "2026-06-26T19:07:11.412Z" }, + { url = "https://files.pythonhosted.org/packages/e9/76/1dc144a2e56e65d36405078ed774224375ea520a1870a6e46e08bb4ac7bf/greenlet-3.5.3-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1c514a468149bf8fbbab874188a3535cd8a48a3e353eb53a3d424296f8dbacd3", size = 619787, upload-time = "2026-06-26T19:10:08.396Z" }, + { url = "https://files.pythonhosted.org/packages/bf/87/c298cee62df1de4ad7fec32abda73526cff347fd143a6ed4ac369246668a/greenlet-3.5.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:915f887cf2682b66419b879423a2e072634aa7b7dce6f3ada4957cfced3f1e9a", size = 616786, upload-time = "2026-06-26T18:32:19.128Z" }, + { url = "https://files.pythonhosted.org/packages/9e/2e/e6f009885ed0705ccf33fe0583c117cfd03cde77e31a596dd5785a30762b/greenlet-3.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:766cfd421c13e450feb340cd472a3ed9957d438727b7b4593ad7c76c5d2b0deb", size = 1574316, upload-time = "2026-06-26T19:09:04.273Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fe/43fd110b01e40da0adb7c90ac7ea744bef2d43dca00de5095fd2351c2a68/greenlet-3.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2ecda9ec22edf38fa389369eaed8c3d37c05f3c54e69f69438dbb2cc1de1458b", size = 1638614, upload-time = "2026-06-26T18:31:46.297Z" }, + { url = "https://files.pythonhosted.org/packages/82/2f/146d218299046a43d1f029fd544b3d110d0f175a09c715c7e8da4a4a345d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df0a0628d1597eb0897b62f55d1343f772405fd25f3b2a796c76874b0c2e22e8", size = 654096, upload-time = "2026-06-26T19:07:12.71Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cc/04738cafb3f45fa991ea44f9de94c47dcec964f5a972300988a6751f49d9/greenlet-3.5.3-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ebd933a6adabc298bab47731a130fe6bfb888bd934eee37810f151159544540d", size = 666304, upload-time = "2026-06-26T19:10:09.503Z" }, + { url = "https://files.pythonhosted.org/packages/ce/aa/4e0dad5e605c270c784ab911c43da6adb136ccd4d81180f763ca429a723d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b9d501b40e80b70e32323c799dd9b420a5577a9601469d362ae1ffb690f3a7c", size = 663635, upload-time = "2026-06-26T18:32:20.802Z" }, + { url = "https://files.pythonhosted.org/packages/d1/50/13efdbea246fe3d3b735e191fec08fb50809f53cd2383ebe123d0809e44b/greenlet-3.5.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a1fad1d11e7d6aab184107baa8e4ece11ccba3ec9599cd7efa5ff4d70d43256a", size = 1621252, upload-time = "2026-06-26T19:09:05.647Z" }, + { url = "https://files.pythonhosted.org/packages/f7/22/c0a336ae4a1410fd5f5121098e5bfbf1865f64c5ef80b4b5412886c4a332/greenlet-3.5.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fad5aec764399f1b5cc347ad250a59660f20c8f8888ea6bae1f93b769cce1154", size = 1684824, upload-time = "2026-06-26T18:31:47.738Z" }, + { url = "https://files.pythonhosted.org/packages/00/92/715c44721abe2b4d1ae9abde4179411868a5bff312479f54e105d372f131/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19131729ae0ddc3c2e1ef85e650169b5e37ee32e400f215f78b94d7b0d567310", size = 653382, upload-time = "2026-06-26T19:07:14.209Z" }, + { url = "https://files.pythonhosted.org/packages/a0/83/37a10372a1090a6624cca8e74c12df1a36c2dc36429ed0255b7fb1aeee23/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1540dd8e5fc2a5aec40fbb98ef8e149fa47c89a4b4a1cf2575a14d3d1869d7a8", size = 659401, upload-time = "2026-06-26T19:10:10.876Z" }, + { url = "https://files.pythonhosted.org/packages/db/e2/d1509cad4207da559cc42986ecdd8fc67ad0d1bba2bf03023c467fd5e0f3/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e81fa194a1d20967877bdf9c7794db2bc99063e5be36aee710c08f04c5bb087f", size = 656969, upload-time = "2026-06-26T18:32:22.272Z" }, + { url = "https://files.pythonhosted.org/packages/86/7d/eaf70de20aadca3a5884aec58362861c64ce45e7b277f47ed026926a3b89/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:55cf4d777485d43110e47133cbba6d74a8885a87ec1227ef0267f9ee80c5aa21", size = 1617822, upload-time = "2026-06-26T19:09:06.893Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f9/414d38fc400ae4350d4185eaad1827676f7cf5287b9136e0ed1cbbe20a7f/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:12a248ba75f6a9a236375f52296c498c89ff1d8badf32deb9eca7abd5853f7da", size = 1677983, upload-time = "2026-06-26T18:31:49.396Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ca/69db42d447a1378043e2c8f19c09cbbd1263371505053c496b49066d3d16/greenlet-3.5.3-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78dbef602fda6d97d957eb7937f70c9ce9e9527330347f8f6b6f9e554a9e7a47", size = 659747, upload-time = "2026-06-26T19:07:15.565Z" }, + { url = "https://files.pythonhosted.org/packages/a8/0b/af7ac2ef8dd41e3da1a40dda6305c23b9a03e13ba975ec916357b50f8575/greenlet-3.5.3-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f73857adb8fee13fa56c172bd11262f888c0c648f9fea113e777bb2c7904a81", size = 670419, upload-time = "2026-06-26T19:10:12.293Z" }, + { url = "https://files.pythonhosted.org/packages/51/1e/1d51640cacbfc455dbe9f9a9f594c49e4e244f63b9971a2f4764e46cc53d/greenlet-3.5.3-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:232fec92e823addaf02d9472cf7381e24a1d046a6ced1103c5caa4c21b9dfc1d", size = 668787, upload-time = "2026-06-26T18:32:24.298Z" }, + { url = "https://files.pythonhosted.org/packages/21/66/4030d5b0b5894500023f003bb054d9bb354dfbd1e186c3a296759172f5f5/greenlet-3.5.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2421c3564da9429d5586d46ca31ebb26516b5498a802cf65c041a8e8a8980d34", size = 1626305, upload-time = "2026-06-26T19:09:08.281Z" }, + { url = "https://files.pythonhosted.org/packages/0e/50/5221371c7550108dfa3c378debc41d032aa9c78e89abb01d8011cfc93289/greenlet-3.5.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e0f0d160f0b2e558e6c75f7930967183255dc9735e5f5b8cae58ee09c9576d8b", size = 1688631, upload-time = "2026-06-26T18:31:51.278Z" }, + { url = "https://files.pythonhosted.org/packages/57/66/b3bfae3e220a9b63ea539a0eea681800c69ab1aada757eae8789f183e7ce/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:629b614d2b786e89c50440e246f33eea78f58a962d0bdbbcc809e6d13605903f", size = 657221, upload-time = "2026-06-26T19:07:16.973Z" }, + { url = "https://files.pythonhosted.org/packages/7b/81/b6d4d73a709684fc77e7fa034d7c2fe82cffa9fc920fadcaa659c2626213/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2b2e857ae16f5f72142edf75f9f176fe7526ba19a2841df1420516f83831c9f2", size = 663226, upload-time = "2026-06-26T19:10:13.723Z" }, + { url = "https://files.pythonhosted.org/packages/f5/07/e210b02b589f16e74ff48b730690e4a34ffe984219fce4f3c1a0e7ec8545/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e515757e2e36bcbf1fad09a46e1557e8b1ae1797d4b44d09da7deed88ad28608", size = 660802, upload-time = "2026-06-26T18:32:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/eb/2e/5303eb3fa06bca089060f479707182a93e360683bc252acf846c3090d34e/greenlet-3.5.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b363d46ed1ea431825fdb01471bb024fc08399bad1572a616e853c7684415adb", size = 1622157, upload-time = "2026-06-26T19:09:09.527Z" }, + { url = "https://files.pythonhosted.org/packages/54/70/50de47a488f14df260b50ae34fb5d56016e308b098eab02c878b5223c26a/greenlet-3.5.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e44da2f5bbdaabaf7d80b73dbb430c7035771e9f244e3c8b769715c9d8fa0a16", size = 1681159, upload-time = "2026-06-26T18:31:52.986Z" }, +] + +[[package]] +name = "gsl" +version = "0.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/36/a8a99ab67918beb6ee8722fb6c27512796a5ff8ed4c10036f787eef51569/gsl-0.0.3.tar.gz", hash = "sha256:85a6071e00a284510c963b97ab74ef8a3e7d1786c5f52dc2ff444cbb7d94ee3c", size = 44404, upload-time = "2018-05-30T09:02:51.203Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/1e/14d4c4657bc7130b900c65033861ee0db508d2d9206f4223f5da7c855acc/gsl-0.0.3-py3-none-any.whl", hash = "sha256:081c77375bf65cb759a3e5129275b8806be18648452f50f7c420f40dac60529a", size = 39432, upload-time = "2018-05-30T09:02:49.93Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "h2" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hpack", marker = "sys_platform == 'linux'" }, + { name = "hyperframe", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" }, +] + +[[package]] +name = "h5py" +version = "3.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/33/acd0ce6863b6c0d7735007df01815403f5589a21ff8c2e1ee2587a38f548/h5py-3.16.0.tar.gz", hash = "sha256:a0dbaad796840ccaa67a4c144a0d0c8080073c34c76d5a6941d6818678ef2738", size = 446526, upload-time = "2026-03-06T13:49:08.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/84/06281c82d4d1686fde1ac6b0f307c50918f1c0151062445ab3b6fa5a921d/h5py-3.16.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ff24039e2573297787c3063df64b60aab0591980ac898329a08b0320e0cf2527", size = 5198852, upload-time = "2026-03-06T13:48:07.482Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e9/1a19e42cd43cc1365e127db6aae85e1c671da1d9a5d746f4d34a50edb577/h5py-3.16.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:dfc21898ff025f1e8e67e194965a95a8d4754f452f83454538f98f8a3fcb207e", size = 5405250, upload-time = "2026-03-06T13:48:09.628Z" }, + { url = "https://files.pythonhosted.org/packages/b7/8e/9790c1655eabeb85b92b1ecab7d7e62a2069e53baefd58c98f0909c7a948/h5py-3.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:698dd69291272642ffda44a0ecd6cd3bda5faf9621452d255f57ce91487b9794", size = 5190108, upload-time = "2026-03-06T13:48:11.26Z" }, + { url = "https://files.pythonhosted.org/packages/51/d7/ab693274f1bd7e8c5f9fdd6c7003a88d59bedeaf8752716a55f532924fbb/h5py-3.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2b2c02b0a160faed5fb33f1ba8a264a37ee240b22e049ecc827345d0d9043074", size = 5419216, upload-time = "2026-03-06T13:48:13.322Z" }, + { url = "https://files.pythonhosted.org/packages/da/1e/6172269e18cc5a484e2913ced33339aad588e02ba407fafd00d369e22ef3/h5py-3.16.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:099f2525c9dcf28de366970a5fb34879aab20491589fa89ce2863a84218bb524", size = 5193860, upload-time = "2026-03-06T13:48:24.071Z" }, + { url = "https://files.pythonhosted.org/packages/bd/98/ef2b6fe2903e377cbe870c3b2800d62552f1e3dbe81ce49e1923c53d1c5c/h5py-3.16.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9300ad32dea9dfc5171f94d5f6948e159ed93e4701280b0f508773b3f582f402", size = 5400417, upload-time = "2026-03-06T13:48:25.728Z" }, + { url = "https://files.pythonhosted.org/packages/bc/81/5b62d760039eed64348c98129d17061fdfc7839fc9c04eaaad6dee1004e4/h5py-3.16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:171038f23bccddfc23f344cadabdfc9917ff554db6a0d417180d2747fe4c75a7", size = 5185214, upload-time = "2026-03-06T13:48:27.436Z" }, + { url = "https://files.pythonhosted.org/packages/28/c4/532123bcd9080e250696779c927f2cb906c8bf3447df98f5ceb8dcded539/h5py-3.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7e420b539fb6023a259a1b14d4c9f6df8cf50d7268f48e161169987a57b737ff", size = 5414598, upload-time = "2026-03-06T13:48:29.49Z" }, + { url = "https://files.pythonhosted.org/packages/58/a5/4964bc0e91e86340c2bbda83420225b2f770dcf1eb8a39464871ad769436/h5py-3.16.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:e2c04d129f180019e216ee5f9c40b78a418634091c8782e1f723a6ca3658b965", size = 5198886, upload-time = "2026-03-06T13:48:38.879Z" }, + { url = "https://files.pythonhosted.org/packages/f1/16/d905e7f53e661ce2c24686c38048d8e2b750ffc4350009d41c4e6c6c9826/h5py-3.16.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e4360f15875a532bc7b98196c7592ed4fc92672a57c0a621355961cafb17a6dd", size = 5404883, upload-time = "2026-03-06T13:48:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f2/58f34cb74af46d39f4cd18ea20909a8514960c5a3e5b92fd06a28161e0a8/h5py-3.16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3fae9197390c325e62e0a1aa977f2f62d994aa87aab182abbea85479b791197c", size = 5192039, upload-time = "2026-03-06T13:48:43.117Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ca/934a39c24ce2e2db017268c08da0537c20fa0be7e1549be3e977313fc8f5/h5py-3.16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:43259303989ac8adacc9986695b31e35dba6fd1e297ff9c6a04b7da5542139cc", size = 5421526, upload-time = "2026-03-06T13:48:44.838Z" }, + { url = "https://files.pythonhosted.org/packages/f5/a1/af87f64b9f986889884243643621ebbd4ac72472ba8ec8cec891ac8e2ca1/h5py-3.16.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:0f456f556e4e2cebeebd9d66adf8dc321770a42593494a0b6f0af54a7567b242", size = 5074061, upload-time = "2026-03-06T13:48:54.089Z" }, + { url = "https://files.pythonhosted.org/packages/cc/d0/146f5eaff3dc246a9c7f6e5e4f42bd45cc613bce16693bcd4d1f7c958bf5/h5py-3.16.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:3e6cb3387c756de6a9492d601553dffea3fe11b5f22b443aac708c69f3f55e16", size = 5279216, upload-time = "2026-03-06T13:48:56.75Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9d/12a13424f1e604fc7df9497b73c0356fb78c2fb206abd7465ce47226e8fd/h5py-3.16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8389e13a1fd745ad2856873e8187fd10268b2d9677877bb667b41aebd771d8b7", size = 5070068, upload-time = "2026-03-06T13:48:59.169Z" }, + { url = "https://files.pythonhosted.org/packages/41/8c/bbe98f813722b4873818a8db3e15aa3e625b59278566905ac439725e8070/h5py-3.16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:346df559a0f7dcb31cf8e44805319e2ab24b8957c45e7708ce503b2ec79ba725", size = 5300253, upload-time = "2026-03-06T13:49:02.033Z" }, +] + +[[package]] +name = "healpix" +version = "2025.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/1a/c1af658a2d06cdd2734b085b0eee34d51646f091ed13b1669d3a596de57c/healpix-2025.1.tar.gz", hash = "sha256:733f16f30c15d911c8209f4045f0b0934e5fcafa63b8b39d62e425a036adccea", size = 22744, upload-time = "2025-07-07T16:38:42.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/9e/a14baf09831eef05592a91b8c0c29918025b5069511c9300c4ce64ef5f25/healpix-2025.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1a1d41e0c2a98ef3d3cf48e18ca7edadb599641f1754e28e8df0a0a96471fdd4", size = 70836, upload-time = "2025-07-07T16:38:22.512Z" }, + { url = "https://files.pythonhosted.org/packages/0b/06/7946ec2c0fd5a4fb2b077c9c49358fb32577b21c4e27c7457464716780d3/healpix-2025.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2e347f3c76407d7c8018cd8da7eccb4aebd715e46b8f60111dcf161e969bd25f", size = 69891, upload-time = "2025-07-07T16:38:23.692Z" }, + { url = "https://files.pythonhosted.org/packages/81/cb/82b05f0779924b73c2522f440a65bd4d27af2099aec876c44168bea01136/healpix-2025.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e8cacfd6359564c12c7bc81633e92fa7d6e74fffb834a1e08a38e88cb9ac3c50", size = 70865, upload-time = "2025-07-07T16:38:29.898Z" }, + { url = "https://files.pythonhosted.org/packages/fa/0c/fc40bad841a2c6d45b8e59973135be667c8f4b2000c870b374a3a746a66a/healpix-2025.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa546ca045be0661dfc42f7a4e7996f2134dd07d0d5639ae8d7d1a6b293470a2", size = 69939, upload-time = "2025-07-07T16:38:31.257Z" }, +] + +[[package]] +name = "healpy" +version = "1.19.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astropy", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e3/4e/6f5328f375f88c8c38314f24f82b7e1486ec0358c557779500382db1503c/healpy-1.19.0.tar.gz", hash = "sha256:28e839cb885a23d36c77fc3423a3cb9271a07fda94085bd12fc329f941130ec5", size = 4075006, upload-time = "2025-12-02T08:27:19.766Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/51/c07439b92bc05966006897762b7291d7d2c83fde8807b8d53a48e140a8b4/healpy-1.19.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:730095eef4bbe3c94039edb330d1800285cf43c7e16a81ce5b87edb31a8ffa42", size = 8239086, upload-time = "2025-12-02T08:26:53.076Z" }, + { url = "https://files.pythonhosted.org/packages/61/c6/42745aeff172755e55af2487491c7a7475fe1068f152850cc14d64e04f30/healpy-1.19.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:89dbe14b270b5479a9652d1ab9f28fc7fce34253549db665f2e91e6cea588883", size = 8255685, upload-time = "2025-12-02T08:26:54.361Z" }, + { url = "https://files.pythonhosted.org/packages/9d/86/2a223bd92dde578de34f07095c71152c84b158644782c4057ca9b25e53b5/healpy-1.19.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:29bb5f7acde68ca5850c3a59ba33cacf4c3768c9e85f7013a99aa5fbd127ce84", size = 9042629, upload-time = "2025-12-02T08:26:55.773Z" }, + { url = "https://files.pythonhosted.org/packages/18/6d/037a7bb889c92b3e3f792de6a692e636de0577ea62bdd0038b1eb68270af/healpy-1.19.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:126bef3ae6594f461435ac85c6bcb32e427eca43f743881d00186bc3a55da1d8", size = 9376051, upload-time = "2025-12-02T08:26:57.516Z" }, + { url = "https://files.pythonhosted.org/packages/de/bb/314bf8c493701b5842cdd74a1b5b34cea1522444042efad5b7cb17b8d159/healpy-1.19.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:086bae2510ac60f7cf9bbb520e1d1b8864cc8fca74b46b16e24857e74d08f896", size = 8198878, upload-time = "2025-12-02T08:27:03.119Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d4/a60ed9a50768ff5e896dd94d878496ae16767925ea32c49d5a4189ab818a/healpy-1.19.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:920c0a1c6749c05c8ad9522a5a2630f7bc83124c5742ef50f91b9f5e6a1bdcc7", size = 8205835, upload-time = "2025-12-02T08:27:04.416Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fd/5f4c989b53423bbad07c91a4b9d52de5cde9f1154e3d2b145b397e1b8cd8/healpy-1.19.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:26e3e6c50f2d256c9218f0fb0624406a93c4f22bc66deebf51a0f31fd5594b89", size = 8991846, upload-time = "2025-12-02T08:27:06.123Z" }, + { url = "https://files.pythonhosted.org/packages/d5/df/00636a5d2f1141b1fa9646069436c4fc68b35ac2c53ef520f8812b900f05/healpy-1.19.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bc3c243985abf1f1b5d91da12a23ceed49181dd9d0e46ed362babdc92c814aa8", size = 9325719, upload-time = "2025-12-02T08:27:08.138Z" }, + { url = "https://files.pythonhosted.org/packages/c5/ca/f583967e1395e41608225d0a6d29e582236f615d83023d730e63bac89baa/healpy-1.19.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b3cf9c627af77a3fc670d6240d8a83485f373c274945ca8ad156fc2a25d6eb61", size = 8172544, upload-time = "2025-12-02T08:27:13.684Z" }, + { url = "https://files.pythonhosted.org/packages/85/ae/80a5e0d36ee51e72b27fb8e11d0d43dfb3f8e2ccc315a38865a2ccd5ed73/healpy-1.19.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:c22360ad7ec6f16bdbb650dececf52d21f7f0e03a943eb1a1046fdb9c51a3c5e", size = 8198134, upload-time = "2025-12-02T08:27:15.156Z" }, + { url = "https://files.pythonhosted.org/packages/36/7b/34b9954fdf76b1f5ab0a76228650c2575b189fb6f9f7865c98477a46086d/healpy-1.19.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1a479826c1d6ae8312476e079a3f5c7aa3e40911d74f2d7069adcd62e98604ad", size = 8986426, upload-time = "2025-12-02T08:27:16.417Z" }, + { url = "https://files.pythonhosted.org/packages/94/df/bb3ef51f423c62576fb0775fe72f0d2028a10e3ffb20b4fc76146ea8757b/healpy-1.19.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a103e327102e124785c1b5bdd379c6e5469137633212b56a6196bf04ddd100ff", size = 9307512, upload-time = "2025-12-02T08:27:18.03Z" }, +] + +[[package]] +name = "healsparse" +version = "1.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hpgeom", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e3/33/36520faabae128fe9a4a5cbeb0916be864d2eb34cf7410d52dc532af044e/healsparse-1.13.0.tar.gz", hash = "sha256:c21e4eb075c686bd306bdc1b40e039134c4ca60ccfa4e7e73845c107a5b16544", size = 133589, upload-time = "2026-07-07T18:33:05.481Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/2f/7674b91031a12a9b3cd69f3bb675d3b1eac59657139e23bc528ed6d33d0a/healsparse-1.13.0-py3-none-any.whl", hash = "sha256:3b475a61d7701526246a592ed980758f94764caba4355429bf470a2f9b812964", size = 70673, upload-time = "2026-07-07T18:33:04.514Z" }, +] + +[[package]] +name = "hpack" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/5b/fcabf6028144a8723726318b07a32c2f3314acdff6265743cf08a344b18e/hpack-4.2.0.tar.gz", hash = "sha256:0895cfa3b5531fc65fe439c05eb65144f123bf7a394fcaa56aa423548d8e45c0", size = 51300, upload-time = "2026-06-23T18:34:46.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/b4/4a9fcfb2aef6ba44d9073ecd301443aa00b3dac95de5619f2a7de7ec8a91/hpack-4.2.0-py3-none-any.whl", hash = "sha256:858ac0b02280fa582b5080d68db0899c62a80375e0e5413a74970c5e518b6986", size = 34246, upload-time = "2026-06-23T18:34:45.472Z" }, +] + +[[package]] +name = "hpgeom" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/72/27640de7d85a566b8ecd27e7894f371a19669c56c268756e18111a137df8/hpgeom-1.5.4.tar.gz", hash = "sha256:85ac73e267c11f3f248920d3541a0e6ffab821cae54f0d44414d1939dfb91200", size = 153906, upload-time = "2026-02-24T19:03:01.954Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/7f/bae5e1a44d76fa7396c4a004e2e86acb5fe70d8aaaa7f91190c65eb09e97/hpgeom-1.5.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d544a9834477b0aedf61ee934ce4598170cf14f16742200cd1729103edb5f506", size = 192550, upload-time = "2026-02-24T19:02:49.454Z" }, + { url = "https://files.pythonhosted.org/packages/68/4d/8b83dcbbd08bd7e713583490c14db848515a909515644037c80d20e2b80c/hpgeom-1.5.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7af0216ff1db0a49a771356562088c49185104a10502b68ab26c8ca2816d17b0", size = 197180, upload-time = "2026-02-24T19:02:50.453Z" }, + { url = "https://files.pythonhosted.org/packages/75/dc/4c0c6d7eba1d42be23794b03e6048af4de0d92c5805f4f2ed29bc1ca0f16/hpgeom-1.5.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d7058b3396fb7deb8b8637ac7f2861bfaa213574c50b9e771bf80f4d1e9064f3", size = 192643, upload-time = "2026-02-24T19:02:54.045Z" }, + { url = "https://files.pythonhosted.org/packages/8d/9a/41ba036857a5489d2c1e6f742903aae452d874a48c5b24889de79e02e668/hpgeom-1.5.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e160d40a18b8e80b7e233b4459adcbd4aea5d1fb48cefe7f07e8fc7dfbdcd9f3", size = 197257, upload-time = "2026-02-24T19:02:55.356Z" }, + { url = "https://files.pythonhosted.org/packages/a7/83/45d4208eb33432195df1998d5b5d75afc52d52cdec5ccdec80fa9d3fb2cc/hpgeom-1.5.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:47fe1c0c334d9722110a0d6346d2e4222aaed4454138996f9b2638a78ec98b60", size = 192606, upload-time = "2026-02-24T19:02:58.915Z" }, + { url = "https://files.pythonhosted.org/packages/1d/67/ab80b95519d453e72cd65ccc6cd411f23a4d997f492ed9ad0dbc51bce0fa/hpgeom-1.5.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fce969f1311a2f44c4f1c58fca1302b472ef798e702725a75bc16a342f924a66", size = 197071, upload-time = "2026-02-24T19:03:00.063Z" }, +] + +[[package]] +name = "html2text" +version = "2025.4.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/27/e158d86ba1e82967cc2f790b0cb02030d4a8bef58e0c79a8590e9678107f/html2text-2025.4.15.tar.gz", hash = "sha256:948a645f8f0bc3abe7fd587019a2197a12436cd73d0d4908af95bfc8da337588", size = 64316, upload-time = "2025-04-15T04:02:30.045Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/84/1a0f9555fd5f2b1c924ff932d99b40a0f8a6b12f6dd625e2a47f415b00ea/html2text-2025.4.15-py3-none-any.whl", hash = "sha256:00569167ffdab3d7767a4cdf589b7f57e777a5ed28d12907d8c58769ec734acc", size = 34656, upload-time = "2025-04-15T04:02:28.44Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi", marker = "sys_platform == 'linux'" }, + { name = "h11", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform == 'linux'" }, + { name = "certifi", marker = "sys_platform == 'linux'" }, + { name = "httpcore", marker = "sys_platform == 'linux'" }, + { name = "idna", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[package.optional-dependencies] +http2 = [ + { name = "h2", marker = "sys_platform == 'linux'" }, +] + +[[package]] +name = "humanfriendly" +version = "10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794, upload-time = "2021-09-17T21:40:39.897Z" }, +] + +[[package]] +name = "hyperframe" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "imageio" +version = "2.37.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "pillow", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/84/93bcd1300216ea50811cee96873b84a1bebf8d0489ffaf7f2a3756bab866/imageio-2.37.3.tar.gz", hash = "sha256:bbb37efbfc4c400fcd534b367b91fcd66d5da639aaa138034431a1c5e0a41451", size = 389673, upload-time = "2026-03-09T11:31:12.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/fa/391e437a34e55095173dca5f24070d89cbc233ff85bf1c29c93248c6588d/imageio-2.37.3-py3-none-any.whl", hash = "sha256:46f5bb8522cd421c0f5ae104d8268f569d856b29eb1a13b92829d1970f32c9f0", size = 317646, upload-time = "2026-03-09T11:31:10.771Z" }, +] + +[[package]] +name = "imagesize" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/e6/7bf14eeb8f8b7251141944835abd42eb20a658d89084b7e1f3e5fe394090/imagesize-2.0.0.tar.gz", hash = "sha256:8e8358c4a05c304f1fccf7ff96f036e7243a189e9e42e90851993c558cfe9ee3", size = 1773045, upload-time = "2026-03-03T14:18:29.941Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/53/fb7122b71361a0d121b669dcf3d31244ef75badbbb724af388948de543e2/imagesize-2.0.0-py2.py3-none-any.whl", hash = "sha256:5667c5bbb57ab3f1fa4bc366f4fbc971db3d5ed011fd2715fd8001f782718d96", size = 9441, upload-time = "2026-03-03T14:18:27.892Z" }, +] + +[[package]] +name = "immutables" +version = "0.21" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/41/0ccaa6ef9943c0609ec5aa663a3b3e681c1712c1007147b84590cec706a0/immutables-0.21.tar.gz", hash = "sha256:b55ffaf0449790242feb4c56ab799ea7af92801a0a43f9e2f4f8af2ab24dfc4a", size = 89008, upload-time = "2024-10-10T00:55:01.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/52/cb9e2bb7a69338155ffabbd2f993c968c750dd2d5c6c6eaa6ebb7bfcbdfa/immutables-0.21-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b48b116aaca4500398058b5a87814857a60c4cb09417fecc12d7da0f5639b73d", size = 104270, upload-time = "2024-10-10T00:54:21.912Z" }, + { url = "https://files.pythonhosted.org/packages/0f/a4/25df835a9b9b372a4a869a8a1ac30a32199f2b3f581ad0e249f7e3d19eed/immutables-0.21-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dad7c0c74b285cc0e555ec0e97acbdc6f1862fcd16b99abd612df3243732e741", size = 104864, upload-time = "2024-10-10T00:54:22.956Z" }, + { url = "https://files.pythonhosted.org/packages/4a/51/b548fbc657134d658e179ee8d201ae82d9049aba5c3cb2d858ed2ecb7e3f/immutables-0.21-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e44346e2221a5a676c880ca8e0e6429fa24d1a4ae562573f5c04d7f2e759b030", size = 99733, upload-time = "2024-10-10T00:54:23.99Z" }, + { url = "https://files.pythonhosted.org/packages/47/db/d7b1e0e88faf07fe9a88579a86f58078a9a37fff871f4b3dbcf28cad9a12/immutables-0.21-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b10139b529a460e53fe8be699ebd848c54c8a33ebe67763bcfcc809a475a26f", size = 101698, upload-time = "2024-10-10T00:54:25.734Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d0/a5fb7c164ddb298ec37537e618b70dfa30c7cae9fac01de374c36489cbc9/immutables-0.21-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:583d2a63e444ce1538cc2bda56ae1f4a1a11473dbc0377c82b516bc7eec3b81e", size = 104334, upload-time = "2024-10-10T00:54:31.284Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a5/5fda0ee4a261a85124011ac0750fec678f00e1b2d4a5502b149a3b4d86d9/immutables-0.21-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b274a52da9b106db55eceb93fc1aea858c4e6f4740189e3548e38613eafc2021", size = 104898, upload-time = "2024-10-10T00:54:32.295Z" }, + { url = "https://files.pythonhosted.org/packages/93/fa/d46bfe92f2c66d35916344176ff87fa839aac9c16849652947e722b7a15f/immutables-0.21-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:338bede057250b33716a3e4892e15df0bf5a5ddbf1d67ead996b3e680b49ef9e", size = 99966, upload-time = "2024-10-10T00:54:34.046Z" }, + { url = "https://files.pythonhosted.org/packages/d7/f5/2a19e2e095f7a39d8d77dcc10669734d2d99773ce00c99bdcfeeb7d714e6/immutables-0.21-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8781c89583b68f604cf30f0978b722165824c3075888639fde771bf1a3e12dc0", size = 101773, upload-time = "2024-10-10T00:54:35.851Z" }, +] + +[[package]] +name = "importlib-metadata" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7", size = 27789, upload-time = "2026-03-20T06:42:55.665Z" }, +] + +[[package]] +name = "importlib-resources" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/06/b56dfa750b44e86157093bc8fca0ab81dccbf5260510de4eaf1cb69b5b99/importlib_resources-7.1.0.tar.gz", hash = "sha256:0722d4c6212489c530f2a145a34c0a7a3b4721bc96a15fada5930e2a0b760708", size = 44985, upload-time = "2026-04-12T16:36:09.232Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/db/55a262f3606bebcae07cc14095338471ad7c0bbcaa37707e6f0ee49725b7/importlib_resources-7.1.0-py3-none-any.whl", hash = "sha256:1bd7b48b4088eddb2cd16382150bb515af0bd2c70128194392725f82ad2c96a1", size = 37232, upload-time = "2026-04-12T16:36:08.219Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "ipykernel" +version = "7.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "comm", marker = "sys_platform == 'linux'" }, + { name = "debugpy", marker = "sys_platform == 'linux'" }, + { name = "ipython", marker = "sys_platform == 'linux'" }, + { name = "jupyter-client", marker = "sys_platform == 'linux'" }, + { name = "jupyter-core", marker = "sys_platform == 'linux'" }, + { name = "matplotlib-inline", marker = "sys_platform == 'linux'" }, + { name = "nest-asyncio2", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "psutil", marker = "sys_platform == 'linux'" }, + { name = "pyzmq", marker = "sys_platform == 'linux'" }, + { name = "tornado", marker = "sys_platform == 'linux'" }, + { name = "traitlets", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/c4/e4a38f579de4225a561305666f7541cdabb30075def2aa1ac17bd73c1fb5/ipykernel-7.3.0.tar.gz", hash = "sha256:9acaaaf97d16355166e4085afe9d225bfbdf2b7ef520f9df3be8f2b248275e09", size = 184899, upload-time = "2026-06-10T08:41:25.481Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/02/77b271f5dc58bfbc0b577c877b2365d1ffea2afe66a80c13f2312820348c/ipykernel-7.3.0-py3-none-any.whl", hash = "sha256:897eb64da762549ef610698fca5e9675195ec6ac8ec7f19d81ce1ca20c876057", size = 120583, upload-time = "2026-06-10T08:41:23.648Z" }, +] + +[[package]] +name = "ipython" +version = "9.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "decorator", marker = "sys_platform == 'linux'" }, + { name = "ipython-pygments-lexers", marker = "sys_platform == 'linux'" }, + { name = "jedi", marker = "sys_platform == 'linux'" }, + { name = "matplotlib-inline", marker = "sys_platform == 'linux'" }, + { name = "pexpect", marker = "sys_platform == 'linux'" }, + { name = "prompt-toolkit", marker = "sys_platform == 'linux'" }, + { name = "psutil", marker = "sys_platform == 'linux'" }, + { name = "pygments", marker = "sys_platform == 'linux'" }, + { name = "stack-data", marker = "sys_platform == 'linux'" }, + { name = "traitlets", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/59/165d3b4d75cc34add3122c4417ecb229085140ac573103c223cd01dde96f/ipython-9.15.0.tar.gz", hash = "sha256:da2819ce2aa83135257df830660b1176d986c3d2876db24df01974fa955b2756", size = 4442580, upload-time = "2026-06-26T11:03:35.913Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/3a/948263ca3b9d65bb2b1b0c521b3a49fad5d59ada58724bd87d2bd5ff3f36/ipython-9.15.0-py3-none-any.whl", hash = "sha256:515ad9c3cdf0c932a5a9f6245419e8aba706b7bd03c3e1d3a1c83d9351d6aa6e", size = 630895, upload-time = "2026-06-26T11:03:33.809Z" }, +] + +[[package]] +name = "ipython-genutils" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/69/fbeffffc05236398ebfcfb512b6d2511c622871dca1746361006da310399/ipython_genutils-0.2.0.tar.gz", hash = "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8", size = 22208, upload-time = "2017-03-13T22:12:26.393Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/bc/9bd3b5c2b4774d5f33b2d544f1460be9df7df2fe42f352135381c347c69a/ipython_genutils-0.2.0-py2.py3-none-any.whl", hash = "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8", size = 26343, upload-time = "2017-03-13T22:12:25.412Z" }, +] + +[[package]] +name = "ipython-pygments-lexers" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pygments", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, +] + +[[package]] +name = "ipywidgets" +version = "8.1.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "comm", marker = "sys_platform == 'linux'" }, + { name = "ipython", marker = "sys_platform == 'linux'" }, + { name = "jupyterlab-widgets", marker = "sys_platform == 'linux'" }, + { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "widgetsnbextension", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/ae/c5ce1edc1afe042eadb445e95b0671b03cee61895264357956e61c0d2ac0/ipywidgets-8.1.8.tar.gz", hash = "sha256:61f969306b95f85fba6b6986b7fe45d73124d1d9e3023a8068710d47a22ea668", size = 116739, upload-time = "2025-11-01T21:18:12.393Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl", hash = "sha256:ecaca67aed704a338f88f67b1181b58f821ab5dc89c1f0f5ef99db43c1c2921e", size = 139808, upload-time = "2025-11-01T21:18:10.956Z" }, +] + +[[package]] +name = "isoduration" +version = "20.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "arrow", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7c/1a/3c8edc664e06e6bd06cce40c6b22da5f1429aa4224d0c590f3be21c91ead/isoduration-20.11.0.tar.gz", hash = "sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9", size = 11649, upload-time = "2020-11-01T11:00:00.312Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl", hash = "sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042", size = 11321, upload-time = "2020-11-01T10:59:58.02Z" }, +] + +[[package]] +name = "jaraco-classes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", size = 6777, upload-time = "2024-03-31T07:27:34.792Z" }, +] + +[[package]] +name = "jaraco-context" +version = "6.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/50/4763cd07e722bb6285316d390a164bc7e479db9d90daa769f22578f698b4/jaraco_context-6.1.2.tar.gz", hash = "sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3", size = 16801, upload-time = "2026-03-20T22:13:33.922Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/58/bc8954bda5fcda97bd7c19be11b85f91973d67a706ed4a3aec33e7de22db/jaraco_context-6.1.2-py3-none-any.whl", hash = "sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535", size = 7871, upload-time = "2026-03-20T22:13:32.808Z" }, +] + +[[package]] +name = "jaraco-functools" +version = "4.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/cf/ea4ef2920830dea3f5ab2ea4da6fb67724e6dca80ee2553788c3607243d0/jaraco_functools-4.5.0.tar.gz", hash = "sha256:3bb5665ea4a020cf78a7040e89154c77edadb3ca74f366479669c5999aa70b03", size = 20272, upload-time = "2026-05-15T21:34:10.025Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/9a/982e48afcffcd727a9144506720ffd4224b6b7e355c98641866f38b7c043/jaraco_functools-4.5.0-py3-none-any.whl", hash = "sha256:79ce39246eddbde4b3a03b77ea5f0f7878dc669b166a66cf3fa8e266aa3fa2f4", size = 10594, upload-time = "2026-05-15T21:34:08.595Z" }, +] + +[[package]] +name = "jedi" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "parso", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/b7/a3635f6a2d7cf5b5dd98064fc1d5fbbafcb25477bcea204a3a92145d158b/jedi-0.20.0.tar.gz", hash = "sha256:c3f4ccbd276696f4b19c54618d4fb18f9fc24b0aef02acf704b23f487daa1011", size = 3119416, upload-time = "2026-05-01T23:38:47.814Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl", hash = "sha256:7bdd9c2634f56713299976f4cbd59cb3fa92165cc5e05ea811fb253480728b67", size = 4884812, upload-time = "2026-05-01T23:38:43.919Z" }, +] + +[[package]] +name = "jeepney" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758, upload-time = "2025-02-27T18:51:01.684Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "joblib" +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, +] + +[[package]] +name = "json5" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/7d/05c46a96a78147ae3bf99c2f4169ce144a70220b8d6fcd56f6ec368b8ce9/json5-0.15.0.tar.gz", hash = "sha256:7424d1f1eb1d56da6e3d70643f53619862b4ce81440bdb8ecfd6f875e5ba4a71", size = 53278, upload-time = "2026-06-19T20:08:27.716Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/be/59527c99478aade6bb33a68d72e6e18dd4e6ff6eacfc7d01bdb15bc76912/json5-0.15.0-py3-none-any.whl", hash = "sha256:56636a30c0e8a4665fe2179c0212f32eae3796dea89ea6f649b9436ecdb39618", size = 36570, upload-time = "2026-06-19T20:08:26.748Z" }, +] + +[[package]] +name = "jsonargparse" +version = "4.49.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/87/0b/f012466077d803b546955bf827ab30a320ad1550e3167195432cc2d6bfad/jsonargparse-4.49.0.tar.gz", hash = "sha256:9e691a09d937fb4c6bc1f6d3bf6c3546efa03381ea1e64ee10759095f1b14da7", size = 125237, upload-time = "2026-05-15T06:48:44.807Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/64/51cbc9044f5b1ee4fb9894fe791f50dfdb52dc080acfafb5818ba19e8c27/jsonargparse-4.49.0-py3-none-any.whl", hash = "sha256:2025880765fd792f6fb9861aa93f26da5f6c350be5d7656321369b22b0d2538e", size = 134834, upload-time = "2026-05-15T06:48:40.76Z" }, +] + +[package.optional-dependencies] +signatures = [ + { name = "docstring-parser", marker = "sys_platform == 'linux'" }, + { name = "typeshed-client", marker = "sys_platform == 'linux'" }, +] + +[[package]] +name = "jsonpointer" +version = "3.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/c7/af399a2e7a67fd18d63c40c5e62d3af4e67b836a2107468b6a5ea24c4304/jsonpointer-3.1.1.tar.gz", hash = "sha256:0b801c7db33a904024f6004d526dcc53bbb8a4a0f4e32bfd10beadf60adf1900", size = 9068, upload-time = "2026-03-23T22:32:32.458Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl", hash = "sha256:8ff8b95779d071ba472cf5bc913028df06031797532f08a7d5b602d8b2a488ca", size = 7659, upload-time = "2026-03-23T22:32:31.568Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs", marker = "sys_platform == 'linux'" }, + { name = "jsonschema-specifications", marker = "sys_platform == 'linux'" }, + { name = "referencing", marker = "sys_platform == 'linux'" }, + { name = "rpds-py", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[package.optional-dependencies] +format-nongpl = [ + { name = "fqdn", marker = "sys_platform == 'linux'" }, + { name = "idna", marker = "sys_platform == 'linux'" }, + { name = "isoduration", marker = "sys_platform == 'linux'" }, + { name = "jsonpointer", marker = "sys_platform == 'linux'" }, + { name = "rfc3339-validator", marker = "sys_platform == 'linux'" }, + { name = "rfc3986-validator", marker = "sys_platform == 'linux'" }, + { name = "rfc3987-syntax", marker = "sys_platform == 'linux'" }, + { name = "uri-template", marker = "sys_platform == 'linux'" }, + { name = "webcolors", marker = "sys_platform == 'linux'" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "jupyter" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ipykernel", marker = "sys_platform == 'linux'" }, + { name = "ipywidgets", marker = "sys_platform == 'linux'" }, + { name = "jupyter-console", marker = "sys_platform == 'linux'" }, + { name = "jupyterlab", marker = "sys_platform == 'linux'" }, + { name = "nbconvert", marker = "sys_platform == 'linux'" }, + { name = "notebook", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/f3/af28ea964ab8bc1e472dba2e82627d36d470c51f5cd38c37502eeffaa25e/jupyter-1.1.1.tar.gz", hash = "sha256:d55467bceabdea49d7e3624af7e33d59c37fff53ed3a350e1ac957bed731de7a", size = 5714959, upload-time = "2024-08-30T07:15:48.299Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/64/285f20a31679bf547b75602702f7800e74dbabae36ef324f716c02804753/jupyter-1.1.1-py2.py3-none-any.whl", hash = "sha256:7a59533c22af65439b24bbe60373a4e95af8f16ac65a6c00820ad378e3f7cc83", size = 2657, upload-time = "2024-08-30T07:15:47.045Z" }, +] + +[[package]] +name = "jupyter-builder" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-core", marker = "sys_platform == 'linux'" }, + { name = "traitlets", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/df/db5efc4e28803a1421350445e53d85ec571a4e6928e3524ccc39f4e16584/jupyter_builder-1.1.0.tar.gz", hash = "sha256:b996e8af616900f18724fa34883169d869be2205497940bec78a3a1031eb897d", size = 971142, upload-time = "2026-07-11T07:24:02.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/fb/53adbc72931b4429275b160044c3e1bcbc8fbb5ea6b8f318a357834842fb/jupyter_builder-1.1.0-py3-none-any.whl", hash = "sha256:d9d5280e8845f726663545c3a6db55bd205127616eeb8958481091d6cba71b6a", size = 912964, upload-time = "2026-07-11T07:24:00.279Z" }, +] + +[[package]] +name = "jupyter-client" +version = "8.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-core", marker = "sys_platform == 'linux'" }, + { name = "python-dateutil", marker = "sys_platform == 'linux'" }, + { name = "pyzmq", marker = "sys_platform == 'linux'" }, + { name = "tornado", marker = "sys_platform == 'linux'" }, + { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/dc/5512503b088997c2250b8bf18258fba9d9ce5ead641183700960d3c9d342/jupyter_client-8.9.1.tar.gz", hash = "sha256:a58f730dd9e728ba16ba1d62ebccf7ffe1ebbdbce4e95cfae941b7321ae1f4fa", size = 359256, upload-time = "2026-06-09T13:15:01.033Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/6f/56d39bf385c5c27988aebaf0c18a2a17e960575740100973511018bd904e/jupyter_client-8.9.1-py3-none-any.whl", hash = "sha256:0b7a295bc46e8751e9adae84781f726c851c1d911bd793edc4a3bde942e3da81", size = 109828, upload-time = "2026-06-09T13:14:58.835Z" }, +] + +[[package]] +name = "jupyter-console" +version = "6.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ipykernel", marker = "sys_platform == 'linux'" }, + { name = "ipython", marker = "sys_platform == 'linux'" }, + { name = "jupyter-client", marker = "sys_platform == 'linux'" }, + { name = "jupyter-core", marker = "sys_platform == 'linux'" }, + { name = "prompt-toolkit", marker = "sys_platform == 'linux'" }, + { name = "pygments", marker = "sys_platform == 'linux'" }, + { name = "pyzmq", marker = "sys_platform == 'linux'" }, + { name = "traitlets", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/2d/e2fd31e2fc41c14e2bcb6c976ab732597e907523f6b2420305f9fc7fdbdb/jupyter_console-6.6.3.tar.gz", hash = "sha256:566a4bf31c87adbfadf22cdf846e3069b59a71ed5da71d6ba4d8aaad14a53539", size = 34363, upload-time = "2023-03-06T14:13:31.02Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/77/71d78d58f15c22db16328a476426f7ac4a60d3a5a7ba3b9627ee2f7903d4/jupyter_console-6.6.3-py3-none-any.whl", hash = "sha256:309d33409fcc92ffdad25f0bcdf9a4a9daa61b6f341177570fdac03de5352485", size = 24510, upload-time = "2023-03-06T14:13:28.229Z" }, +] + +[[package]] +name = "jupyter-core" +version = "5.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "platformdirs", marker = "sys_platform == 'linux'" }, + { name = "traitlets", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/49/9d1284d0dc65e2c757b74c6687b6d319b02f822ad039e5c512df9194d9dd/jupyter_core-5.9.1.tar.gz", hash = "sha256:4d09aaff303b9566c3ce657f580bd089ff5c91f5f89cf7d8846c3cdf465b5508", size = 89814, upload-time = "2025-10-16T19:19:18.444Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl", hash = "sha256:ebf87fdc6073d142e114c72c9e29a9d7ca03fad818c5d300ce2adc1fb0743407", size = 29032, upload-time = "2025-10-16T19:19:16.783Z" }, +] + +[[package]] +name = "jupyter-events" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonschema", extra = ["format-nongpl"], marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "python-json-logger", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "referencing", marker = "sys_platform == 'linux'" }, + { name = "rfc3339-validator", marker = "sys_platform == 'linux'" }, + { name = "rfc3986-validator", marker = "sys_platform == 'linux'" }, + { name = "traitlets", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/f8/475c4241b2b75af0deaae453ed003c6c851766dbc44d332d8baf245dc931/jupyter_events-0.12.1.tar.gz", hash = "sha256:faff25f77218335752f35f23c5fe6e4a392a7bd99a5939ccb9b8fbf594636cf3", size = 62854, upload-time = "2026-04-20T23:17:50.66Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/6c/6fcde0c8f616ed360ffd3587f7db9e225a7e62b583a04494d2f069cf64ea/jupyter_events-0.12.1-py3-none-any.whl", hash = "sha256:c366585253f537a627da52fa7ca7410c5b5301fe893f511e7b077c2d93ec8bcf", size = 19512, upload-time = "2026-04-20T23:17:48.927Z" }, +] + +[[package]] +name = "jupyter-lsp" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-server", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/ff/1e4a61f5170a9a1d978f3ac3872449de6c01fc71eaf89657824c878b1549/jupyter_lsp-2.3.1.tar.gz", hash = "sha256:fdf8a4aa7d85813976d6e29e95e6a2c8f752701f926f2715305249a3829805a6", size = 55677, upload-time = "2026-04-02T08:10:06.749Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/e8/9d61dcbd1dce8ef418f06befd4ac084b4720429c26b0b1222bc218685eff/jupyter_lsp-2.3.1-py3-none-any.whl", hash = "sha256:71b954d834e85ff3096400554f2eefaf7fe37053036f9a782b0f7c5e42dadb81", size = 77513, upload-time = "2026-04-02T08:10:01.753Z" }, +] + +[[package]] +name = "jupyter-server" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform == 'linux'" }, + { name = "argon2-cffi", marker = "sys_platform == 'linux'" }, + { name = "jinja2", marker = "sys_platform == 'linux'" }, + { name = "jupyter-client", marker = "sys_platform == 'linux'" }, + { name = "jupyter-core", marker = "sys_platform == 'linux'" }, + { name = "jupyter-events", marker = "sys_platform == 'linux'" }, + { name = "jupyter-server-terminals", marker = "sys_platform == 'linux'" }, + { name = "nbconvert", marker = "sys_platform == 'linux'" }, + { name = "nbformat", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "prometheus-client", marker = "sys_platform == 'linux'" }, + { name = "pyzmq", marker = "sys_platform == 'linux'" }, + { name = "send2trash", marker = "sys_platform == 'linux'" }, + { name = "terminado", marker = "sys_platform == 'linux'" }, + { name = "tornado", marker = "sys_platform == 'linux'" }, + { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "websocket-client", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6b/dc/db3a582633170186f8c8b31298d7eb26ad0eb031a1f53476c258b64eed05/jupyter_server-2.20.0.tar.gz", hash = "sha256:b5778ba337d8015a3dc2b80803ecdd5ac18d3797fddf61a50ea5fb472b4ebe14", size = 756523, upload-time = "2026-06-17T12:09:09.435Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/71/8c002223e873a870f5c41dc69b0a7c922301123e4a31d5d01ecb700aef77/jupyter_server-2.20.0-py3-none-any.whl", hash = "sha256:c3b67c93c471e947c18b5026f04f21614218adb706df8f48227d3ee8e0a7cdcc", size = 393143, upload-time = "2026-06-17T12:09:07.234Z" }, +] + +[[package]] +name = "jupyter-server-terminals" +version = "0.5.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "terminado", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f4/a7/bcd0a9b0cbba88986fe944aaaf91bfda603e5a50bda8ed15123f381a3b2f/jupyter_server_terminals-0.5.4.tar.gz", hash = "sha256:bbda128ed41d0be9020349f9f1f2a4ab9952a73ed5f5ac9f1419794761fb87f5", size = 31770, upload-time = "2026-01-14T16:53:20.213Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/2d/6674563f71c6320841fc300911a55143925112a72a883e2ca71fba4c618d/jupyter_server_terminals-0.5.4-py3-none-any.whl", hash = "sha256:55be353fc74a80bc7f3b20e6be50a55a61cd525626f578dcb66a5708e2007d14", size = 13704, upload-time = "2026-01-14T16:53:18.738Z" }, +] + +[[package]] +name = "jupyterlab" +version = "4.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-lru", marker = "sys_platform == 'linux'" }, + { name = "httpx", marker = "sys_platform == 'linux'" }, + { name = "ipykernel", marker = "sys_platform == 'linux'" }, + { name = "jinja2", marker = "sys_platform == 'linux'" }, + { name = "jupyter-builder", marker = "sys_platform == 'linux'" }, + { name = "jupyter-core", marker = "sys_platform == 'linux'" }, + { name = "jupyter-lsp", marker = "sys_platform == 'linux'" }, + { name = "jupyter-server", marker = "sys_platform == 'linux'" }, + { name = "jupyterlab-server", marker = "sys_platform == 'linux'" }, + { name = "notebook-shim", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "tornado", marker = "sys_platform == 'linux'" }, + { name = "traitlets", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bc/2a/d6af53bfd45a43a5bfe7e40ba47ee7a8921a807daf4bb708e3a295bbb54d/jupyterlab-4.6.1.tar.gz", hash = "sha256:75315982ed28427edaa62bb85eadb5105e4043a757643c910efd787fe6ed0837", size = 28179125, upload-time = "2026-06-29T12:48:45.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/81/90ac6cc31d248e83a0d1eab343a5e6e68bc783d3f74fbe61640f42a61da4/jupyterlab-4.6.1-py3-none-any.whl", hash = "sha256:85a58546c831f3dce6cf919468c26874c9065e99c42279fb4abb8e1b552a98bb", size = 17164660, upload-time = "2026-06-29T12:48:41.21Z" }, +] + +[[package]] +name = "jupyterlab-pygments" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/90/51/9187be60d989df97f5f0aba133fa54e7300f17616e065d1ada7d7646b6d6/jupyterlab_pygments-0.3.0.tar.gz", hash = "sha256:721aca4d9029252b11cfa9d185e5b5af4d54772bb8072f9b7036f4170054d35d", size = 512900, upload-time = "2023-11-23T09:26:37.44Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/dd/ead9d8ea85bf202d90cc513b533f9c363121c7792674f78e0d8a854b63b4/jupyterlab_pygments-0.3.0-py3-none-any.whl", hash = "sha256:841a89020971da1d8693f1a99997aefc5dc424bb1b251fd6322462a1b8842780", size = 15884, upload-time = "2023-11-23T09:26:34.325Z" }, +] + +[[package]] +name = "jupyterlab-server" +version = "2.28.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "babel", marker = "sys_platform == 'linux'" }, + { name = "jinja2", marker = "sys_platform == 'linux'" }, + { name = "json5", marker = "sys_platform == 'linux'" }, + { name = "jsonschema", marker = "sys_platform == 'linux'" }, + { name = "jupyter-server", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "requests", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d6/2c/90153f189e421e93c4bb4f9e3f59802a1f01abd2ac5cf40b152d7f735232/jupyterlab_server-2.28.0.tar.gz", hash = "sha256:35baa81898b15f93573e2deca50d11ac0ae407ebb688299d3a5213265033712c", size = 76996, upload-time = "2025-10-22T13:59:18.37Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/07/a000fe835f76b7e1143242ab1122e6362ef1c03f23f83a045c38859c2ae0/jupyterlab_server-2.28.0-py3-none-any.whl", hash = "sha256:e4355b148fdcf34d312bbbc80f22467d6d20460e8b8736bf235577dd18506968", size = 59830, upload-time = "2025-10-22T13:59:16.767Z" }, +] + +[[package]] +name = "jupyterlab-widgets" +version = "3.0.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/2d/ef58fed122b268c69c0aa099da20bc67657cdfb2e222688d5731bd5b971d/jupyterlab_widgets-3.0.16.tar.gz", hash = "sha256:423da05071d55cf27a9e602216d35a3a65a3e41cdf9c5d3b643b814ce38c19e0", size = 897423, upload-time = "2025-11-01T21:11:29.724Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/b5/36c712098e6191d1b4e349304ef73a8d06aed77e56ceaac8c0a306c7bda1/jupyterlab_widgets-3.0.16-py3-none-any.whl", hash = "sha256:45fa36d9c6422cf2559198e4db481aa243c7a32d9926b500781c830c80f7ecf8", size = 914926, upload-time = "2025-11-01T21:11:28.008Z" }, +] + +[[package]] +name = "jupytext" +version = "1.19.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, + { name = "mdit-py-plugins", marker = "sys_platform == 'linux'" }, + { name = "nbformat", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/52/e014296ac8f40ca783aeb73dae52e65edbb0eaae0dcdc1ea41bfaa8aebf7/jupytext-1.19.4.tar.gz", hash = "sha256:739bcd4bc12aa4fe298a38017cdb5ae27b08a6ba3a5470728d2fe9e04b155db1", size = 4581977, upload-time = "2026-06-21T21:48:58.32Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/e9/e2ae007456069dbe01865c69a4203a7ada6f7e337b78fc2f12e51bd3fae7/jupytext-1.19.4-py3-none-any.whl", hash = "sha256:032d4ef4bd2e96addcac780b9b1d6b5a266ca39beceaaca95bfb4f06e0b77029", size = 170889, upload-time = "2026-06-21T21:48:56.352Z" }, +] + +[[package]] +name = "keyring" +version = "25.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jaraco-classes", marker = "sys_platform == 'linux'" }, + { name = "jaraco-context", marker = "sys_platform == 'linux'" }, + { name = "jaraco-functools", marker = "sys_platform == 'linux'" }, + { name = "jeepney", marker = "sys_platform == 'linux'" }, + { name = "secretstorage", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516, upload-time = "2025-11-16T16:26:09.482Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, +] + +[[package]] +name = "kiwisolver" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/67/9c61eccb13f0bdca9307614e782fec49ffdde0f7a2314935d489fa93cd9c/kiwisolver-1.5.0.tar.gz", hash = "sha256:d4193f3d9dc3f6f79aaed0e5637f45d98850ebf01f7ca20e69457f3e8946b66a", size = 103482, upload-time = "2026-03-09T13:15:53.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/13/680c54afe3e65767bed7ec1a15571e1a2f1257128733851ade24abcefbcc/kiwisolver-1.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb5136fb5352d3f422df33f0c879a1b0c204004324150cc3b5e3c4f310c9049f", size = 1477934, upload-time = "2026-03-09T13:13:27.166Z" }, + { url = "https://files.pythonhosted.org/packages/c8/2f/cebfcdb60fd6a9b0f6b47a9337198bcbad6fbe15e68189b7011fd914911f/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2af221f268f5af85e776a73d62b0845fc8baf8ef0abfae79d29c77d0e776aaf", size = 1278537, upload-time = "2026-03-09T13:13:28.707Z" }, + { url = "https://files.pythonhosted.org/packages/f2/0d/9b782923aada3fafb1d6b84e13121954515c669b18af0c26e7d21f579855/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b0f172dc8ffaccb8522d7c5d899de00133f2f1ca7b0a49b7da98e901de87bf2d", size = 1296685, upload-time = "2026-03-09T13:13:30.528Z" }, + { url = "https://files.pythonhosted.org/packages/27/70/83241b6634b04fe44e892688d5208332bde130f38e610c0418f9ede47ded/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6ab8ba9152203feec73758dad83af9a0bbe05001eb4639e547207c40cfb52083", size = 1346024, upload-time = "2026-03-09T13:13:32.818Z" }, + { url = "https://files.pythonhosted.org/packages/e4/db/30ed226fb271ae1a6431fc0fe0edffb2efe23cadb01e798caeb9f2ceae8f/kiwisolver-1.5.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:cdee07c4d7f6d72008d3f73b9bf027f4e11550224c7c50d8df1ae4a37c1402a6", size = 987241, upload-time = "2026-03-09T13:13:34.435Z" }, + { url = "https://files.pythonhosted.org/packages/ec/bd/c314595208e4c9587652d50959ead9e461995389664e490f4dce7ff0f782/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7c60d3c9b06fb23bd9c6139281ccbdc384297579ae037f08ae90c69f6845c0b1", size = 2227742, upload-time = "2026-03-09T13:13:36.4Z" }, + { url = "https://files.pythonhosted.org/packages/c1/43/0499cec932d935229b5543d073c2b87c9c22846aab48881e9d8d6e742a2d/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e315e5ec90d88e140f57696ff85b484ff68bb311e36f2c414aa4286293e6dee0", size = 2323966, upload-time = "2026-03-09T13:13:38.204Z" }, + { url = "https://files.pythonhosted.org/packages/3d/6f/79b0d760907965acfd9d61826a3d41f8f093c538f55cd2633d3f0db269f6/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:1465387ac63576c3e125e5337a6892b9e99e0627d52317f3ca79e6930d889d15", size = 1977417, upload-time = "2026-03-09T13:13:39.966Z" }, + { url = "https://files.pythonhosted.org/packages/ab/31/01d0537c41cb75a551a438c3c7a80d0c60d60b81f694dac83dd436aec0d0/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:530a3fd64c87cffa844d4b6b9768774763d9caa299e9b75d8eca6a4423b31314", size = 2491238, upload-time = "2026-03-09T13:13:41.698Z" }, + { url = "https://files.pythonhosted.org/packages/e4/34/8aefdd0be9cfd00a44509251ba864f5caf2991e36772e61c408007e7f417/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1d9daea4ea6b9be74fe2f01f7fbade8d6ffab263e781274cffca0dba9be9eec9", size = 2294947, upload-time = "2026-03-09T13:13:43.343Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0a/7b98e1e119878a27ba8618ca1e18b14f992ff1eda40f47bccccf4de44121/kiwisolver-1.5.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:332b4f0145c30b5f5ad9374881133e5aa64320428a57c2c2b61e9d891a51c2f3", size = 1477903, upload-time = "2026-03-09T13:13:52.084Z" }, + { url = "https://files.pythonhosted.org/packages/18/d8/55638d89ffd27799d5cc3d8aa28e12f4ce7a64d67b285114dbedc8ea4136/kiwisolver-1.5.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c50b89ffd3e1a911c69a1dd3de7173c0cd10b130f56222e57898683841e4f96", size = 1278751, upload-time = "2026-03-09T13:13:54.673Z" }, + { url = "https://files.pythonhosted.org/packages/b8/97/b4c8d0d18421ecceba20ad8701358453b88e32414e6f6950b5a4bad54e65/kiwisolver-1.5.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4db576bb8c3ef9365f8b40fe0f671644de6736ae2c27a2c62d7d8a1b4329f099", size = 1296793, upload-time = "2026-03-09T13:13:56.287Z" }, + { url = "https://files.pythonhosted.org/packages/c4/10/f862f94b6389d8957448ec9df59450b81bec4abb318805375c401a1e6892/kiwisolver-1.5.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0b85aad90cea8ac6797a53b5d5f2e967334fa4d1149f031c4537569972596cb8", size = 1346041, upload-time = "2026-03-09T13:13:58.269Z" }, + { url = "https://files.pythonhosted.org/packages/a3/6a/f1650af35821eaf09de398ec0bc2aefc8f211f0cda50204c9f1673741ba9/kiwisolver-1.5.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:d36ca54cb4c6c4686f7cbb7b817f66f5911c12ddb519450bbe86707155028f87", size = 987292, upload-time = "2026-03-09T13:13:59.871Z" }, + { url = "https://files.pythonhosted.org/packages/de/19/d7fb82984b9238115fe629c915007be608ebd23dc8629703d917dbfaffd4/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:38f4a703656f493b0ad185211ccfca7f0386120f022066b018eb5296d8613e23", size = 2227865, upload-time = "2026-03-09T13:14:01.401Z" }, + { url = "https://files.pythonhosted.org/packages/7f/b9/46b7f386589fd222dac9e9de9c956ce5bcefe2ee73b4e79891381dda8654/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3ac2360e93cb41be81121755c6462cff3beaa9967188c866e5fce5cf13170859", size = 2324369, upload-time = "2026-03-09T13:14:02.972Z" }, + { url = "https://files.pythonhosted.org/packages/92/8b/95e237cf3d9c642960153c769ddcbe278f182c8affb20cecc1cc983e7cc5/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c95cab08d1965db3d84a121f1c7ce7479bdd4072c9b3dafd8fecce48a2e6b902", size = 1977989, upload-time = "2026-03-09T13:14:04.503Z" }, + { url = "https://files.pythonhosted.org/packages/1b/95/980c9df53501892784997820136c01f62bc1865e31b82b9560f980c0e649/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fc20894c3d21194d8041a28b65622d5b86db786da6e3cfe73f0c762951a61167", size = 2491645, upload-time = "2026-03-09T13:14:06.106Z" }, + { url = "https://files.pythonhosted.org/packages/cb/32/900647fd0840abebe1561792c6b31e6a7c0e278fc3973d30572a965ca14c/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7a32f72973f0f950c1920475d5c5ea3d971b81b6f0ec53b8d0a956cc965f22e0", size = 2295237, upload-time = "2026-03-09T13:14:08.891Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7f/f943879cda9007c45e1f7dba216d705c3a18d6b35830e488b6c6a4e7cdf0/kiwisolver-1.5.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4432b835675f0ea7414aab3d37d119f7226d24869b7a829caeab49ebda407b0c", size = 1584848, upload-time = "2026-03-09T13:14:19.745Z" }, + { url = "https://files.pythonhosted.org/packages/37/f8/4d4f85cc1870c127c88d950913370dd76138482161cd07eabbc450deff01/kiwisolver-1.5.0-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b0feb50971481a2cc44d94e88bdb02cdd497618252ae226b8eb1201b957e368", size = 1391542, upload-time = "2026-03-09T13:14:21.54Z" }, + { url = "https://files.pythonhosted.org/packages/04/0b/65dd2916c84d252b244bd405303220f729e7c17c9d7d33dca6feeff9ffc4/kiwisolver-1.5.0-cp313-cp313t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56fa888f10d0f367155e76ce849fa1166fc9730d13bd2d65a2aa13b6f5424489", size = 1404447, upload-time = "2026-03-09T13:14:23.205Z" }, + { url = "https://files.pythonhosted.org/packages/39/5c/2606a373247babce9b1d056c03a04b65f3cf5290a8eac5d7bdead0a17e21/kiwisolver-1.5.0-cp313-cp313t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:940dda65d5e764406b9fb92761cbf462e4e63f712ab60ed98f70552e496f3bf1", size = 1455918, upload-time = "2026-03-09T13:14:24.74Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d1/c6078b5756670658e9192a2ef11e939c92918833d2745f85cd14a6004bdf/kiwisolver-1.5.0-cp313-cp313t-manylinux_2_39_riscv64.whl", hash = "sha256:89fc958c702ee9a745e4700378f5d23fddbc46ff89e8fdbf5395c24d5c1452a3", size = 1072856, upload-time = "2026-03-09T13:14:26.597Z" }, + { url = "https://files.pythonhosted.org/packages/cb/c8/7def6ddf16eb2b3741d8b172bdaa9af882b03c78e9b0772975408801fa63/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9027d773c4ff81487181a925945743413f6069634d0b122d0b37684ccf4f1e18", size = 2333580, upload-time = "2026-03-09T13:14:28.237Z" }, + { url = "https://files.pythonhosted.org/packages/9e/87/2ac1fce0eb1e616fcd3c35caa23e665e9b1948bb984f4764790924594128/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:5b233ea3e165e43e35dba1d2b8ecc21cf070b45b65ae17dd2747d2713d942021", size = 2423018, upload-time = "2026-03-09T13:14:30.018Z" }, + { url = "https://files.pythonhosted.org/packages/67/13/c6700ccc6cc218716bfcda4935e4b2997039869b4ad8a94f364c5a3b8e63/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ce9bf03dad3b46408c08649c6fbd6ca28a9fce0eb32fdfffa6775a13103b5310", size = 2062804, upload-time = "2026-03-09T13:14:32.888Z" }, + { url = "https://files.pythonhosted.org/packages/1b/bd/877056304626943ff0f1f44c08f584300c199b887cb3176cd7e34f1515f1/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:fc4d3f1fb9ca0ae9f97b095963bc6326f1dbfd3779d6679a1e016b9baaa153d3", size = 2597482, upload-time = "2026-03-09T13:14:34.971Z" }, + { url = "https://files.pythonhosted.org/packages/75/19/c60626c47bf0f8ac5dcf72c6c98e266d714f2fbbfd50cf6dab5ede3aaa50/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f443b4825c50a51ee68585522ab4a1d1257fac65896f282b4c6763337ac9f5d2", size = 2394328, upload-time = "2026-03-09T13:14:36.816Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f9/b06c934a6aa8bc91f566bd2a214fd04c30506c2d9e2b6b171953216a65b6/kiwisolver-1.5.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:80aa065ffd378ff784822a6d7c3212f2d5f5e9c3589614b5c228b311fd3063ac", size = 1475913, upload-time = "2026-03-09T13:14:46.247Z" }, + { url = "https://files.pythonhosted.org/packages/6b/f0/f768ae564a710135630672981231320bc403cf9152b5596ec5289de0f106/kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e7f886f47ab881692f278ae901039a234e4025a68e6dfab514263a0b1c4ae05", size = 1282782, upload-time = "2026-03-09T13:14:48.458Z" }, + { url = "https://files.pythonhosted.org/packages/e2/9f/1de7aad00697325f05238a5f2eafbd487fb637cc27a558b5367a5f37fb7f/kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5060731cc3ed12ca3a8b57acd4aeca5bbc2f49216dd0bec1650a1acd89486bcd", size = 1300815, upload-time = "2026-03-09T13:14:50.721Z" }, + { url = "https://files.pythonhosted.org/packages/5a/c2/297f25141d2e468e0ce7f7a7b92e0cf8918143a0cbd3422c1ad627e85a06/kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7a4aa69609f40fce3cbc3f87b2061f042eee32f94b8f11db707b66a26461591a", size = 1347925, upload-time = "2026-03-09T13:14:52.304Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d3/f4c73a02eb41520c47610207b21afa8cdd18fdbf64ffd94674ae21c4812d/kiwisolver-1.5.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:d168fda2dbff7b9b5f38e693182d792a938c31db4dac3a80a4888de603c99554", size = 991322, upload-time = "2026-03-09T13:14:54.637Z" }, + { url = "https://files.pythonhosted.org/packages/7b/46/d3f2efef7732fcda98d22bf4ad5d3d71d545167a852ca710a494f4c15343/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:413b820229730d358efd838ecbab79902fe97094565fdc80ddb6b0a18c18a581", size = 2232857, upload-time = "2026-03-09T13:14:56.471Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ec/2d9756bf2b6d26ae4349b8d3662fb3993f16d80c1f971c179ce862b9dbae/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:5124d1ea754509b09e53738ec185584cc609aae4a3b510aaf4ed6aa047ef9303", size = 2329376, upload-time = "2026-03-09T13:14:58.072Z" }, + { url = "https://files.pythonhosted.org/packages/8f/9f/876a0a0f2260f1bde92e002b3019a5fabc35e0939c7d945e0fa66185eb20/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e4415a8db000bf49a6dd1c478bf70062eaacff0f462b92b0ba68791a905861f9", size = 1982549, upload-time = "2026-03-09T13:14:59.668Z" }, + { url = "https://files.pythonhosted.org/packages/6c/4f/ba3624dfac23a64d54ac4179832860cb537c1b0af06024936e82ca4154a0/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d618fd27420381a4f6044faa71f46d8bfd911bd077c555f7138ed88729bfbe79", size = 2494680, upload-time = "2026-03-09T13:15:01.364Z" }, + { url = "https://files.pythonhosted.org/packages/39/b7/97716b190ab98911b20d10bf92eca469121ec483b8ce0edd314f51bc85af/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5092eb5b1172947f57d6ea7d89b2f29650414e4293c47707eb499ec07a0ac796", size = 2297905, upload-time = "2026-03-09T13:15:03.925Z" }, + { url = "https://files.pythonhosted.org/packages/f8/8a/685b297052dd041dcebce8e8787b58923b6e78acc6115a0dc9189011c44b/kiwisolver-1.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7c4c09a490dc4d4a7f8cbee56c606a320f9dc28cf92a7157a39d1ce7676a657", size = 1584858, upload-time = "2026-03-09T13:15:15.103Z" }, + { url = "https://files.pythonhosted.org/packages/9e/80/04865e3d4638ac5bddec28908916df4a3075b8c6cc101786a96803188b96/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a075bd7bd19c70cf67c8badfa36cf7c5d8de3c9ddb8420c51e10d9c50e94920", size = 1392539, upload-time = "2026-03-09T13:15:16.661Z" }, + { url = "https://files.pythonhosted.org/packages/ba/01/77a19cacc0893fa13fafa46d1bba06fb4dc2360b3292baf4b56d8e067b24/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bdd3e53429ff02aa319ba59dfe4ceeec345bf46cf180ec2cf6fd5b942e7975e9", size = 1405310, upload-time = "2026-03-09T13:15:18.229Z" }, + { url = "https://files.pythonhosted.org/packages/53/39/bcaf5d0cca50e604cfa9b4e3ae1d64b50ca1ae5b754122396084599ef903/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cdcb35dc9d807259c981a85531048ede628eabcffb3239adf3d17463518992d", size = 1456244, upload-time = "2026-03-09T13:15:20.444Z" }, + { url = "https://files.pythonhosted.org/packages/d0/7a/72c187abc6975f6978c3e39b7cf67aeb8b3c0a8f9790aa7fd412855e9e1f/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:70d593af6a6ca332d1df73d519fddb5148edb15cd90d5f0155e3746a6d4fcc65", size = 1073154, upload-time = "2026-03-09T13:15:22.039Z" }, + { url = "https://files.pythonhosted.org/packages/c7/ca/cf5b25783ebbd59143b4371ed0c8428a278abe68d6d0104b01865b1bbd0f/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:377815a8616074cabbf3f53354e1d040c35815a134e01d7614b7692e4bf8acfa", size = 2334377, upload-time = "2026-03-09T13:15:23.741Z" }, + { url = "https://files.pythonhosted.org/packages/4a/e5/b1f492adc516796e88751282276745340e2a72dcd0d36cf7173e0daf3210/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0255a027391d52944eae1dbb5d4cc5903f57092f3674e8e544cdd2622826b3f0", size = 2425288, upload-time = "2026-03-09T13:15:25.789Z" }, + { url = "https://files.pythonhosted.org/packages/e6/e5/9b21fbe91a61b8f409d74a26498706e97a48008bfcd1864373d32a6ba31c/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:012b1eb16e28718fa782b5e61dc6f2da1f0792ca73bd05d54de6cb9561665fc9", size = 2063158, upload-time = "2026-03-09T13:15:27.63Z" }, + { url = "https://files.pythonhosted.org/packages/b1/02/83f47986138310f95ea95531f851b2a62227c11cbc3e690ae1374fe49f0f/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:0e3aafb33aed7479377e5e9a82e9d4bf87063741fc99fc7ae48b0f16e32bdd6f", size = 2597260, upload-time = "2026-03-09T13:15:29.421Z" }, + { url = "https://files.pythonhosted.org/packages/07/18/43a5f24608d8c313dd189cf838c8e68d75b115567c6279de7796197cfb6a/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e7a116ae737f0000343218c4edf5bd45893bfeaff0993c0b215d7124c9f77646", size = 2394403, upload-time = "2026-03-09T13:15:31.517Z" }, + { url = "https://files.pythonhosted.org/packages/fa/06/7399a607f434119c6e1fdc8ec89a8d51ccccadf3341dee4ead6bd14caaf5/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31c13da98624f957b0fb1b5bae5383b2333c2c3f6793d9825dd5ce79b525cb7", size = 194295, upload-time = "2026-03-09T13:15:38.22Z" }, +] + +[[package]] +name = "lark" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/34/28fff3ab31ccff1fd4f6c7c7b0ceb2b6968d8ea4950663eadcb5720591a0/lark-1.3.1.tar.gz", hash = "sha256:b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905", size = 382732, upload-time = "2025-10-27T18:25:56.653Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, +] + +[[package]] +name = "latexcodec" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/dd/4270b2c5e2ee49316c3859e62293bd2ea8e382339d63ab7bbe9f39c0ec3b/latexcodec-3.0.1.tar.gz", hash = "sha256:e78a6911cd72f9dec35031c6ec23584de6842bfbc4610a9678868d14cdfb0357", size = 31222, upload-time = "2025-06-17T18:47:34.051Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/40/23569737873cc9637fd488606347e9dd92b9fa37ba4fcda1f98ee5219a97/latexcodec-3.0.1-py3-none-any.whl", hash = "sha256:a9eb8200bff693f0437a69581f7579eb6bca25c4193515c09900ce76451e452e", size = 18532, upload-time = "2025-06-17T18:47:30.726Z" }, +] + +[[package]] +name = "lenspack" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astropy", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/de/18cfc64028334a1907d44ba2b40e233a2c7a06d7c1e827acfd1785c0b293/lenspack-1.0.0.tar.gz", hash = "sha256:e528a5409473eb996b39695cccb744db067341eb5fac6f240cd8d0aa669c85ec", size = 32632, upload-time = "2020-09-04T08:33:53.267Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/3d/a93a1332b1412301ddb454f206507b5e0b3679e012471fbbb075ca3b0845/lenspack-1.0.0-py3-none-any.whl", hash = "sha256:f9c632697516bb681e16b3eff66a7210d6355926179b95aa7060c8936c2259cc", size = 37746, upload-time = "2020-09-04T08:33:50.114Z" }, +] + +[[package]] +name = "llvmlite" +version = "0.48.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/a0/acc8ffcd5bdc63df0097e22c719bfcd61b604358343089313a8aebbb24ab/llvmlite-0.48.0.tar.gz", hash = "sha256:543b19f9ef8f3c7c60d1468191e4ee1b1537bf9f8a3d56f64c0ddd98de92edd2", size = 184016, upload-time = "2026-07-02T20:20:05.308Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/f2/72409351db66d0a317ec5087e076f31fb7b773a640db8a90ce6b5cac9edd/llvmlite-0.48.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:416fa4c2c66c2c6dc6d0a402648c19206e548efa0aa1eff01ad5cdad0af8217d", size = 59890118, upload-time = "2026-07-01T18:41:44.886Z" }, + { url = "https://files.pythonhosted.org/packages/3a/27/5ae2f3722606360480707adb47f001ad89df8251d06b14ee80336e660b66/llvmlite-0.48.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f5e5a5131045b72345c71062ea1a91910dde913792b6c9b28ebb2c1c0a712e98", size = 58343459, upload-time = "2026-07-01T18:41:40.306Z" }, + { url = "https://files.pythonhosted.org/packages/1b/4a/90715fa12006d681270b08d881195b6fab3ec39572e048764a1f7f59fed7/llvmlite-0.48.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8761b9e522f55207e24424fcd98370289eec2710bf8e915c82d1053f642450dc", size = 59890120, upload-time = "2026-07-01T18:42:00.748Z" }, + { url = "https://files.pythonhosted.org/packages/70/5e/7b3e20d64650ca3c80af0cdb664ec4b575ec83d9d4dd05bea8bd31f9bbb6/llvmlite-0.48.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2fe5cb59b2063bfa039dcb8ca6481c0181bf552f340d10dcf61d7996a665556e", size = 58343457, upload-time = "2026-07-01T18:41:56.41Z" }, + { url = "https://files.pythonhosted.org/packages/d6/e1/05b50692b647cac3c18200ac485b04f342f00ed173c9cc46767274469a15/llvmlite-0.48.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:05f0103c8f2f96a37441337e3643863c01b8e83e530aff38960dcb383c54a065", size = 59890115, upload-time = "2026-07-01T18:42:17.805Z" }, + { url = "https://files.pythonhosted.org/packages/f7/c3/470b8c4ff9ae2db2f9cf5c3e73de76ed908a32788ae9eb5602d43e6a476b/llvmlite-0.48.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37d66fae72802175b0bfe1ea06e624b51e2d7aee6c3c34bbd09739b8f88e8e0b", size = 58343457, upload-time = "2026-07-01T18:42:13.217Z" }, + { url = "https://files.pythonhosted.org/packages/27/98/a29133b4728671a175f7d616fab8b1c6e1d8c269d1523581d3160697bfb1/llvmlite-0.48.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d9f952dff6c350c529997423d4fa43abae9722a884ac7ffafc37a3af676e7db", size = 59890119, upload-time = "2026-07-01T18:42:33.88Z" }, + { url = "https://files.pythonhosted.org/packages/1a/cf/7aac11a1f1c7ec54b60c7f6814e87561fb6b55b2f290455d7941eb113420/llvmlite-0.48.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:054aa7d46595935565f276cf0c1659b4f10929c996dd4a606875fae26fba2a23", size = 58343460, upload-time = "2026-07-01T18:42:29.545Z" }, +] + +[[package]] +name = "lmfit" +version = "1.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asteval", marker = "sys_platform == 'linux'" }, + { name = "dill", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "uncertainties", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/e5/a35942aed2de95e228728c34609b51fe3ec9182398eac50d288eef313aa2/lmfit-1.3.4.tar.gz", hash = "sha256:3c22c28c43f717f6c5b4a3bd81e893a2149739c26a592c046f2e33c23cfbe497", size = 630720, upload-time = "2025-07-19T20:09:01.876Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/7e/7b91c89a4cf0f543a83be978657afb20c86af6d725253e319589dcc4ce52/lmfit-1.3.4-py3-none-any.whl", hash = "sha256:afce1593b42324d37ae2908249b0c55445e2f4c1a0474ff706a8e2f7b5d949fa", size = 97662, upload-time = "2025-07-19T20:09:00.32Z" }, +] + +[[package]] +name = "locket" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/83/97b29fe05cb6ae28d2dbd30b81e2e402a3eed5f460c26e9eaa5895ceacf5/locket-1.0.0.tar.gz", hash = "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632", size = 4350, upload-time = "2022-04-20T22:04:44.312Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/bc/83e112abc66cd466c6b83f99118035867cecd41802f8d044638aa78a106e/locket-1.0.0-py2.py3-none-any.whl", hash = "sha256:b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3", size = 4398, upload-time = "2022-04-20T22:04:42.23Z" }, +] + +[[package]] +name = "lsstdesc-coord" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "future", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/e0/6806df4cfa54927a2d8e425407611b4fc031dd125bc952b4a31deb743305/lsstdesc_coord-1.3.1.tar.gz", hash = "sha256:60f878c29e1f30a9b50bf60dca3c466dc9cfb2cbc71f0a27c575ced969de57ab", size = 41647, upload-time = "2026-02-13T18:37:32.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/f8/63121ece13d3dc7521e8989b4224389868f092ce13d3aae4c410b0f5de12/lsstdesc_coord-1.3.1-py3-none-any.whl", hash = "sha256:06afe1621ef5da89f51e1225b1016f9ab640f6eb955bbab1231d60342e536336", size = 27105, upload-time = "2026-02-13T18:37:31.214Z" }, +] + +[[package]] +name = "lxml" +version = "6.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/3b/aab6728cae887456f409b4d75e8a01856e4f04bd510de38052a47768b680/lxml-6.1.1.tar.gz", hash = "sha256:ba96ae44888e0185281e937633a743ea90d5a196c6000f82565ebb0580012d40", size = 4197430, upload-time = "2026-05-18T19:19:06.424Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/95/bb63f0fd62e554fe078e1fb3c8fe9083c14ddc7ad7fa178d10e57e071ac7/lxml-6.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c921ba5c51e4e9f63b8b00267d06566e1f63407408a0496da2d1d0bfc819c7fc", size = 4930746, upload-time = "2026-05-18T19:18:29.637Z" }, + { url = "https://files.pythonhosted.org/packages/eb/99/0013e8d9b5960f4f041cf0b73e2f80c23eb5205b1f7bfb20203243651359/lxml-6.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:54a7f95e4de5fb94e2f9f4b9055c6ba33bf3d628fd77a1d647c5923caa2cdcdc", size = 5093723, upload-time = "2026-05-18T19:18:34.168Z" }, + { url = "https://files.pythonhosted.org/packages/29/91/317b332636bfc7bddcff828d41b3307f50043f4b237e40849c333d80fa1a/lxml-6.1.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f2ec43df44b1f76249ee0a615334f9b5b060e1c8bd90e706dad2d14d02f383", size = 5005557, upload-time = "2026-05-18T19:18:39.798Z" }, + { url = "https://files.pythonhosted.org/packages/42/2f/cc9bf06afe70f9c9093ae60855d9759da9db601ec4080f7473319666ffd7/lxml-6.1.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:70ef8a7e102a1508f8121aae5b0867abd663f72c14f0a9c937e6554cb4587b7b", size = 5631036, upload-time = "2026-05-18T19:18:44.858Z" }, + { url = "https://files.pythonhosted.org/packages/08/f6/af32e23e563971ffb0fb86be52bc5be5c2c118858ffc119bf6a9039b173d/lxml-6.1.1-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ebe6af670449830d6d9b752c256a983291c766a1365ba5d5460048f9e33a7818", size = 5240367, upload-time = "2026-05-18T19:18:49.217Z" }, + { url = "https://files.pythonhosted.org/packages/78/83/8555d40948b09ce86f1bd0c68a7ac31d07b1929f92cc1b074006c97ef2d2/lxml-6.1.1-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:27acc820660aaffa4f7c087f29120e12980f7779d56d8492d263170111284740", size = 5350171, upload-time = "2026-05-18T19:18:52.779Z" }, + { url = "https://files.pythonhosted.org/packages/63/75/5d92da93729b7bad783689e6496049fa40927b45bec7bf183c981de3ca70/lxml-6.1.1-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:1db753c9115ec7100d073b744d17e25e88a8f90f5c39b2f5dd878149af59671f", size = 4694874, upload-time = "2026-05-18T19:18:55.139Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b5/3aad415a9a25b822e783f15deeb4dffccf5113030f1afa2222dd929313d9/lxml-6.1.1-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c4f469aebd783bb741c2ecb2a681008fd26bfe5c16a9a72ed5467f834e810df2", size = 5244492, upload-time = "2026-05-18T19:19:01.28Z" }, + { url = "https://files.pythonhosted.org/packages/f1/a1/5fcf7eb9904b80086aa47dcf0027de07b1bb990afad2e6823144c368ae04/lxml-6.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:766b010012d59470072c1816b5b6c69f1d243e5db36ea5968e94accf430a4635", size = 5048232, upload-time = "2026-05-18T19:18:12.67Z" }, + { url = "https://files.pythonhosted.org/packages/77/74/1f601b63c7a69fcdf10fa9b148c81da8442204194f6c55509cc485c786b9/lxml-6.1.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b8d812c6011c08b8111a15e54dd990b8923692d80adf35488bee34026c35accf", size = 4777023, upload-time = "2026-05-18T19:18:15.928Z" }, + { url = "https://files.pythonhosted.org/packages/a2/b9/7a78f51aec95b1bf780d78e12705a9f6533284f8693dc5c0e6724fa53d3f/lxml-6.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:fe0306bd29505a9177aac19f1877174b0e7422c222a59f70b2cd41633448c3dc", size = 5645773, upload-time = "2026-05-18T19:18:23.223Z" }, + { url = "https://files.pythonhosted.org/packages/a5/6e/98a7b7ad54e4e74fa1f20fff776913980619d0ebe5558232d7da6580bdd8/lxml-6.1.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5ba186ad207446c65d3bb3d3e0412b032b1d9f595e59861e2354798c5703d955", size = 5233088, upload-time = "2026-05-18T19:18:31.433Z" }, + { url = "https://files.pythonhosted.org/packages/65/d1/bc0ed2427bf609f2ee10da303a6a226f9c8bce94f945dc29a32ce55de6e4/lxml-6.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aa366a1e55b8ebfe8ca8ddc3cfe75c8ebade181aeb0f661d0cb05986b647f72a", size = 5260995, upload-time = "2026-05-18T19:18:37.091Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ca/ab7bfe2bf4c972af5e7878262845ead3a24a929a9b04bc11c7c1ece6c82a/lxml-6.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb7c9811bfaa8b1ed5ed319f5d370dfbcaa59d52ea64be2a5a85e18195930354", size = 4924139, upload-time = "2026-05-18T19:19:04.873Z" }, + { url = "https://files.pythonhosted.org/packages/6b/55/a0c72851dfee5ecc689f949723a73dea457758912542cb955b108eaf0d8f/lxml-6.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:762ff394d5bd56da0cf034a23dcce4e13923f15321a2adfa2ac00201dc6d3fca", size = 5082329, upload-time = "2026-05-18T19:19:09.728Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b6/0608f7d61a3b96cc67e5648a3d906e31a5082093e10e7be65b3886289938/lxml-6.1.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a088f287f7d8275a33c07f2cac6c50b9319309a0200a39e7e75d80c707723099", size = 4993564, upload-time = "2026-05-18T19:19:13.608Z" }, + { url = "https://files.pythonhosted.org/packages/4c/66/ae227524b066d29d55bf0b453d93d2d793c40218657d643dcbbca13b8faf/lxml-6.1.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e902da4b04e6b52e5893900d4b8ab46068f75f3561f01bf1080957f9fd932ed6", size = 5613467, upload-time = "2026-05-18T19:19:16.228Z" }, + { url = "https://files.pythonhosted.org/packages/a6/76/dbe4a00b50385e40194231dcfe5a12c059de7cf90e89c83407d2b085b719/lxml-6.1.1-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d4962d4c66bf830a7e59ed6cfc17d148149898a3aefa8ec6e59763e6e3ed085", size = 5228304, upload-time = "2026-05-18T19:19:19.354Z" }, + { url = "https://files.pythonhosted.org/packages/1c/01/00b1b8442ed2041793336868ba0b9ea4b13d7da7c085c6404c207a63bf79/lxml-6.1.1-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:581d4c8ae690a6609e64862dd6b7c2489635c2d13907fc2b20f2bc200ff1d21e", size = 5341607, upload-time = "2026-05-18T19:19:22.297Z" }, + { url = "https://files.pythonhosted.org/packages/63/36/1ad29931e9a4638bb707869f01d423a6c815f82152138d1a40dfcfde2b95/lxml-6.1.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:876e1ff5930ed8bf295ec5ef9a8155e9b6b1876bbf1deed8b3a8069311875a8f", size = 4700168, upload-time = "2026-05-18T19:19:25.133Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d1/a9536cecf9be18a0dc72d32bead283a2332d1ffebd2dd3ac70ce444686e5/lxml-6.1.1-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9eb9b5a968f6e0f6d640092a567e14529ff8cea2e29d00da6f78a79fa49f013c", size = 5232487, upload-time = "2026-05-18T19:19:28.603Z" }, + { url = "https://files.pythonhosted.org/packages/0e/77/b4fb1e03bf5d130e879214d3100092e386418807fb74dd0adc4b0a48f351/lxml-6.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:aa49e06d94aba782c6a02eecb7e507969e7e7a41b267f1b359bb35585f295d5b", size = 5044231, upload-time = "2026-05-18T19:18:42.246Z" }, + { url = "https://files.pythonhosted.org/packages/26/4c/d00daeeb0a5530c4028a9232aa1b93db3ef4ed2158c116ea73c79a9765b3/lxml-6.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:70cdfd80589d59e43e18005dd7244e8895e93db8ab6a620b7e23df5445a4e3d2", size = 4769450, upload-time = "2026-05-18T19:18:48.013Z" }, + { url = "https://files.pythonhosted.org/packages/ed/6a/715a3a8d156ce42f29cf014706f5410c2ff3b02267774110fc23266409fe/lxml-6.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:aad9aa39483ed8ec44d6d2e59e5b98a0d80676ef0d92f44bfc374836111f62f5", size = 5635874, upload-time = "2026-05-18T19:18:51.914Z" }, + { url = "https://files.pythonhosted.org/packages/45/37/0544bc21dde2a88f3a17b504e6fc79c0e01d25a33c2f6079724e9e72b9c7/lxml-6.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d49514be2f28d895c38cf9d2b72d7b9a07d00314519f456c0b50b53cfcf4c785", size = 5223987, upload-time = "2026-05-18T19:18:59.715Z" }, + { url = "https://files.pythonhosted.org/packages/4d/f8/f6a5e8185bcb28c2befae3d31f8e3df3b811cb0f47746517a81279fcafe1/lxml-6.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:47402e62c52ff5988c1e8c6c63177f5708bccf48e366dea4e3dcf1e645e04947", size = 5250276, upload-time = "2026-05-18T19:19:03.834Z" }, + { url = "https://files.pythonhosted.org/packages/d2/51/3904907c063451cf8d4a5c9fe0cad95fa1f4ec57f4e3884fa0731bd7a305/lxml-6.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:74a9717fd0d82effef5c2854f0d917231d5324b5a3eb7275c43ac9fa32f97a14", size = 4950022, upload-time = "2026-05-18T19:19:31.958Z" }, + { url = "https://files.pythonhosted.org/packages/94/cd/9c7611a51c37a2830928405817cc5d56a97f64fab83cc3f628748b135749/lxml-6.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:efe0374196335f93b53269acd811b944f2e6bdc88e8894f214bd636455484909", size = 5086695, upload-time = "2026-05-18T19:19:34.764Z" }, + { url = "https://files.pythonhosted.org/packages/da/d6/24e3b5906abb0b674ff2ae195bc3ce59708df2bcd17cf17703b2d7dd643a/lxml-6.1.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac931cdc9442c1763b8a8f6cd62c0c938737eafc5be75eff88df55fc73bc0d00", size = 5031642, upload-time = "2026-05-18T19:19:37.771Z" }, + { url = "https://files.pythonhosted.org/packages/2d/db/6ec54f99019838bff54785c51da07f189eb4676861c5f2730962b0d8d665/lxml-6.1.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aee395f5d0927f947758b4ec119fd5fc8ec71f07a1c5c52077b30b04c0fa6955", size = 5647338, upload-time = "2026-05-18T19:19:40.553Z" }, + { url = "https://files.pythonhosted.org/packages/42/3d/ef4dcfffd22d27a61805d8ed9f7fb888495bc6aa88648fa07c1eaa5586b6/lxml-6.1.1-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9395002973c827b3ed67db77e6ec09f092919a587022174554096a269378fb13", size = 5239528, upload-time = "2026-05-18T19:19:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/62/bb/37fb3f0dff146bdcfa78eec47879273820b2a0bf350ec236ce14bd0b1c26/lxml-6.1.1-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:73bc2086f141224ebddb7fc5c6a36ca58b31b94b561e1dfe8e073e3270fad1e7", size = 5350730, upload-time = "2026-05-18T19:19:46.307Z" }, + { url = "https://files.pythonhosted.org/packages/90/42/43253f168388df4fae1f38c01df36ddb9bee39e2048167b54cdcbae85ea3/lxml-6.1.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3779def59032b81e44a5f70096ef6bf2082f8d901937dca354474ba09782e245", size = 4697530, upload-time = "2026-05-18T19:19:49.889Z" }, + { url = "https://files.pythonhosted.org/packages/eb/a8/c5a8504f81bbdfc8e7094c2c850cdb4ed6777fc4d5ddd9e5ab819f3b0d54/lxml-6.1.1-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:86c89b9d55ebf820ad7c90bc533410f0d098054f293351f10603c0c46ff598f5", size = 5250670, upload-time = "2026-05-18T19:19:53.199Z" }, + { url = "https://files.pythonhosted.org/packages/77/b7/c7e76ab18744d75e21f320ebf9ff9d1ceae2b54dd431ea5a64caf26c9672/lxml-6.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19607c6bbff2a44cf3fe8250abccd20942d3462473e0a721d01d379ed017e462", size = 5084485, upload-time = "2026-05-18T19:19:08.422Z" }, + { url = "https://files.pythonhosted.org/packages/31/31/b35c53f8ef7b7c31cacd23d3638652fff7bcd1deb6eedb709ab43b685908/lxml-6.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c6ed5141a5c7507cf3ee76bd363b0d6f801e3321adc35b5d825a23115faa5465", size = 4737635, upload-time = "2026-05-18T19:19:12.321Z" }, + { url = "https://files.pythonhosted.org/packages/d9/06/31f23c813a7fe8e0cb1b175e915b08c9bf4e86d225b210feadbdbe519667/lxml-6.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:62aeb7e85b5d60320b9d77eef2e773994e2c0ce10121b277e0a19804e1654a5a", size = 5670681, upload-time = "2026-05-18T19:19:15.001Z" }, + { url = "https://files.pythonhosted.org/packages/1a/bc/ce619bccc89b1fd9ad8a8e1330ee3f3beff9f2ff95b712d7bbcdd6e22fc3/lxml-6.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b1b963fd8f5caa68e99dfae060d54de1fe9cba899b8718b44a00cdca53c3e590", size = 5238229, upload-time = "2026-05-18T19:19:18.131Z" }, + { url = "https://files.pythonhosted.org/packages/2f/5d/b329acbbedc0b619ebc2be6cf7ee9ed07e80892c88d4dfd612c33805789a/lxml-6.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:63876be28efefa04a1df615b46770e82042cce445cfdce55160522f57b231ccb", size = 5264191, upload-time = "2026-05-18T19:19:21.118Z" }, + { url = "https://files.pythonhosted.org/packages/16/02/6f7061f4f95f51e545d48e87647c54791d204a4e881be4156e7a26ba5338/lxml-6.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:acd7d70b64c0aae0c7922cca83d288a16f5f6da523637697872253415269baef", size = 4970291, upload-time = "2026-05-18T19:19:56.215Z" }, + { url = "https://files.pythonhosted.org/packages/b0/02/55fc057d8283427dea7d6edb102e7a840239c77a64a983d92f62a304c0e9/lxml-6.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4f0dd2f01f9f8a89f565d000e03abcf0a13d692a346c8d22f628d49af098777a", size = 5102822, upload-time = "2026-05-18T19:19:59.223Z" }, + { url = "https://files.pythonhosted.org/packages/e4/48/8e1cf78d89d66850121d9255a2a24414c98f775da93b90cf976956c24b14/lxml-6.1.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b7e8a14c8634bf6f7a568634cb395305a6d964aeb5b7ee32248094bed3a7e2c", size = 5027923, upload-time = "2026-05-18T19:20:01.549Z" }, + { url = "https://files.pythonhosted.org/packages/ed/00/0632a0647612c8af24d26997b3b961397daa9d5b2581444805933629a4cb/lxml-6.1.1-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:86281fbdd6a8162756f8d603f37e3435bfa38043adb79c6dc6a2dfee065e7525", size = 5595843, upload-time = "2026-05-18T19:20:03.93Z" }, + { url = "https://files.pythonhosted.org/packages/bc/86/ab008a7dc360711b66858d61c80a5979a70a09f2aa2b05d9698df80b803d/lxml-6.1.1-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5d7152ec39ca7c402d8fb9bad86140a15b9503bd0c54484e3f1bbe3dd37ceca", size = 5224515, upload-time = "2026-05-18T19:20:06.381Z" }, + { url = "https://files.pythonhosted.org/packages/75/c6/2702ff375e728e34f56d9a45339a9cf7e4427e917f542225242d63a05afa/lxml-6.1.1-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:88d8cb75b9d82858497a5393e3c63cfbf03035225e4b35a49ed7ccb151e4dc0e", size = 5312511, upload-time = "2026-05-18T19:20:09.308Z" }, + { url = "https://files.pythonhosted.org/packages/b7/57/a5807c98f87a86f10ef9ffab35516df7c0f0c4b6d5d33e9f608ab9c04a31/lxml-6.1.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f64ec5397ea6a41fc1b4af0380d79b44a755b5531dcaccd9940fb260dca93038", size = 4639206, upload-time = "2026-05-18T19:20:11.704Z" }, + { url = "https://files.pythonhosted.org/packages/1f/e1/8a0a2c35734812395f4da4eaf33748a7e5705bfb2a58b128da764339d5ec/lxml-6.1.1-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d34bbf07dbc7ca5970671b1512e928991fb5e9d95365636c9b2d8b4f53af405e", size = 5232404, upload-time = "2026-05-18T19:20:14.064Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e2/0e6a4dd5ad84d01d99aa7bae7cfefd4a760a0e0f8176818241de17d9b6c0/lxml-6.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:17e0e18d4ad8adbd0399291bc44845b69d9dd68439a3cdebdf35ff902ec05072", size = 5083769, upload-time = "2026-05-18T19:19:23.758Z" }, + { url = "https://files.pythonhosted.org/packages/a0/7e/161f33d463f6ffc1c7679104b65086dea120080d49dde4d238f015aaee2f/lxml-6.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:3ab541146f1f6968c462d6c2ac495148e8cdba2f8347700b2141b6ec5a75bf52", size = 4758936, upload-time = "2026-05-18T19:19:27.256Z" }, + { url = "https://files.pythonhosted.org/packages/f1/fb/2369825e3f6ca99305bf9f7b7085fda91c8b0922a89e54d900974aa3ef85/lxml-6.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2a0217714657e023ef4293500f65aa20fce6164c8fd6b08fa5bd4a859fb14b9b", size = 5620296, upload-time = "2026-05-18T19:19:29.993Z" }, + { url = "https://files.pythonhosted.org/packages/30/90/d61e383146f74c5ab683947ea14dc7b82778838ab9b95ea73a23b60d0191/lxml-6.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:05a82eb6e1530a64f26225b55cbd178113bd0b5af1c2b625f25e5296742c26d2", size = 5228598, upload-time = "2026-05-18T19:19:33.523Z" }, + { url = "https://files.pythonhosted.org/packages/76/2d/2dafd8149e94b05bb070690efd5bb2680720681e03ff03fc57d2b70a1105/lxml-6.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9e36f163528fc50cbef305f02a5fd66d404edf7049cdaff211dbc2cba5a7013e", size = 5247845, upload-time = "2026-05-18T19:19:36.649Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, +] + +[[package]] +name = "matplotlib" +version = "3.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "contourpy", marker = "sys_platform == 'linux'" }, + { name = "cycler", marker = "sys_platform == 'linux'" }, + { name = "fonttools", marker = "sys_platform == 'linux'" }, + { name = "kiwisolver", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "pillow", marker = "sys_platform == 'linux'" }, + { name = "pyparsing", marker = "sys_platform == 'linux'" }, + { name = "python-dateutil", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/24/080c99d223d158d3a8902769269ab6da5b50f7a0e6e072513907e02b7a6c/matplotlib-3.11.0.tar.gz", hash = "sha256:68c0c7be01b30dcca3638934f7f591df73401235cbdbf0d1ab1c71e7db7f8b57", size = 33251176, upload-time = "2026-06-12T02:29:15.508Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/95/7f522393c88313336b20d70fc849555757b2e5febc22b83b3a3f0fd4bce9/matplotlib-3.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be5f93a1d21981bfb802ded0d77a0caa92d4342a47d45754fac77e314a506344", size = 10031353, upload-time = "2026-06-12T02:27:55.215Z" }, + { url = "https://files.pythonhosted.org/packages/87/ce/8f25a0e3186aefd61913e7467d1b999465bcd0d0c03ac695c1b26ca559b7/matplotlib-3.11.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41635d7909d19e52e924a521dde6d8f670b0f53ab1d0e8c331fa831554f681d1", size = 10839232, upload-time = "2026-06-12T02:27:57.746Z" }, + { url = "https://files.pythonhosted.org/packages/85/c2/db15da2bbdf9e3ca66df7db8e2c33a1dfed67be24a24d2c878efaaff01d6/matplotlib-3.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:94f5000f67ca9faa300863ea17f8bce9175cb67b88bec4bc7780502d53dd7c9e", size = 10923899, upload-time = "2026-06-12T02:28:00.223Z" }, + { url = "https://files.pythonhosted.org/packages/a4/c0/1117d53077e3ac3152503a84e9cf7a5c239576805ee71276e80c2aaa7471/matplotlib-3.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be152b7570324dc8d01574cc9474dd2d803237acf528bcbb5b211fa347461a09", size = 10031623, upload-time = "2026-06-12T02:28:11.26Z" }, + { url = "https://files.pythonhosted.org/packages/92/7e/e937138daffad65b71bf831a377809dcbc830fb4f31a31e067dc1faa2575/matplotlib-3.11.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:126f256df600652d7e4b394cf3164ff75210a00038f287c95a012a6f58d0e83f", size = 10839372, upload-time = "2026-06-12T02:28:14.102Z" }, + { url = "https://files.pythonhosted.org/packages/1d/c2/438ecc197ffb8023b6b9922915542f2172f5fd45b76703b0b4fc47322243/matplotlib-3.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:03acfeddf87b0dddb11b081ef7740ad445a3ca8bcb6b8e3011b08f2cf802b75c", size = 10924099, upload-time = "2026-06-12T02:28:16.383Z" }, + { url = "https://files.pythonhosted.org/packages/95/e4/b4b7c33151e74e5c802f3cde1ba807ebfc38401e329b44e215a5888dd76d/matplotlib-3.11.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:565af866fd63e4bd3f987d580afe27c44c2552a3b3305f4ecbb85133601ea6f3", size = 10045491, upload-time = "2026-06-12T02:28:27.141Z" }, + { url = "https://files.pythonhosted.org/packages/71/28/394548efd68354110c1a1be11fe6b6e559e06d1a23da35908a0e316c55a9/matplotlib-3.11.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b3e64dea5062c570f04358e2711859f3531b459f29516274fbad889079e4f3", size = 10857059, upload-time = "2026-06-12T02:28:29.222Z" }, + { url = "https://files.pythonhosted.org/packages/c8/44/e7922e6e2a4d63bdfbc9dc4a53e3850ab438d46cf42e6779bb15ec92c948/matplotlib-3.11.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:942b37c5db1899610bd1543ce8e13e4ecff9a4633e7f63bb6aa9205d2644ebd1", size = 10939576, upload-time = "2026-06-12T02:28:31.66Z" }, + { url = "https://files.pythonhosted.org/packages/b7/2d/4e1240ea82ee197dfb3851e71f71c87eeeb975f1753b56a0588e4e80739a/matplotlib-3.11.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9695457a467ff86d23f35037a43deb6f1134dd6d3e2ac8ce1e2087cff09ffb9", size = 10843005, upload-time = "2026-06-12T02:28:42.39Z" }, + { url = "https://files.pythonhosted.org/packages/29/dc/6377ecfaa5fef79430f74a1a16638b4e2aa30d4692bae2c19f9d76fe3b01/matplotlib-3.11.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19c16c61dea63b3582918503e6b294193961261d9daa806d4ae2151f1ad05430", size = 11127459, upload-time = "2026-06-12T02:28:44.483Z" }, + { url = "https://files.pythonhosted.org/packages/6f/41/795c405aa7560443a3b01309424cde4a1113b85c90b8a63417444a749617/matplotlib-3.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2d72ea8b7924f3cb955e61518d21e43b3df1e6c8a793b480a0c1214f185d30ba", size = 10925160, upload-time = "2026-06-12T02:28:46.564Z" }, + { url = "https://files.pythonhosted.org/packages/4b/cb/22f6bc35711a0b5639a784e74e653e77c86210bd4304449dd399a482f74e/matplotlib-3.11.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfabef0230d0697aa0d717385194dd41162e00207a68bf4abf94c2bf4c27dca0", size = 10854181, upload-time = "2026-06-12T02:28:57.856Z" }, + { url = "https://files.pythonhosted.org/packages/3f/7e/9a9eaca731a2939589da520f0ebe8fd8753d0f51fca98c7d20af6dbe261a/matplotlib-3.11.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1644db30e759199443493ac5e5caec24fdb775a8f6123021f85ba47c4133c3cb", size = 11137715, upload-time = "2026-06-12T02:29:00.555Z" }, + { url = "https://files.pythonhosted.org/packages/ef/f9/9b030b6088354acb0296871bb624b25befc1c42509d3c6cd17420c83a5b8/matplotlib-3.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:15b0d160079cb10699a0e98b5989c70677b2df7cacdc62af67c30f2facec46d9", size = 10939427, upload-time = "2026-06-12T02:29:02.527Z" }, +] + +[[package]] +name = "matplotlib-inline" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "traitlets", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/c0/9f7c9a46090390368a4d7bcb76bb87a4a36c421e4c0792cdb53486ffac7a/matplotlib_inline-0.2.2.tar.gz", hash = "sha256:72f3fe8fce36b70d4a5b612f899090cd0401deddc4ea90e1572b9f4bfb058c79", size = 8150, upload-time = "2026-05-08T17:33:33.49Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" }, +] + +[[package]] +name = "mdit-py-plugins" +version = "0.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/59/fc/f8d0863f8862f25602c0404d75568e89fb6b4109804645e5cdfb1be5cf56/mdit_py_plugins-0.6.1.tar.gz", hash = "sha256:a2bca0f039f39dbd35fb74ae1b5f998608c437463371f0ff7f49a19a17a114d0", size = 56114, upload-time = "2026-05-13T09:03:38.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/69/6da5581c6a7fede7dc261bf4e67d6adca4196f176b43288b55b3db395b6e/mdit_py_plugins-0.6.1-py3-none-any.whl", hash = "sha256:214c82fb2ac524472ab6a5bcab1de80f73b50443e187f401bfd77efbc7c6481d", size = 66663, upload-time = "2026-05-13T09:03:37.76Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "mistune" +version = "3.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/a5/2dab368d6950e6808904dec98f54c7e726ee7be4a0c6afe00e6e011bd52d/mistune-3.3.3.tar.gz", hash = "sha256:c4c6c0c840b8637a2e9b8b6d607eb7c8f00888bf14c754409bcd339e848c2477", size = 115363, upload-time = "2026-07-09T06:18:05.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/70/b1e4737b84163db5bb1dfde6f216dbfbf32783330a9989c965e121172830/mistune-3.3.3-py3-none-any.whl", hash = "sha256:99de1585e42dcbd826faa9e11a202727a5e202e4e4722a4c69ac1ff615793dd7", size = 63569, upload-time = "2026-07-09T06:18:03.839Z" }, +] + +[[package]] +name = "more-itertools" +version = "11.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/1d/f4da6f02cdffe04d6362210b807146a26044c88d839208aec273bb0d9184/more_itertools-11.1.0.tar.gz", hash = "sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d", size = 145772, upload-time = "2026-05-22T14:14:29.909Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192", size = 72226, upload-time = "2026-05-22T14:14:28.824Z" }, +] + +[[package]] +name = "mpi4py" +version = "4.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/75/83/231445bbcf7ef10864746c244ff2d82000011449b79275642c5d4ed8c8f4/mpi4py-4.1.2.tar.gz", hash = "sha256:56860286dc45f20e8821e93cb06669e30462348bf866f685553fa4b712d58d02", size = 501709, upload-time = "2026-05-16T10:35:23.618Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/0a/1da7f403e0d8ce0e26d541f7538302cec00cf5b0a98a7a52b929f938a25c/mpi4py-4.1.2-cp310-abi3-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2ef63b2e3083e6062fd90e4de8c4e3acbf81e0772406e0226eb8dde6a48cab8e", size = 1327130, upload-time = "2026-05-16T10:34:00.269Z" }, + { url = "https://files.pythonhosted.org/packages/e6/f9/65999152ae82bad914c6a083821ee774afefd6d0544e633b940c9a9ebf3f/mpi4py-4.1.2-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6508e654b9c8ff9f611b19548b2a17d1e323b520a15168189f92221e6757b8ff", size = 1182268, upload-time = "2026-05-16T10:34:02.206Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2b/1e48c4c5f9acbdca8dd28beeba9123dde140cd2ca520f8e3a3cf22faeeaa/mpi4py-4.1.2-cp312-cp312-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:00f4cce8999d19f35243c3442ea22debbe3336f69c309cd5d3176df4e51c717a", size = 1358844, upload-time = "2026-05-16T10:34:28.247Z" }, + { url = "https://files.pythonhosted.org/packages/94/46/a37225d47997fcf30adca25d3849d035bbb61d972118b024db900306e528/mpi4py-4.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0a58e164776acb7b52414548b1bf0e5caafce0ee90345deff147873a64b6b2cc", size = 1227206, upload-time = "2026-05-16T10:34:29.866Z" }, + { url = "https://files.pythonhosted.org/packages/0d/5e/493415cdb0da0c1c0b6ec9a1fb65ab57a174e8111b25c87f7507f663d0b4/mpi4py-4.1.2-cp313-cp313-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:085e0cb05427398fe2281856f27a87984b9f234cd6d98a2a384a6fbfe679a56f", size = 1358571, upload-time = "2026-05-16T10:34:37.478Z" }, + { url = "https://files.pythonhosted.org/packages/6f/51/3822e834fc9cafd96811501b85232e02bd9b31596d42965536a269a6c112/mpi4py-4.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:429437883b4511583d56a57bc58ad0d02965fb6c19f84f39e30c0646e6c0cab9", size = 1226562, upload-time = "2026-05-16T10:34:39.652Z" }, + { url = "https://files.pythonhosted.org/packages/ec/5e/d358fadb8672d58abd6dce16c95eda56f497b378dec5642a31cd6ededfc4/mpi4py-4.1.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:85e332feaac3323d8ed1c71f17478fe3f529468f28f6f9bb4c9133ad2d3a3a6d", size = 1419007, upload-time = "2026-05-16T10:34:45.47Z" }, + { url = "https://files.pythonhosted.org/packages/27/eb/5cd53880337009cab9a9d17a007ad5aec731f3a55e211bdcc99dbb98a0e3/mpi4py-4.1.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e05dba75f6ad17ae761bdc01ee5c5271d15cd296e8a508e22c99c91b540dc99", size = 1298482, upload-time = "2026-05-16T10:34:46.858Z" }, + { url = "https://files.pythonhosted.org/packages/26/e4/32575e9b0d5380b08ffa04958c7a13b1bab86e3674f33b6de3827fbbe14c/mpi4py-4.1.2-cp314-cp314-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d53e8f7182bf125b37155f679b301a21cd64e5ae6861ca0ab989c9d2b2073bbd", size = 1368388, upload-time = "2026-05-16T10:34:53.665Z" }, + { url = "https://files.pythonhosted.org/packages/87/a8/ad5e925da9de402704ecfe8715348b068fb0b65059109c65dafb111f95e4/mpi4py-4.1.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cbf982d3425fd06890fbcff79ba19a7435e93bc74e5711ca8ac0b64d0c3ece9", size = 1238211, upload-time = "2026-05-16T10:34:55.094Z" }, + { url = "https://files.pythonhosted.org/packages/39/29/45a41d081342896af52d38a848017728bf6834c4c92814c50aa791dc750b/mpi4py-4.1.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:db189238c37be98933eeb078a3e2b827eef969837e8fe0f3a2e6cdb7f4b1b05f", size = 1423092, upload-time = "2026-05-16T10:35:01.672Z" }, + { url = "https://files.pythonhosted.org/packages/0f/6f/c2127d426d87f2f7cdb4beacbb4febe7b0c93047a3c615d0e56cecb42563/mpi4py-4.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d0fbf0bc100ed966c3d7f44231a19aef0025d523d1b813de468a1b114d0d9436", size = 1303134, upload-time = "2026-05-16T10:35:03.122Z" }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + +[[package]] +name = "msgpack" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/31/f9/c0a1c127f9049db9155afc316952ea571720dd01833ff5e4d7e8e6352dbb/msgpack-1.2.1.tar.gz", hash = "sha256:04c721c2c7448767e9e3f2520a475663d8ee0f09c31890f6d2bd70fd636a9647", size = 183960, upload-time = "2026-06-18T16:13:52.594Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/aa/753ad8b007b464e1d8aa0c8e650b9c5f4f725e658fc5ac8a7635c55b7f6e/msgpack-1.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60926b75d00c8e816ef98f3034f484a8bc64242d66839cef4cf7e503142316a0", size = 410634, upload-time = "2026-06-18T16:13:00.383Z" }, + { url = "https://files.pythonhosted.org/packages/6a/fd/6adabd4f6d5e686f97dd02ce7fce3fe4cf672cbac36b8f67ff4040e8ad8b/msgpack-1.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:020e881a764b20d8d7ca1a54fc01b8175519d108e3c3f194fddc200bda95951a", size = 419989, upload-time = "2026-06-18T16:13:01.776Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cc/85039b7b0eb168aaad7383a23c97e291a11f08351cb45a606ce865e4e3f1/msgpack-1.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4202c74688ca06591f78cb18988228bd4cca2cc75d57b60008372892d2f1e6e6", size = 377544, upload-time = "2026-06-18T16:13:03.637Z" }, + { url = "https://files.pythonhosted.org/packages/ed/bf/35963899493b32030c85fc513b723ae66144ac70c11ebc52e889e16e3d99/msgpack-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8b267ce94efb76fbd1b3373511420074ee3187f0f7811bf394531de13294735a", size = 400842, upload-time = "2026-06-18T16:13:05.012Z" }, + { url = "https://files.pythonhosted.org/packages/a6/df/8e2ac970c8f99264cd9997d1c73df5466bc19da3301d7dc5500862a9b089/msgpack-1.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f1d0f8f98ade9634e01fb704a408f9336c0a8f1117b369f5db83dc7551d8b1", size = 374108, upload-time = "2026-06-18T16:13:06.232Z" }, + { url = "https://files.pythonhosted.org/packages/17/dd/fa8bd265110dfa51c20cb529f9e6d240a16fafe7e645004c6af2d01353ba/msgpack-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f02cf17a6ca1abe29b5f980644f7551f94d71f2011509b26d8625ce038f0df64", size = 414939, upload-time = "2026-06-18T16:13:07.478Z" }, + { url = "https://files.pythonhosted.org/packages/e3/10/ddf7b06db879e8792d13934ddda09ff20bd2a583fd84c9b59aae9b0e650b/msgpack-1.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f310233ef7fb9c14e201c93639fe5f5260b005f56f0b29048e999c30935596cc", size = 407518, upload-time = "2026-06-18T16:13:14.233Z" }, + { url = "https://files.pythonhosted.org/packages/79/d3/36a46a8ed992b781acbc05928bd5bee3c810cb0c3563bf81a7b0c04a1a76/msgpack-1.2.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:787c9bebb5833e8f6fc8abca3c0597683d8d87f56a8842b6b89c75a5f3176e2d", size = 416405, upload-time = "2026-06-18T16:13:15.435Z" }, + { url = "https://files.pythonhosted.org/packages/f9/84/e8e9598b557c0ba6ddae901a73780a4c75ac667dddf59414b1e56a42fb34/msgpack-1.2.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dc871b997a9370d855b7394465f2f350e847a5b806dd38dcc9c989e7d87da155", size = 376257, upload-time = "2026-06-18T16:13:17.022Z" }, + { url = "https://files.pythonhosted.org/packages/40/16/738fe6d875ad7e2a9429c165322a4ec088f4f273cdfae63d96a89c467961/msgpack-1.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:85f57e960d877f2977f6430896191b04a21f8901b3b4baf2e4604329f4db5402", size = 397469, upload-time = "2026-06-18T16:13:18.287Z" }, + { url = "https://files.pythonhosted.org/packages/ca/be/6d5952df75a7f24f35833af764c3a6860780364cb3a0030beb8099e1b2b4/msgpack-1.2.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1233ee2dd0cefba127583de50ea654677277047d238303521db35def3d7b2e7c", size = 372802, upload-time = "2026-06-18T16:13:19.685Z" }, + { url = "https://files.pythonhosted.org/packages/e1/39/e2ef7dbf0473bcb8dc7c50bf782a892d67414877b63e47fc88eb189ef5e6/msgpack-1.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e3dc2feb0876209d9c38aa56cb1de169bd6c4348f1aa48271f241226590993e6", size = 411273, upload-time = "2026-06-18T16:13:21.028Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a6/99e58722feaffc5f2fbcc0c8c0d1451ab9f84097f7af87291b46af2390f4/msgpack-1.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca0dacff965c47afdc3749a8469d7302a8f801d6a28758d55120d75e66ce6889", size = 406090, upload-time = "2026-06-18T16:13:28.072Z" }, + { url = "https://files.pythonhosted.org/packages/19/03/8c63e8cf52958534ef688625965ab04c269a6cadd8caef16758b380a821a/msgpack-1.2.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e2bf9280bceb5efca998435904b5d3e9fdbcc11d90dc9df30aec7973252b720", size = 412106, upload-time = "2026-06-18T16:13:29.427Z" }, + { url = "https://files.pythonhosted.org/packages/63/d2/155d9e71b40e41fd934bc0c48b9b2770f22263e1ac20aad8e29fdca7be3f/msgpack-1.2.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6c4be5d1c02a42b066ca6ddb71adf36432868fdcdb6ee87e634e86e0674190", size = 374851, upload-time = "2026-06-18T16:13:30.631Z" }, + { url = "https://files.pythonhosted.org/packages/98/48/deaf2326262a8d5ea3295ce9649912ecd3f551ba7ec8e33c665d2ba583f3/msgpack-1.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec0e675d59150a6269ddc9139087c722292664a37d071a849c05c473350f1f2d", size = 396168, upload-time = "2026-06-18T16:13:31.977Z" }, + { url = "https://files.pythonhosted.org/packages/10/2a/b4410f906c2ec0008f1608d3ab5143afc3ad3f4e6da0fed3ea2231d0bef4/msgpack-1.2.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:dd3bfe82d53edfe4b7fc9a7ec9761e23a7a5b1dac22264505af428253c29ed24", size = 371959, upload-time = "2026-06-18T16:13:33.282Z" }, + { url = "https://files.pythonhosted.org/packages/59/86/1edc67270099a528fa2093ea60fe191233cd238e4bd30cfacf7db79fc959/msgpack-1.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5ad5467fc3f68b5468e06c5f788d712e9f8ffc8b0cd1bcb160c105c1ee92dae7", size = 408457, upload-time = "2026-06-18T16:13:34.567Z" }, + { url = "https://files.pythonhosted.org/packages/a9/b4/b774d7eb95561739907fec675582f83203cf41c597a418c2589b4bfb8e9d/msgpack-1.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20466cca18c49c7292a8984bc15d65857b171e7264bdcb5f96baf8be238791fc", size = 427661, upload-time = "2026-06-18T16:13:41.574Z" }, + { url = "https://files.pythonhosted.org/packages/b2/f9/3243191dc9937e00756c8bc1b0272fed8f23758e43df2a3b46f533e5090f/msgpack-1.2.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:196300e7e5d6e74d50f1607ab9c06c4a1484c383cd22defd727902591f7e8dde", size = 426375, upload-time = "2026-06-18T16:13:42.936Z" }, + { url = "https://files.pythonhosted.org/packages/23/c7/1693111db9944ba4ad4b67a1e788400d78a0b6af7a6523dc7e4e58f8274b/msgpack-1.2.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:575957e79cd51903a4e8495a242442949641e08f1efd5197b43bebd3ea7682b4", size = 380495, upload-time = "2026-06-18T16:13:44.306Z" }, + { url = "https://files.pythonhosted.org/packages/3e/2b/92f86956a0c13e8662f7e2ad630c4eb4db07497b967589bd5245e018b2c1/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8c2ed1e48cc0f460bf3c7780e7137ff21a4e18433451916f2442c1b21036cd7d", size = 410897, upload-time = "2026-06-18T16:13:45.629Z" }, + { url = "https://files.pythonhosted.org/packages/da/ea/1479f72d200313a76fc2f823a79d1e07ed052ab7b8a0280640aa7b95de42/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5f6277e5f783c36786a145e0247fc189a03f35f84b251646e53592d2bc12b355", size = 378519, upload-time = "2026-06-18T16:13:46.998Z" }, + { url = "https://files.pythonhosted.org/packages/f5/4d/fa006060ffa1011d32bfae826fe766fe73e02982183601633b7121058ab3/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f9389552ecf4784886345ead0647e4edc96bee37cbab05b75540f542f766c48c", size = 419815, upload-time = "2026-06-18T16:13:48.205Z" }, +] + +[[package]] +name = "myst-parser" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils", marker = "sys_platform == 'linux'" }, + { name = "jinja2", marker = "sys_platform == 'linux'" }, + { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, + { name = "mdit-py-plugins", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "sphinx", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/dc/603751677fff302f34396e206b610f556a59d7fe58b9a2145f54e96b48e8/myst_parser-5.1.0.tar.gz", hash = "sha256:ab69322dc6719dcc7f296479dbb70181b66df6ed315064f92dbc85c0e1bf2f02", size = 101182, upload-time = "2026-05-13T09:38:19.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/dc/f3dfb7488b770f3f67e6545085bf2abea5172e88f57b8ad25ef860ca704c/myst_parser-5.1.0-py3-none-any.whl", hash = "sha256:9c91c52b3cdb4d94a6506e4fab4e2f296c7623a0da0dcbe6de1565c3dad67a8a", size = 85817, upload-time = "2026-05-13T09:38:17.904Z" }, +] + +[[package]] +name = "nbclient" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-client", marker = "sys_platform == 'linux'" }, + { name = "jupyter-core", marker = "sys_platform == 'linux'" }, + { name = "nbformat", marker = "sys_platform == 'linux'" }, + { name = "traitlets", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/a5/b3bae4b590c0cbcada2c63a34f7580024e834a8ba213e949a2f906705787/nbclient-0.11.0.tar.gz", hash = "sha256:04a134a5b087f2c5887f228aca155db50169b8cd9334dee6942c8e927e56081a", size = 62535, upload-time = "2026-06-05T07:52:41.746Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/c9/94d73e5a01c5b926c3fa2496e97d7a8dc28ed5a77c0b2ed712f1a62e6694/nbclient-0.11.0-py3-none-any.whl", hash = "sha256:ef7fa0d59d6e1d41103933d8a445a18d5de860ca6b613b87b8574accdb3c2895", size = 25288, upload-time = "2026-06-05T07:52:40.115Z" }, +] + +[[package]] +name = "nbconvert" +version = "7.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4", marker = "sys_platform == 'linux'" }, + { name = "bleach", extra = ["css"], marker = "sys_platform == 'linux'" }, + { name = "defusedxml", marker = "sys_platform == 'linux'" }, + { name = "jinja2", marker = "sys_platform == 'linux'" }, + { name = "jupyter-core", marker = "sys_platform == 'linux'" }, + { name = "jupyterlab-pygments", marker = "sys_platform == 'linux'" }, + { name = "markupsafe", marker = "sys_platform == 'linux'" }, + { name = "mistune", marker = "sys_platform == 'linux'" }, + { name = "nbclient", marker = "sys_platform == 'linux'" }, + { name = "nbformat", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "pandocfilters", marker = "sys_platform == 'linux'" }, + { name = "pygments", marker = "sys_platform == 'linux'" }, + { name = "traitlets", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/b1/708e53fe2e429c103c6e6e159106bcf0357ac41aa4c28772bd8402339051/nbconvert-7.17.1.tar.gz", hash = "sha256:34d0d0a7e73ce3cbab6c5aae8f4f468797280b01fd8bd2ca746da8569eddd7d2", size = 865311, upload-time = "2026-04-08T00:44:14.914Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl", hash = "sha256:aa85c087b435e7bf1ffd03319f658e285f2b89eccab33bc1ba7025495ab3e7c8", size = 261927, upload-time = "2026-04-08T00:44:12.845Z" }, +] + +[[package]] +name = "nbformat" +version = "5.10.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fastjsonschema", marker = "sys_platform == 'linux'" }, + { name = "jsonschema", marker = "sys_platform == 'linux'" }, + { name = "jupyter-core", marker = "sys_platform == 'linux'" }, + { name = "traitlets", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/fd/91545e604bc3dad7dca9ed03284086039b294c6b3d75c0d2fa45f9e9caf3/nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a", size = 142749, upload-time = "2024-04-04T11:20:37.371Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/82/0340caa499416c78e5d8f5f05947ae4bc3cba53c9f038ab6e9ed964e22f1/nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b", size = 78454, upload-time = "2024-04-04T11:20:34.895Z" }, +] + +[[package]] +name = "ndindex" +version = "1.10.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/92/4b9d2f4e0f3eabcfc7b02b48261f6e5ad36a3e2c1bbdcc4e3b7b6c768fa6/ndindex-1.10.1.tar.gz", hash = "sha256:0f6113c1f031248f8818cbee1aa92aa3c9472b7701debcce9fddebcd2f610f11", size = 271395, upload-time = "2025-11-19T20:40:08.899Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/40/139b6b050ba2b2a0bb40e0381a352b1eb6551302dcb8f86fb4c97dd34e92/ndindex-1.10.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:046c1e88d46b2bd2fd3483e06d27b4e85132b55bc693f2fca2db0bb56eea1e78", size = 542901, upload-time = "2025-11-19T20:38:44.43Z" }, + { url = "https://files.pythonhosted.org/packages/27/ae/defd665dbbeb2fffa077491365ed160acaec49274ce8d4b979f55db71f18/ndindex-1.10.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03cf1e6cdac876bd8fc92d3b65bb223496b1581d10eab3ba113f7c195121a959", size = 546875, upload-time = "2025-11-19T20:38:45.938Z" }, + { url = "https://files.pythonhosted.org/packages/59/43/6d54d48e8eaee25cdab70d3e4c4f579ddb0255e4f1660040d5ad55e029c6/ndindex-1.10.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:752e78a5e87911ded117c57a7246596f26c9c6da066de3c2b533b3db694949bb", size = 1510036, upload-time = "2025-11-19T20:38:47.444Z" }, + { url = "https://files.pythonhosted.org/packages/09/61/e28ba3b98eacd18193176526526b34d7d70d2a6f9fd2b4d8309ab5692678/ndindex-1.10.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c9dd58d91220b1c1fe516324bfcf4114566c98e84b1cbbe416abe345c75bd557", size = 1571849, upload-time = "2025-11-19T20:38:48.951Z" }, + { url = "https://files.pythonhosted.org/packages/77/34/a51b7c6f7159718a6a0a694fc1058b94d793c416d9a4fd649f1924cce5f8/ndindex-1.10.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6928e47fb008903f2e41309b7ff1e59b16abbcd59e2e945454571c28b2433c9e", size = 524127, upload-time = "2025-11-19T20:38:59.412Z" }, + { url = "https://files.pythonhosted.org/packages/21/91/d8f19f0b8fc9c5585b50fda44c05415da0bdc5fa9c9c69011015dac27880/ndindex-1.10.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e69a2cb1ac7be955c3c77f1def83f410775a81525c9ce2d4c0a3f2a61589ed47", size = 528213, upload-time = "2025-11-19T20:39:00.882Z" }, + { url = "https://files.pythonhosted.org/packages/2c/a9/77d9d037e871a3faa8579b354ca2dd09cc5bbf3e085d9e3c67f786d55ee3/ndindex-1.10.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cb76e0f3f235d8b1c768b17e771de48775d281713795c3aa045e8114ad61bdda", size = 1492172, upload-time = "2025-11-19T20:39:02.387Z" }, + { url = "https://files.pythonhosted.org/packages/ac/29/ad13676fc9312e0aa1a80a7c04bcb0b502b877ed4956136117ad663eced0/ndindex-1.10.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7da34a78410c14341d5fff73be5ce924bd36500bf7f640fc59b8607d3a0df95e", size = 1552614, upload-time = "2025-11-19T20:39:04.232Z" }, + { url = "https://files.pythonhosted.org/packages/e2/d7/1df02df24880ce3f3c8137b6f3ca5a901a58d9079dcfd8c818419277ff87/ndindex-1.10.1-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d0ebdba2fade3f6916fe21fd49e2a0935af4f58c56100a60f3f2eb26e20baee7", size = 632517, upload-time = "2025-11-19T20:39:13.259Z" }, + { url = "https://files.pythonhosted.org/packages/34/96/b509c2b14e9b10710fe6ab6ba8bda1ee6ce36ab16397ff2f5bbb33bbbba3/ndindex-1.10.1-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:346a4bf09f5771548665c8206e81daadb6b9925d409746e709894bdd98adc701", size = 616179, upload-time = "2025-11-19T20:39:14.757Z" }, + { url = "https://files.pythonhosted.org/packages/38/e3/f89d60cf351c33a484bf1a4546a5dee6f4e7a6a973613ffa12bd316b14ad/ndindex-1.10.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:23d35696f802548143b5cc199bf2f171efb0061aa7934959251dd3bae56d038c", size = 1588373, upload-time = "2025-11-19T20:39:16.62Z" }, + { url = "https://files.pythonhosted.org/packages/ee/19/002fc1e6a4abeef8d92e9aa2e43aea4d462f6b170090f7752ea8887f4897/ndindex-1.10.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a91e1a0398120233d5c3b23ccb2d4b78e970d66136f1a7221fa9a53873c3d5c5", size = 1636436, upload-time = "2025-11-19T20:39:18.266Z" }, + { url = "https://files.pythonhosted.org/packages/09/4a/028ab78a9f29fd2a7e86a90337cde4658eaa77b425c63045d83a1d2e4f26/ndindex-1.10.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf80241b40adffdc3276b2c9fb63a96c6c98b4a9d941892738de8add65083962", size = 528125, upload-time = "2025-11-19T20:39:26.798Z" }, + { url = "https://files.pythonhosted.org/packages/00/a9/bd823b345fb06c83ade6ef1c1933521d4357cd04490e684d4fa30126926c/ndindex-1.10.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf5855881884b8467dfcf45764ccf2e4279075be14b155b89c96994bb08d2e6f", size = 527328, upload-time = "2025-11-19T20:39:28.292Z" }, + { url = "https://files.pythonhosted.org/packages/91/4f/40b9c15588cbf9dde43c4fb88a31dd1f636a913fa29649f18f8e3ebca36a/ndindex-1.10.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e81a9bd36fe054b6c9fcc53d26bc9a28cf15d1ab52a0f5b854f894116f3a54e1", size = 1497508, upload-time = "2025-11-19T20:39:30.735Z" }, + { url = "https://files.pythonhosted.org/packages/24/8f/b8048f7837d2e9dff0af507b398307fa84a2aa9ea3db71b4aa800b21da4a/ndindex-1.10.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:588e8875d836a93b3cd9af482c8074bb02288ae1aff92cf277e1f02d9ae0f992", size = 1552625, upload-time = "2025-11-19T20:39:32.404Z" }, + { url = "https://files.pythonhosted.org/packages/4d/05/a6fda696a2f02a3f8dd2ee9d816cb2edff6423bf0110a4876cc3b1259732/ndindex-1.10.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c65ca639a7abf72d79f22424f4abd18dece1f289a2b7b028a0ca455edd2168d4", size = 630898, upload-time = "2025-11-19T20:39:41.495Z" }, + { url = "https://files.pythonhosted.org/packages/73/78/eb2e5d067d4c054451e33eaece74cbdcb58236dc60516e73d783dae34c7e/ndindex-1.10.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c3634a8df43e7928122225a3d64d850c8957bd1edf2e403907deacb478af27b", size = 614419, upload-time = "2025-11-19T20:39:43.254Z" }, + { url = "https://files.pythonhosted.org/packages/78/51/261bfb49eb7920c2a7314cacba5821930a529911dce48c7c6cd786096a5a/ndindex-1.10.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9d581f931e61f182478f18bdf5edd3955899df5da4892ed0d5de547a4cfd5b6f", size = 1587517, upload-time = "2025-11-19T20:39:44.809Z" }, + { url = "https://files.pythonhosted.org/packages/ec/37/084a332ecdf8b0049151bd78001a7baf2daf7f500d043beb8a1f95d0f4e3/ndindex-1.10.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:78ce45106ebf67aeba99714818c721d8fd5fb9534daebd2565665a2d64b50fc9", size = 1635372, upload-time = "2025-11-19T20:39:47.231Z" }, +] + +[[package]] +name = "nest-asyncio2" +version = "1.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/73/731debf26e27e0a0323d7bda270dc2f634b398e38f040a09da1f4351d0aa/nest_asyncio2-1.7.2.tar.gz", hash = "sha256:1921d70b92cc4612c374928d081552efb59b83d91b2b789d935c665fa01729a8", size = 14743, upload-time = "2026-02-13T00:34:04.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/3c/3179b85b0e1c3659f0369940200cd6d0fa900e6cefcc7ea0bc6dd0e29ffb/nest_asyncio2-1.7.2-py3-none-any.whl", hash = "sha256:f5dfa702f3f81f6a03857e9a19e2ba578c0946a4ad417b4c50a24d7ba641fe01", size = 7843, upload-time = "2026-02-13T00:34:02.691Z" }, +] + +[[package]] +name = "notebook" +version = "5.7.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ipykernel", marker = "sys_platform == 'linux'" }, + { name = "ipython-genutils", marker = "sys_platform == 'linux'" }, + { name = "jinja2", marker = "sys_platform == 'linux'" }, + { name = "jupyter-client", marker = "sys_platform == 'linux'" }, + { name = "jupyter-core", marker = "sys_platform == 'linux'" }, + { name = "nbconvert", marker = "sys_platform == 'linux'" }, + { name = "nbformat", marker = "sys_platform == 'linux'" }, + { name = "prometheus-client", marker = "sys_platform == 'linux'" }, + { name = "pyzmq", marker = "sys_platform == 'linux'" }, + { name = "send2trash", marker = "sys_platform == 'linux'" }, + { name = "terminado", marker = "sys_platform == 'linux'" }, + { name = "tornado", marker = "sys_platform == 'linux'" }, + { name = "traitlets", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6e/22/b5dcce67559d63d0f22e46d806305710808c698a1b91c07eb09e389785e0/notebook-5.7.4.tar.gz", hash = "sha256:d908673a4010787625c8952e91a22adf737db031f2aa0793ad92f6558918a74a", size = 13389469, upload-time = "2018-12-17T10:06:00.805Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/a6/dbdf0954d073ab2cf5421cad20334d1e4c780da37205c04522c045298f67/notebook-5.7.4-py2.py3-none-any.whl", hash = "sha256:3ab2db8bc10e6edbd264c3c4b800bee276c99818386ee0c146d98d7e6bcf0a67", size = 8980706, upload-time = "2018-12-17T10:05:52.01Z" }, +] + +[[package]] +name = "notebook-shim" +version = "0.2.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-server", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/54/d2/92fa3243712b9a3e8bafaf60aac366da1cada3639ca767ff4b5b3654ec28/notebook_shim-0.2.4.tar.gz", hash = "sha256:b4b2cfa1b65d98307ca24361f5b30fe785b53c3fd07b7a47e89acb5e6ac638cb", size = 13167, upload-time = "2024-02-14T23:35:18.353Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/33/bd5b9137445ea4b680023eb0469b2bb969d61303dedb2aac6560ff3d14a1/notebook_shim-0.2.4-py3-none-any.whl", hash = "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef", size = 13307, upload-time = "2024-02-14T23:35:16.286Z" }, +] + +[[package]] +name = "numba" +version = "0.66.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llvmlite", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/a0/570e3dc53e5602b49108f62a13e529f1eec8bfc7ef37d49c825924dcf546/numba-0.66.0.tar.gz", hash = "sha256:b900e63a0e26c05ea9a6d5a3a5a0a177cb64c5011887bf43edb8c3ed2c38d363", size = 2806181, upload-time = "2026-07-01T23:12:46.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/55/25c319845e9a4e08f16611ddbda56a192eb7b6ed13e1a2bff2da272ffb97/numba-0.66.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0999e3ee1b18c48e1fb51d11af35ef59852c7f4f50569c9550c25faef0616ad1", size = 3866252, upload-time = "2026-07-01T23:12:20.429Z" }, + { url = "https://files.pythonhosted.org/packages/71/ef/a82d6fd6bf1b0fe461651e924d3647eeec9ac17f8eee4896264bf7480930/numba-0.66.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efe0d2d5099790df945e0cb6e1b3104bd965d7bbfac50d62f1d5d1d6ade0825d", size = 3566974, upload-time = "2026-07-01T23:12:22.116Z" }, + { url = "https://files.pythonhosted.org/packages/44/b5/e930010965568fe7f2c6c962fd2849d458cb9f62c3ab7584af8a19a2b40a/numba-0.66.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:939316d5d8619751207b8972a67852b5a7646665298cb4de693cd6bf135152f4", size = 3873663, upload-time = "2026-07-01T23:12:27.308Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ec/5b51457cbe96e4831141d83e892e65191b23a1b78728456c62909d231ace/numba-0.66.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cdf506775d9f02eb92a87bf5c5b1e0d25506fd18cafd769f4ed914a8feac73e7", size = 3573529, upload-time = "2026-07-01T23:12:28.944Z" }, + { url = "https://files.pythonhosted.org/packages/0f/26/885774c006de6620ed3d10f45d8e20fe0b8e6aad6d573211a2cbc8b3e528/numba-0.66.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e2b101f23b8b63978d334574d2039f27f0dccfe1d891756f33a2e2f3e4c88cf4", size = 3842720, upload-time = "2026-07-01T23:12:33.938Z" }, + { url = "https://files.pythonhosted.org/packages/93/99/edebf7de890b73973d839dd971cf73734adfb81ffa1b4504f84b9059c3e5/numba-0.66.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63b943eb2c9ba371908ce2cd6dfc643db51fc40f7966993376a1701bc922f537", size = 3543537, upload-time = "2026-07-01T23:12:35.566Z" }, + { url = "https://files.pythonhosted.org/packages/39/fd/e9c9680a3813f3d781c20e5d53c1074801b787d4feecca0472fdd7c05ce1/numba-0.66.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:651a2b53298340956db26ecbe7ab043106b50a40c2807e66d617a4917245a4ab", size = 3878695, upload-time = "2026-07-01T23:12:40.302Z" }, + { url = "https://files.pythonhosted.org/packages/61/3a/9b363287b85fcd4537ea3878793822878b2ac1008a78159d2096fea628de/numba-0.66.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8c1144ba1720ea59ad79f4f488ed54d149b2613b357f7e445678b7d0739c70e9", size = 3596323, upload-time = "2026-07-01T23:12:42.805Z" }, +] + +[[package]] +name = "numbaquadpack" +version = "0.1" +source = { git = "https://github.com/Nicholaswogan/NumbaQuadpack.git#9fa67a4a952860042177a3dba61e7102799671b3" } + +[[package]] +name = "numcodecs" +version = "0.16.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/bd/8a391e7c356366224734efd24da929cc4796fff468bfb179fe1af6548535/numcodecs-0.16.5.tar.gz", hash = "sha256:0d0fb60852f84c0bd9543cc4d2ab9eefd37fc8efcc410acd4777e62a1d300318", size = 6276387, upload-time = "2025-11-21T02:49:48.986Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/1e/98aaddf272552d9fef1f0296a9939d1487914a239e98678f6b20f8b0a5c8/numcodecs-0.16.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b554ab9ecf69de7ca2b6b5e8bc696bd9747559cb4dd5127bd08d7a28bec59c3a", size = 8534814, upload-time = "2025-11-21T02:49:28.547Z" }, + { url = "https://files.pythonhosted.org/packages/fb/53/78c98ef5c8b2b784453487f3e4d6c017b20747c58b470393e230c78d18e8/numcodecs-0.16.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad1a379a45bd3491deab8ae6548313946744f868c21d5340116977ea3be5b1d6", size = 9173471, upload-time = "2025-11-21T02:49:30.444Z" }, + { url = "https://files.pythonhosted.org/packages/0b/00/787ea5f237b8ea7bc67140c99155f9c00b5baf11c49afc5f3bfefa298f95/numcodecs-0.16.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:015a7c859ecc2a06e2a548f64008c0ec3aaecabc26456c2c62f4278d8fc20597", size = 8483064, upload-time = "2025-11-21T02:49:36.454Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e6/d359fdd37498e74d26a167f7a51e54542e642ea47181eb4e643a69a066c3/numcodecs-0.16.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84230b4b9dad2392f2a84242bd6e3e659ac137b5a1ce3571d6965fca673e0903", size = 9126063, upload-time = "2025-11-21T02:49:38.018Z" }, + { url = "https://files.pythonhosted.org/packages/4e/15/e2e1151b5a8b14a15dfd4bb4abccce7fff7580f39bc34092780088835f3a/numcodecs-0.16.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49f7b7d24f103187f53135bed28bb9f0ed6b2e14c604664726487bb6d7c882e1", size = 8476987, upload-time = "2025-11-21T02:49:43.363Z" }, + { url = "https://files.pythonhosted.org/packages/6d/30/16a57fc4d9fb0ba06c600408bd6634f2f1753c54a7a351c99c5e09b51ee2/numcodecs-0.16.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aec9736d81b70f337d89c4070ee3ffeff113f386fd789492fa152d26a15043e4", size = 9102377, upload-time = "2025-11-21T02:49:45.508Z" }, +] + +[[package]] +name = "numexpr" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/2f/fdba158c9dbe5caca9c3eca3eaffffb251f2fb8674bf8e2d0aed5f38d319/numexpr-2.14.1.tar.gz", hash = "sha256:4be00b1086c7b7a5c32e31558122b7b80243fe098579b170967da83f3152b48b", size = 119400, upload-time = "2025-10-13T16:17:27.351Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/94/cc921e35593b820521e464cbbeaf8212bbdb07f16dc79fe283168df38195/numexpr-2.14.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d686dfb2c1382d9e6e0ee0b7647f943c1886dba3adbf606c625479f35f1956c1", size = 452468, upload-time = "2025-10-13T16:13:29.531Z" }, + { url = "https://files.pythonhosted.org/packages/d9/43/560e9ba23c02c904b5934496486d061bcb14cd3ebba2e3cf0e2dccb6c22b/numexpr-2.14.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee6d4fbbbc368e6cdd0772734d6249128d957b3b8ad47a100789009f4de7083", size = 443631, upload-time = "2025-10-13T16:15:02.473Z" }, + { url = "https://files.pythonhosted.org/packages/7b/6c/78f83b6219f61c2c22d71ab6e6c2d4e5d7381334c6c29b77204e59edb039/numexpr-2.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3a2839efa25f3c8d4133252ea7342d8f81226c7c4dda81f97a57e090b9d87a48", size = 1417670, upload-time = "2025-10-13T16:13:33.464Z" }, + { url = "https://files.pythonhosted.org/packages/0e/bb/1ccc9dcaf46281568ce769888bf16294c40e98a5158e4b16c241de31d0d3/numexpr-2.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9f9137f1351b310436662b5dc6f4082a245efa8950c3b0d9008028df92fefb9b", size = 1466212, upload-time = "2025-10-13T16:15:12.828Z" }, + { url = "https://files.pythonhosted.org/packages/13/65/72b065f9c75baf8f474fd5d2b768350935989d4917db1c6c75b866d4067c/numexpr-2.14.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:06855053de7a3a8425429bd996e8ae3c50b57637ad3e757e0fa0602a7874be30", size = 455860, upload-time = "2025-10-13T16:13:35.811Z" }, + { url = "https://files.pythonhosted.org/packages/fc/f9/c9457652dfe28e2eb898372da2fe786c6db81af9540c0f853ee04a0699cc/numexpr-2.14.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f9366d23a2e991fd5a8b5e61a17558f028ba86158a4552f8f239b005cdf83c", size = 446574, upload-time = "2025-10-13T16:15:17.367Z" }, + { url = "https://files.pythonhosted.org/packages/b6/99/8d3879c4d67d3db5560cf2de65ce1778b80b75f6fa415eb5c3e7bd37ba27/numexpr-2.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c5f1b1605695778896534dfc6e130d54a65cd52be7ed2cd0cfee3981fd676bf5", size = 1417306, upload-time = "2025-10-13T16:13:42.813Z" }, + { url = "https://files.pythonhosted.org/packages/ea/05/6bddac9f18598ba94281e27a6943093f7d0976544b0cb5d92272c64719bd/numexpr-2.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a4ba71db47ea99c659d88ee6233fa77b6dc83392f1d324e0c90ddf617ae3f421", size = 1466145, upload-time = "2025-10-13T16:15:27.464Z" }, + { url = "https://files.pythonhosted.org/packages/84/78/3c8335f713d4aeb99fa758d7c62f0be1482d4947ce5b508e2052bb7aeee9/numexpr-2.14.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33265294376e7e2ae4d264d75b798a915d2acf37b9dd2b9405e8b04f84d05cfc", size = 465972, upload-time = "2025-10-13T16:13:45.061Z" }, + { url = "https://files.pythonhosted.org/packages/35/81/9ee5f69b811e8f18746c12d6f71848617684edd3161927f95eee7a305631/numexpr-2.14.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83647d846d3eeeb9a9255311236135286728b398d0d41d35dedb532dca807fe9", size = 456953, upload-time = "2025-10-13T16:15:31.186Z" }, + { url = "https://files.pythonhosted.org/packages/6d/39/9b8bc6e294d85cbb54a634e47b833e9f3276a8bdf7ce92aa808718a0212d/numexpr-2.14.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6e575fd3ad41ddf3355d0c7ef6bd0168619dc1779a98fe46693cad5e95d25e6e", size = 1426199, upload-time = "2025-10-13T16:13:48.231Z" }, + { url = "https://files.pythonhosted.org/packages/1e/ce/0d4fcd31ab49319740d934fba1734d7dad13aa485532ca754e555ca16c8b/numexpr-2.14.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:67ea4771029ce818573b1998f5ca416bd255156feea017841b86176a938f7d19", size = 1474214, upload-time = "2025-10-13T16:15:38.893Z" }, + { url = "https://files.pythonhosted.org/packages/9a/ed/aabd8678077848dd9a751c5558c2057839f5a09e2a176d8dfcd0850ee00e/numexpr-2.14.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5aedf38d4c0c19d3cecfe0334c3f4099fb496f54c146223d30fa930084bc8574", size = 455918, upload-time = "2025-10-13T16:13:50.338Z" }, + { url = "https://files.pythonhosted.org/packages/88/e1/3db65117f02cdefb0e5e4c440daf1c30beb45051b7f47aded25b7f4f2f34/numexpr-2.14.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439ec4d57b853792ebe5456e3160312281c3a7071ecac5532ded3278ede614de", size = 446512, upload-time = "2025-10-13T16:15:42.313Z" }, + { url = "https://files.pythonhosted.org/packages/9a/fb/7ceb9ee55b5f67e4a3e4d73d5af4c7e37e3c9f37f54bee90361b64b17e3f/numexpr-2.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e23b87f744e04e302d82ac5e2189ae20a533566aec76a46885376e20b0645bf8", size = 1417845, upload-time = "2025-10-13T16:13:53.836Z" }, + { url = "https://files.pythonhosted.org/packages/45/2d/9b5764d0eafbbb2889288f80de773791358acf6fad1a55767538d8b79599/numexpr-2.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:44f84e0e5af219dbb62a081606156420815890e041b87252fbcea5df55214c4c", size = 1466211, upload-time = "2025-10-13T16:15:48.985Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d3/956a13e628d722d649fbf2fded615134a308c082e122a48bad0e90a99ce9/numexpr-2.14.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd72d8c2a165fe45ea7650b16eb8cc1792a94a722022006bb97c86fe51fd2091", size = 466242, upload-time = "2025-10-13T16:13:55.795Z" }, + { url = "https://files.pythonhosted.org/packages/d6/dd/abe848678d82486940892f2cacf39e82eec790e8930d4d713d3f9191063b/numexpr-2.14.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70d80fcb418a54ca208e9a38e58ddc425c07f66485176b261d9a67c7f2864f73", size = 457149, upload-time = "2025-10-13T16:15:52.036Z" }, + { url = "https://files.pythonhosted.org/packages/fd/bb/797b583b5fb9da5700a5708ca6eb4f889c94d81abb28de4d642c0f4b3258/numexpr-2.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:edea2f20c2040df8b54ee8ca8ebda63de9545b2112872466118e9df4d0ae99f3", size = 1426493, upload-time = "2025-10-13T16:13:59.244Z" }, + { url = "https://files.pythonhosted.org/packages/77/c4/0519ab028fdc35e3e7ee700def7f2b4631b175cd9e1202bd7966c1695c33/numexpr-2.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:790447be6879a6c51b9545f79612d24c9ea0a41d537a84e15e6a8ddef0b6268e", size = 1474413, upload-time = "2025-10-13T16:15:59.211Z" }, +] + +[[package]] +name = "numpy" +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, + { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, + { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, + { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, + { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, + { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, + { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, + { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, + { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, + { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, + { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, + { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, +] + +[[package]] +name = "numpydoc" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/3c/dfccc9e7dee357fb2aa13c3890d952a370dd0ed071e0f7ed62ed0df567c1/numpydoc-1.10.0.tar.gz", hash = "sha256:3f7970f6eee30912260a6b31ac72bba2432830cd6722569ec17ee8d3ef5ffa01", size = 94027, upload-time = "2025-12-02T16:39:12.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/5e/3a6a3e90f35cea3853c45e5d5fb9b7192ce4384616f932cf7591298ab6e1/numpydoc-1.10.0-py3-none-any.whl", hash = "sha256:3149da9874af890bcc2a82ef7aae5484e5aa81cb2778f08e3c307ba6d963721b", size = 69255, upload-time = "2025-12-02T16:39:11.561Z" }, +] + +[[package]] +name = "opencv-python-headless" +version = "5.0.0.93" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/99/76b7c80252aa83c1af16393454aafd125a0287101afe8deb0a6821af0e30/opencv_python_headless-5.0.0.93.tar.gz", hash = "sha256:b82f9831daab90b725c7c1ee1b36cb5732c367096ac76d119e64e14eb70d5f3c", size = 81817738, upload-time = "2026-07-02T07:01:06.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/78/afca939f40ffe2b2380bfa86f812b2f7d4acc5a27b27dc41b49cad7ce7b4/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10818d91510e05c04568ae12b5cd120779c70c01bf897b001a6221fe430df80f", size = 36521085, upload-time = "2026-07-02T06:55:24.429Z" }, + { url = "https://files.pythonhosted.org/packages/2b/97/8170e9819764c47e436c130d3ff6cfb73b58f923eae9d3a03d8982b04aec/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:09a872a157c1376ab922a69bbf22f9a95bcc7b658a9d8b436a60212b02b2eeb4", size = 56563598, upload-time = "2026-07-02T06:55:47.355Z" }, + { url = "https://files.pythonhosted.org/packages/3a/98/1a28a7101e31801042b3098871a74b76c61581d328ef40774ff4edb53a56/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:840bd717c21e5c11cadadc022a823315ea417f961213d06b4df010e019eb16f4", size = 39648433, upload-time = "2026-07-02T06:56:04.255Z" }, + { url = "https://files.pythonhosted.org/packages/9b/21/f6ef335f6e65724aa78b8d792b48d40a48c381715f1e62f5a5049e09d07e/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ed709fdf9aa0bd1f2ed8549e71d19449b03a675bb581eb292285f6861953be37", size = 61204038, upload-time = "2026-07-02T06:56:41.823Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pandas" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "python-dateutil", marker = "sys_platform == 'linux'" }, + { name = "pytz", marker = "sys_platform == 'linux'" }, + { name = "tzdata", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/56/cf2dbe1a3f5271370669475ead12ce77c61726ffd19a35546e31aa8edf4e/pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908", size = 11737212, upload-time = "2025-09-29T23:19:59.765Z" }, + { url = "https://files.pythonhosted.org/packages/e5/63/cd7d615331b328e287d8233ba9fdf191a9c2d11b6af0c7a59cfcec23de68/pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89", size = 12362693, upload-time = "2025-09-29T23:20:14.098Z" }, + { url = "https://files.pythonhosted.org/packages/a6/de/8b1895b107277d52f2b42d3a6806e69cfef0d5cf1d0ba343470b9d8e0a04/pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98", size = 12771002, upload-time = "2025-09-29T23:20:26.76Z" }, + { url = "https://files.pythonhosted.org/packages/87/21/84072af3187a677c5893b170ba2c8fbe450a6ff911234916da889b698220/pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084", size = 13450971, upload-time = "2025-09-29T23:20:41.344Z" }, + { url = "https://files.pythonhosted.org/packages/16/87/9472cf4a487d848476865321de18cc8c920b8cab98453ab79dbbc98db63a/pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d", size = 11709872, upload-time = "2025-09-29T23:21:27.165Z" }, + { url = "https://files.pythonhosted.org/packages/15/07/284f757f63f8a8d69ed4472bfd85122bd086e637bf4ed09de572d575a693/pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac", size = 12306371, upload-time = "2025-09-29T23:21:40.532Z" }, + { url = "https://files.pythonhosted.org/packages/33/81/a3afc88fca4aa925804a27d2676d22dcd2031c2ebe08aabd0ae55b9ff282/pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c", size = 12765333, upload-time = "2025-09-29T23:21:55.77Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0f/b4d4ae743a83742f1153464cf1a8ecfafc3ac59722a0b5c8602310cb7158/pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493", size = 13418120, upload-time = "2025-09-29T23:22:10.109Z" }, + { url = "https://files.pythonhosted.org/packages/46/b1/85331edfc591208c9d1a63a06baa67b21d332e63b7a591a5ba42a10bb507/pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78", size = 11645189, upload-time = "2025-09-29T23:22:51.688Z" }, + { url = "https://files.pythonhosted.org/packages/44/23/78d645adc35d94d1ac4f2a3c4112ab6f5b8999f4898b8cdf01252f8df4a9/pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110", size = 12121912, upload-time = "2025-09-29T23:23:05.042Z" }, + { url = "https://files.pythonhosted.org/packages/53/da/d10013df5e6aaef6b425aa0c32e1fc1f3e431e4bcabd420517dceadce354/pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86", size = 12712160, upload-time = "2025-09-29T23:23:28.57Z" }, + { url = "https://files.pythonhosted.org/packages/bd/17/e756653095a083d8a37cbd816cb87148debcfcd920129b25f99dd8d04271/pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc", size = 13199233, upload-time = "2025-09-29T23:24:24.876Z" }, + { url = "https://files.pythonhosted.org/packages/ca/05/d01ef80a7a3a12b2f8bbf16daba1e17c98a2f039cbc8e2f77a2c5a63d382/pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c", size = 11814049, upload-time = "2025-09-29T23:27:15.384Z" }, + { url = "https://files.pythonhosted.org/packages/15/b2/0e62f78c0c5ba7e3d2c5945a82456f4fac76c480940f805e0b97fcbc2f65/pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b", size = 12332638, upload-time = "2025-09-29T23:27:51.625Z" }, + { url = "https://files.pythonhosted.org/packages/c5/33/dd70400631b62b9b29c3c93d2feee1d0964dc2bae2e5ad7a6c73a7f25325/pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6", size = 12886834, upload-time = "2025-09-29T23:28:21.289Z" }, + { url = "https://files.pythonhosted.org/packages/d3/18/b5d48f55821228d0d2692b34fd5034bb185e854bdb592e9c640f6290e012/pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3", size = 13409925, upload-time = "2025-09-29T23:28:58.261Z" }, + { url = "https://files.pythonhosted.org/packages/f9/88/702bde3ba0a94b8c73a0181e05144b10f13f29ebfc2150c3a79062a8195d/pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450", size = 11634535, upload-time = "2025-09-29T23:30:21.003Z" }, + { url = "https://files.pythonhosted.org/packages/a4/1e/1bac1a839d12e6a82ec6cb40cda2edde64a2013a66963293696bbf31fbbb/pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5", size = 12121582, upload-time = "2025-09-29T23:30:43.391Z" }, + { url = "https://files.pythonhosted.org/packages/44/91/483de934193e12a3b1d6ae7c8645d083ff88dec75f46e827562f1e4b4da6/pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788", size = 12699963, upload-time = "2025-09-29T23:31:10.009Z" }, + { url = "https://files.pythonhosted.org/packages/70/44/5191d2e4026f86a2a109053e194d3ba7a31a2d10a9c2348368c63ed4e85a/pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87", size = 13202175, upload-time = "2025-09-29T23:31:59.173Z" }, +] + +[[package]] +name = "pandocfilters" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/70/6f/3dd4940bbe001c06a65f88e36bad298bc7a0de5036115639926b0c5c0458/pandocfilters-1.5.1.tar.gz", hash = "sha256:002b4a555ee4ebc03f8b66307e287fa492e4a77b4ea14d3f934328297bb4939e", size = 8454, upload-time = "2024-01-18T20:08:13.726Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/af/4fbc8cab944db5d21b7e2a5b8e9211a03a79852b1157e2c102fcc61ac440/pandocfilters-1.5.1-py2.py3-none-any.whl", hash = "sha256:93be382804a9cdb0a7267585f157e5d1731bbe5545a85b268d6f5fe6232de2bc", size = 8663, upload-time = "2024-01-18T20:08:11.28Z" }, +] + +[[package]] +name = "parso" +version = "0.8.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/4b/90c937815137d43ce71ba043cd3566221e9df6b9c805f24b5d138c9d40a7/parso-0.8.7.tar.gz", hash = "sha256:eaaac4c9fdd5e9e8852dc778d2d7405897ec510f2a298071453e5e3a07914bb1", size = 401824, upload-time = "2026-05-01T23:13:02.138Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, +] + +[[package]] +name = "partd" +version = "1.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "locket", marker = "sys_platform == 'linux'" }, + { name = "toolz", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b2/3a/3f06f34820a31257ddcabdfafc2672c5816be79c7e353b02c1f318daa7d4/partd-1.4.2.tar.gz", hash = "sha256:d022c33afbdc8405c226621b015e8067888173d85f7f5ecebb3cafed9a20f02c", size = 21029, upload-time = "2024-05-06T19:51:41.945Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/e7/40fb618334dcdf7c5a316c0e7343c5cd82d3d866edc100d98e29bc945ecd/partd-1.4.2-py3-none-any.whl", hash = "sha256:978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f", size = 18905, upload-time = "2024-05-06T19:51:39.271Z" }, +] + +[[package]] +name = "patsy" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/44/ed13eccdd0519eff265f44b670d46fbb0ec813e2274932dc1c0e48520f7d/patsy-1.0.2.tar.gz", hash = "sha256:cdc995455f6233e90e22de72c37fcadb344e7586fb83f06696f54d92f8ce74c0", size = 399942, upload-time = "2025-10-20T16:17:37.535Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/70/ba4b949bdc0490ab78d545459acd7702b211dfccf7eb89bbc1060f52818d/patsy-1.0.2-py2.py3-none-any.whl", hash = "sha256:37bfddbc58fcf0362febb5f54f10743f8b21dd2aa73dec7e7ef59d1b02ae668a", size = 233301, upload-time = "2025-10-20T16:17:36.563Z" }, +] + +[[package]] +name = "pexpect" +version = "4.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ptyprocess", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, +] + +[[package]] +name = "pillow" +version = "12.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, + { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, + { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, + { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ac/31fb64e1e7efb5a4b50cd3d92049ba89ac6e4d8d3bb6a74e15048ca3353e/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89", size = 4161684, upload-time = "2026-07-01T11:54:25.934Z" }, + { url = "https://files.pythonhosted.org/packages/87/b4/9805e23d2b4d77842b468513841fda254ee42f0289d25088340e4ff46e2d/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace", size = 4255487, upload-time = "2026-07-01T11:54:27.935Z" }, + { url = "https://files.pythonhosted.org/packages/df/39/ecf519435a200c693fe053a6ee4d835b41cf963a4dfc2551c4e637cb2a71/pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec", size = 3696433, upload-time = "2026-07-01T11:54:29.813Z" }, + { url = "https://files.pythonhosted.org/packages/1f/01/e18aff37cb0b4aac47ac90f016d347a49aca667ef97f190b06ac2aabc928/pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65", size = 6263736, upload-time = "2026-07-01T11:54:36.131Z" }, + { url = "https://files.pythonhosted.org/packages/f7/62/de5bdd77d935331f4f802edc11e4d82950f642caad6cb2f949837b8560e2/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3", size = 6937129, upload-time = "2026-07-01T11:54:38.216Z" }, + { url = "https://files.pythonhosted.org/packages/70/4d/105627a13300c5e0df1d174230b32fd1273062c96f7745fd552b945d1e1d/pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a", size = 6339562, upload-time = "2026-07-01T11:54:40.354Z" }, + { url = "https://files.pythonhosted.org/packages/6b/1d/f13de01a553988ab895ba1c722e06cf3144d4f57656fd5b81b6d881f1179/pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e", size = 7049439, upload-time = "2026-07-01T11:54:42.489Z" }, + { url = "https://files.pythonhosted.org/packages/dc/01/001f65b68192f0228cc1dbbc8d2530ab5d58b61037ba0587f946fea607cd/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330", size = 4161736, upload-time = "2026-07-01T11:54:51.156Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d2/0219746d0fd16fc8a84498e79452375be3797d3ce4044596ce565164b84f/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217", size = 4255435, upload-time = "2026-07-01T11:54:53.414Z" }, + { url = "https://files.pythonhosted.org/packages/c8/02/8d0bc62ef0302318c46ff2a512822d2610e81c7aa46c9b3abe6cbaca5ad0/pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930", size = 3696262, upload-time = "2026-07-01T11:54:55.739Z" }, + { url = "https://files.pythonhosted.org/packages/b1/9d/8b2c807dbef61a5197c047afe99823787eb66f63daf9fb2432f91d6f0462/pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321", size = 6263757, upload-time = "2026-07-01T11:55:01.778Z" }, + { url = "https://files.pythonhosted.org/packages/5c/44/c85361f65dbe00eea8576ee467c768d25129989efb76e94f205e9ca9bb46/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b", size = 6936962, upload-time = "2026-07-01T11:55:03.93Z" }, + { url = "https://files.pythonhosted.org/packages/18/7e/e483414b35800b86b6f08dbbc7803fb5cd52c4d6f897f47d53ea2c7e6f65/pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198", size = 6339171, upload-time = "2026-07-01T11:55:05.989Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f4/68c491844841ede6bed70189546b3ee9731cf9f2cbad396faff5e1ccba45/pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130", size = 7048116, upload-time = "2026-07-01T11:55:08.131Z" }, + { url = "https://files.pythonhosted.org/packages/d5/37/25c6692f06927ee973ff18c8d9ee98ad0b4d84ee67a09610c2dd1447958e/pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385", size = 6322855, upload-time = "2026-07-01T11:55:21.613Z" }, + { url = "https://files.pythonhosted.org/packages/cc/91/420637fcb8f1bc11029e403b4538e6694744428d8246118e45719f944556/pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c", size = 6989642, upload-time = "2026-07-01T11:55:24.006Z" }, + { url = "https://files.pythonhosted.org/packages/10/08/b94d7811281ccf0d143a1cf768d1c49e1e54af63e7b708ab2ee3eb87face/pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d", size = 6391281, upload-time = "2026-07-01T11:55:26.252Z" }, + { url = "https://files.pythonhosted.org/packages/d2/87/24233f785f55474dc02ce3e739c5528a77e3a862e9333d1dd7a25cc31f70/pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931", size = 7096716, upload-time = "2026-07-01T11:55:28.318Z" }, + { url = "https://files.pythonhosted.org/packages/5d/dc/8fdce34ec725a33c81c6ba122b904d6b9024e50ea9ac7bede62fab54506c/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139", size = 4162063, upload-time = "2026-07-01T11:55:37.941Z" }, + { url = "https://files.pythonhosted.org/packages/76/66/2044b9a63d3b84ff048228dfcb7cd9bf0df983e8470971bf7d4c57b693de/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402", size = 4255549, upload-time = "2026-07-01T11:55:40.022Z" }, + { url = "https://files.pythonhosted.org/packages/52/7e/1f67e6f4ece6b582ee4b539decbcc9f848dc245a93ed8cd7338bafef72f1/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c", size = 3696331, upload-time = "2026-07-01T11:55:41.98Z" }, + { url = "https://files.pythonhosted.org/packages/0c/08/93fa2e70e30a2d81547e481b6ee2bb9522117221fb1e0ce4b5df70967677/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace", size = 6273659, upload-time = "2026-07-01T11:55:48.264Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6d/043e96ff814fc31a33077e4cba86082167db520c93632afdf2042febbb0c/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4", size = 6947439, upload-time = "2026-07-01T11:55:50.503Z" }, + { url = "https://files.pythonhosted.org/packages/af/92/ba71d2ee2ac0edf3fa33bd9d5ee9ee080da70b1766f3ca3934f9938ddac9/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39", size = 6353577, upload-time = "2026-07-01T11:55:52.697Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ce/e63064e2122923ff687c8ad792d0d736a7b3920a56a46982e81a7fdd25d6/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71", size = 7060394, upload-time = "2026-07-01T11:55:55.149Z" }, + { url = "https://files.pythonhosted.org/packages/13/4f/9e049dfa21af7c22427275720e2490267ba8138120add5c4c574deb69782/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e", size = 6329237, upload-time = "2026-07-01T11:56:08.868Z" }, + { url = "https://files.pythonhosted.org/packages/36/16/cf6eeaae8d0fce8dd390a33437cf68c5d5bd73834a2bc6e2f14efda0ab45/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777", size = 6997047, upload-time = "2026-07-01T11:56:11.379Z" }, + { url = "https://files.pythonhosted.org/packages/1e/69/dbf769bdd55f48bf5733cac28edc6364ffaa072ec9ba336266e4fe66be55/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1", size = 6400440, upload-time = "2026-07-01T11:56:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, +] + +[[package]] +name = "pims" +version = "0.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "imageio", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "slicerator", marker = "sys_platform == 'linux'" }, + { name = "tifffile", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/02/5bf3639f5b77e9b183011c08541c5039ba3d04f5316c70312b48a8e003a9/pims-0.7.tar.gz", hash = "sha256:55907a4c301256086d2aa4e34a5361b9109f24e375c2071e1117b9491e82946b", size = 87779, upload-time = "2024-06-10T19:20:42.842Z" } + +[[package]] +name = "platformdirs" +version = "4.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/47/e4501f49c178ae1d9f4a75073fda4204f52647993f075a9db4d14930e0c5/platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7", size = 31224, upload-time = "2026-05-28T03:32:53.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "prometheus-client" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/fb/d9aa83ffe43ce1f19e557c0971d04b90561b0cfd50762aafb01968285553/prometheus_client-0.25.0.tar.gz", hash = "sha256:5e373b75c31afb3c86f1a52fa1ad470c9aace18082d39ec0d2f918d11cc9ba28", size = 86035, upload-time = "2026-04-09T19:53:42.359Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/9b/d4b1e644385499c8346fa9b622a3f030dce14cd6ef8a1871c221a17a67e7/prometheus_client-0.25.0-py3-none-any.whl", hash = "sha256:d5aec89e349a6ec230805d0df882f3807f74fd6c1a2fa86864e3c2279059fed1", size = 64154, upload-time = "2026-04-09T19:53:41.324Z" }, +] + +[[package]] +name = "prompt-toolkit" +version = "3.0.52" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, +] + +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, +] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, +] + +[[package]] +name = "pulp" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/70/69be07a67621ad804d6cf347965eb4e0d7786a97330d99c31d735aaa6c5a/pulp-3.3.2.tar.gz", hash = "sha256:d0904700c207ac11e25e3b1213b70eae1d6fb25faa719d75f3f15054901258c0", size = 16305346, upload-time = "2026-05-25T09:41:26.207Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/6e/d674f1dde91c71e2ac19e5e5cf1ee6d5845e3aefecd9c39ed9c4b0c9a696/pulp-3.3.2-py3-none-any.whl", hash = "sha256:631b166f72086971a9597f7a0233ababa99bb8d50a01cd543f7758be5a9f86c0", size = 16391742, upload-time = "2026-05-25T09:41:22.2Z" }, +] + +[[package]] +name = "pure-eval" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/05/0a34433a064256a578f1783a10da6df098ceaa4a57bbeaa96a6c0352786b/pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42", size = 19752, upload-time = "2024-07-21T12:58:21.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, +] + +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, +] + +[[package]] +name = "pyarrow" +version = "25.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/f3/95428098d1fa7d04432fb750eed06b41304c2f6a5d3319985e64db2d9d41/pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4", size = 1199181, upload-time = "2026-07-10T08:29:50.116Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/1a/22bfb6597dcdc861fa83c39c06e1457cb56f698940eff42fbb25de30e8e5/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f", size = 46841966, upload-time = "2026-07-10T08:27:07.685Z" }, + { url = "https://files.pythonhosted.org/packages/55/0e/cd705c042bc4fe7022478db577fcab4abdcfabb9bc37ab7a75556b3fcb2b/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778", size = 50088993, upload-time = "2026-07-10T08:27:14.268Z" }, + { url = "https://files.pythonhosted.org/packages/98/ee/d822e1ee31fe31ec5d057210e0605c950b975dcd8d9a332976cc859a9df8/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537", size = 49941005, upload-time = "2026-07-10T08:27:21.274Z" }, + { url = "https://files.pythonhosted.org/packages/33/1b/207a90cc64619a095eb75a263ae069735f2810056d43c667befd573ec083/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b", size = 53112355, upload-time = "2026-07-10T08:27:27.911Z" }, + { url = "https://files.pythonhosted.org/packages/53/55/6d1d5f5aff317ec5de9421594679ed51ed828fe7e2ce209327f819d801e4/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be", size = 46841701, upload-time = "2026-07-10T08:27:49.741Z" }, + { url = "https://files.pythonhosted.org/packages/b5/5d/f790fb6965ab54c9da0dda7856abc75fd0d7648d865f8d603c111d203a64/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9", size = 50090118, upload-time = "2026-07-10T08:27:56.051Z" }, + { url = "https://files.pythonhosted.org/packages/0c/8c/faf025357ebf31bc96777f234277aa31e2aeca6dd4ecaa391f29085473c2/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d", size = 49945559, upload-time = "2026-07-10T08:28:01.927Z" }, + { url = "https://files.pythonhosted.org/packages/07/a1/bd051871708ea99a5e0fc711926c26c6f2c6d0130c7aaac8093e34998af6/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1", size = 53114238, upload-time = "2026-07-10T08:28:08.594Z" }, + { url = "https://files.pythonhosted.org/packages/af/9f/2d81ba89d1e4198d0cb25fe7529de936830fdaec0db926bb52a1ef7080d4/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec", size = 46905617, upload-time = "2026-07-10T08:28:29.376Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/0ed312ec800fb536f93783215126cee4b8977dcfeccba6f0f44df0cc87d7/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887", size = 50119765, upload-time = "2026-07-10T08:28:35.826Z" }, + { url = "https://files.pythonhosted.org/packages/ca/88/cab5063ba0c4d46a9f6b4b7eb1c9029dc0302d65cd5ab3510c949a386568/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62", size = 50027563, upload-time = "2026-07-10T08:28:43.848Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fb/4d24f1b7fe2e042dc4ef315ef75e4e702d8e46fe10c37e63caff00502b03/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f", size = 53162437, upload-time = "2026-07-10T08:28:52.819Z" }, + { url = "https://files.pythonhosted.org/packages/8a/18/b37fc31a69cff4bdfb8842683def5612f551b93fff6f44375e4a4a6a5535/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50", size = 46912304, upload-time = "2026-07-10T08:29:14.656Z" }, + { url = "https://files.pythonhosted.org/packages/32/35/5cae19ba72493e5598022468b56f6a5571f399f485bf412f157356476caa/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe", size = 50073652, upload-time = "2026-07-10T08:29:22.467Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a5/ddd508424bdfd5e6945765e9e2ffc687e2f6115972badc8ecf423076c407/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc", size = 50058654, upload-time = "2026-07-10T08:29:29.689Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a4/324d0db203ff5eebe8694ec2d6ec5a23f9aaa5d02e5b8c692914c518c33c/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0", size = 53140153, upload-time = "2026-07-10T08:29:36.041Z" }, +] + +[[package]] +name = "pyavm" +version = "0.9.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/87/ac/a925d36dd37fc37f89afccc1f62ffa03b7344c8e4ff7850be7879b4497e6/pyavm-0.9.9.tar.gz", hash = "sha256:bc0f605d957c1fd6d7765523fcba8b9a72377ac6c51461c2a838fe44600bcd9a", size = 220572, upload-time = "2026-03-12T09:54:50.969Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/ab/ba8d2b40aee05cd986807d58ee324369eb16248b717a08b51eee977f8d33/pyavm-0.9.9-py3-none-any.whl", hash = "sha256:8bba0ee9645a8a9f215af9ceea67b494a6ee1fe380cebdc00ba99d781539ad76", size = 379786, upload-time = "2026-03-12T09:54:49.455Z" }, +] + +[[package]] +name = "pybind11" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/f0/35145a3c3baffeef55d4b8324caa33abaa8fa56ab345ecd4b2211d09163e/pybind11-3.0.4.tar.gz", hash = "sha256:3286b59c8a774b9ee650169302dd5a4eedc30a8617905a0560dd8ee44775130c", size = 589533, upload-time = "2026-04-19T03:08:15.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/06/c3a23c9a0263b136c519f033a58d4641e73065fefc7754e9667ec206d992/pybind11-3.0.4-py3-none-any.whl", hash = "sha256:961720ee652da51d531b7b2451a6bd2bc042b0106e6d9baa48ecb7d58034ce63", size = 314166, upload-time = "2026-04-19T03:08:14.091Z" }, +] + +[[package]] +name = "pybtex" +version = "0.26.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "latexcodec", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/f5/f30da9c93f0fa6d619332b2f69597219b625f35780473a05164a9981fd9a/pybtex-0.26.1.tar.gz", hash = "sha256:2e5543bea424e60e9e42eef70bff597be48649d8f68ba061a7a092b2477d5464", size = 692991, upload-time = "2026-04-03T13:05:39.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/f6/775eb92e865b28cdb4ad1f2bed7a5446197516f76b58a950faa3be3fd08d/pybtex-0.26.1-py3-none-any.whl", hash = "sha256:e26c0412cc54f5f21b2a6d9d175762a2d2af9ccf3a8f651cdb89ec035db77aa1", size = 126134, upload-time = "2026-04-03T13:05:40.623Z" }, +] + +[[package]] +name = "pybtex-docutils" +version = "1.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils", marker = "sys_platform == 'linux'" }, + { name = "pybtex", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/84/796ea94d26188a853660f81bded39f8de4cfe595130aef0dea1088705a11/pybtex-docutils-1.0.3.tar.gz", hash = "sha256:3a7ebdf92b593e00e8c1c538aa9a20bca5d92d84231124715acc964d51d93c6b", size = 18348, upload-time = "2023-08-22T18:47:54.833Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/b1/ce1f4596211efb5410e178a803f08e59b20bedb66837dcf41e21c54f9ec1/pybtex_docutils-1.0.3-py3-none-any.whl", hash = "sha256:8fd290d2ae48e32fcb54d86b0efb8d573198653c7e2447d5bec5847095f430b9", size = 6385, upload-time = "2023-08-22T06:43:20.513Z" }, +] + +[[package]] +name = "pyccl" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/40/ff357d61a20c21420f1fe7417ca6d889be2e07a67b3564775aeaf6c3f366/pyccl-3.3.4.tar.gz", hash = "sha256:a9d22557354fd161df909e8f4f199e4f1c434c080fd6cddb2246609ddbd70e7c", size = 16783719, upload-time = "2026-05-14T16:26:27.327Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/80/1db2504f21eec9b481e1fe298b05deb96e108d06ddb1828494d16d32bf1c/pyccl-3.3.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0229c863acea5c3fc6748dc7c40611dd529709c9d26e3679ebdbc467511ef993", size = 3707759, upload-time = "2026-05-14T16:26:16.779Z" }, + { url = "https://files.pythonhosted.org/packages/1a/91/3000c5fc1ee8379636dc894946a3633baae74a3f1cce3c2d402e8991a0b5/pyccl-3.3.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:685d528a096251a2eb5d569e611b5a2e68cc3a4a0134734b16ac4e73a07e7f27", size = 3706950, upload-time = "2026-05-14T16:26:20.394Z" }, + { url = "https://files.pythonhosted.org/packages/e5/66/803e8196149d8d8a75319b4f20268210b1e1b31b3158a1143ce99cacf953/pyccl-3.3.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:601b687b50b506f7f5775475d79fa55d221616a827a96ad5d6d68bf7a15de9b9", size = 3706955, upload-time = "2026-05-14T16:26:25.302Z" }, +] + +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types", marker = "sys_platform == 'linux'" }, + { name = "pydantic-core", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "typing-inspection", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, +] + +[[package]] +name = "pyerfa" +version = "2.0.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/39/63cc8291b0cf324ae710df41527faf7d331bce573899199d926b3e492260/pyerfa-2.0.1.5.tar.gz", hash = "sha256:17d6b24fe4846c65d5e7d8c362dcb08199dc63b30a236aedd73875cc83e1f6c0", size = 818430, upload-time = "2024-11-11T15:22:30.852Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/96/b6210fc624123c8ae13e1eecb68fb75e3f3adff216d95eee1c7b05843e3e/pyerfa-2.0.1.5-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0603e8e1b839327d586c8a627cdc634b795e18b007d84f0cda5500a0908254e", size = 692794, upload-time = "2024-11-11T15:22:19.429Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e0/050018d855d26d3c0b4a7d1b2ed692be758ce276d8289e2a2b44ba1014a5/pyerfa-2.0.1.5-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e43c7194e3242083f2350b46c09fd4bf8ba1bcc0ebd1460b98fc47fe2389906", size = 738711, upload-time = "2024-11-11T15:22:20.661Z" }, + { url = "https://files.pythonhosted.org/packages/b9/f5/ff91ee77308793ae32fa1e1de95e9edd4551456dd888b4e87c5938657ca5/pyerfa-2.0.1.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:07b80cd70701f5d066b1ac8cce406682cfcd667a1186ec7d7ade597239a6021d", size = 722966, upload-time = "2024-11-11T15:22:21.905Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pymaster" +version = "2.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ducc0", marker = "sys_platform == 'linux'" }, + { name = "healpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/45/470858f2a6a3f5133955bb87aed978a5e921d4052aea1c19718e6b72f0ae/pymaster-2.6.tar.gz", hash = "sha256:056a09bc8631cf338e93ef14d8c70272ad9554ef10498d919e920994531c57a2", size = 33707181, upload-time = "2025-12-16T09:18:28.905Z" } + +[[package]] +name = "pyopenssl" +version = "26.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/b7/da07bae88f5a9506b4def6f2f4903cf4c3b8831e560dba8fa18ca08f758f/pyopenssl-26.3.0.tar.gz", hash = "sha256:589de7fae1c9ea670d18422ed00fc04da787bbde8e1454aea872aa57b49ad341", size = 182024, upload-time = "2026-06-12T20:28:07.458Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/18/1dd71c9b43192ab83f1d531ad6002dc81108ac36c475f79fb7a295abe2f4/pyopenssl-26.3.0-py3-none-any.whl", hash = "sha256:46367f8f66b92271e6d218da9c87607e1ef5a0bc5c8dea5bb3db82f395c385a3", size = 56008, upload-time = "2026-06-12T20:28:05.999Z" }, +] + +[[package]] +name = "pyparsing" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "iniconfig", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "pluggy", marker = "sys_platform == 'linux'" }, + { name = "pygments", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", marker = "sys_platform == 'linux'" }, + { name = "pluggy", marker = "sys_platform == 'linux'" }, + { name = "pytest", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-json-logger" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f7/ff/3cc9165fd44106973cd7ac9facb674a65ed853494592541d339bdc9a30eb/python_json_logger-4.1.0.tar.gz", hash = "sha256:b396b9e3ed782b09ff9d6e4f1683d46c83ad0d35d2e407c09a9ebbf038f88195", size = 17573, upload-time = "2026-03-29T04:39:56.805Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/be/0631a861af4d1c875f096c07d34e9a63639560a717130e7a87cbc82b7e3f/python_json_logger-4.1.0-py3-none-any.whl", hash = "sha256:132994765cf75bf44554be9aa49b06ef2345d23661a96720262716438141b6b2", size = 15021, upload-time = "2026-03-29T04:39:55.266Z" }, +] + +[[package]] +name = "pytz" +version = "2026.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/46/dd499ec9038423421951e4fad73051febaa13d2df82b4064f87af8b8c0c3/pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a", size = 320861, upload-time = "2026-05-04T01:35:29.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141, upload-time = "2026-05-04T01:35:27.408Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, +] + +[[package]] +name = "pyzmq" +version = "27.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "implementation_name == 'pypy' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/0b/3c9baedbdf613ecaa7aa07027780b8867f57b6293b6ee50de316c9f3222b/pyzmq-27.1.0.tar.gz", hash = "sha256:ac0765e3d44455adb6ddbf4417dcce460fc40a05978c08efdf2948072f6db540", size = 281750, upload-time = "2025-09-08T23:10:18.157Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/5e/c3c49fdd0f535ef45eefcc16934648e9e59dace4a37ee88fc53f6cd8e641/pyzmq-27.1.0-cp312-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1c179799b118e554b66da67d88ed66cd37a169f1f23b5d9f0a231b4e8d44a113", size = 895645, upload-time = "2025-09-08T23:08:05.301Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e5/b0b2504cb4e903a74dcf1ebae157f9e20ebb6ea76095f6cfffea28c42ecd/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3837439b7f99e60312f0c926a6ad437b067356dc2bc2ec96eb395fd0fe804233", size = 652574, upload-time = "2025-09-08T23:08:06.828Z" }, + { url = "https://files.pythonhosted.org/packages/f8/9b/c108cdb55560eaf253f0cbdb61b29971e9fb34d9c3499b0e96e4e60ed8a5/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43ad9a73e3da1fab5b0e7e13402f0b2fb934ae1c876c51d0afff0e7c052eca31", size = 840995, upload-time = "2025-09-08T23:08:08.396Z" }, + { url = "https://files.pythonhosted.org/packages/c2/bb/b79798ca177b9eb0825b4c9998c6af8cd2a7f15a6a1a4272c1d1a21d382f/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0de3028d69d4cdc475bfe47a6128eb38d8bc0e8f4d69646adfbcd840facbac28", size = 1642070, upload-time = "2025-09-08T23:08:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/9c/80/2df2e7977c4ede24c79ae39dcef3899bfc5f34d1ca7a5b24f182c9b7a9ca/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:cf44a7763aea9298c0aa7dbf859f87ed7012de8bda0f3977b6fb1d96745df856", size = 2021121, upload-time = "2025-09-08T23:08:11.907Z" }, + { url = "https://files.pythonhosted.org/packages/46/bd/2d45ad24f5f5ae7e8d01525eb76786fa7557136555cac7d929880519e33a/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f30f395a9e6fbca195400ce833c731e7b64c3919aa481af4d88c3759e0cb7496", size = 1878550, upload-time = "2025-09-08T23:08:13.513Z" }, + { url = "https://files.pythonhosted.org/packages/14/1d/d343f3ce13db53a54cb8946594e567410b2125394dafcc0268d8dda027e0/pyzmq-27.1.0-cp313-cp313t-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:08363b2011dec81c354d694bdecaef4770e0ae96b9afea70b3f47b973655cc05", size = 897275, upload-time = "2025-09-08T23:08:26.063Z" }, + { url = "https://files.pythonhosted.org/packages/69/2d/d83dd6d7ca929a2fc67d2c3005415cdf322af7751d773524809f9e585129/pyzmq-27.1.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d54530c8c8b5b8ddb3318f481297441af102517602b569146185fa10b63f4fa9", size = 660469, upload-time = "2025-09-08T23:08:27.623Z" }, + { url = "https://files.pythonhosted.org/packages/3e/cd/9822a7af117f4bc0f1952dbe9ef8358eb50a24928efd5edf54210b850259/pyzmq-27.1.0-cp313-cp313t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f3afa12c392f0a44a2414056d730eebc33ec0926aae92b5ad5cf26ebb6cc128", size = 847961, upload-time = "2025-09-08T23:08:29.672Z" }, + { url = "https://files.pythonhosted.org/packages/9a/12/f003e824a19ed73be15542f172fd0ec4ad0b60cf37436652c93b9df7c585/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c65047adafe573ff023b3187bb93faa583151627bc9c51fc4fb2c561ed689d39", size = 1650282, upload-time = "2025-09-08T23:08:31.349Z" }, + { url = "https://files.pythonhosted.org/packages/d5/4a/e82d788ed58e9a23995cee70dbc20c9aded3d13a92d30d57ec2291f1e8a3/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:90e6e9441c946a8b0a667356f7078d96411391a3b8f80980315455574177ec97", size = 2024468, upload-time = "2025-09-08T23:08:33.543Z" }, + { url = "https://files.pythonhosted.org/packages/d9/94/2da0a60841f757481e402b34bf4c8bf57fa54a5466b965de791b1e6f747d/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:add071b2d25f84e8189aaf0882d39a285b42fa3853016ebab234a5e78c7a43db", size = 1885394, upload-time = "2025-09-08T23:08:35.51Z" }, + { url = "https://files.pythonhosted.org/packages/48/43/d72ccdbf0d73d1343936296665826350cb1e825f92f2db9db3e61c2162a2/pyzmq-27.1.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1779be8c549e54a1c38f805e56d2a2e5c009d26de10921d7d51cfd1c8d4632ea", size = 897175, upload-time = "2025-09-08T23:08:46.601Z" }, + { url = "https://files.pythonhosted.org/packages/2f/2e/a483f73a10b65a9ef0161e817321d39a770b2acf8bcf3004a28d90d14a94/pyzmq-27.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7200bb0f03345515df50d99d3db206a0a6bee1955fbb8c453c76f5bf0e08fb96", size = 660427, upload-time = "2025-09-08T23:08:48.187Z" }, + { url = "https://files.pythonhosted.org/packages/f5/d2/5f36552c2d3e5685abe60dfa56f91169f7a2d99bbaf67c5271022ab40863/pyzmq-27.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01c0e07d558b06a60773744ea6251f769cd79a41a97d11b8bf4ab8f034b0424d", size = 847929, upload-time = "2025-09-08T23:08:49.76Z" }, + { url = "https://files.pythonhosted.org/packages/c4/2a/404b331f2b7bf3198e9945f75c4c521f0c6a3a23b51f7a4a401b94a13833/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:80d834abee71f65253c91540445d37c4c561e293ba6e741b992f20a105d69146", size = 1650193, upload-time = "2025-09-08T23:08:51.7Z" }, + { url = "https://files.pythonhosted.org/packages/1c/0b/f4107e33f62a5acf60e3ded67ed33d79b4ce18de432625ce2fc5093d6388/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:544b4e3b7198dde4a62b8ff6685e9802a9a1ebf47e77478a5eb88eca2a82f2fd", size = 2024388, upload-time = "2025-09-08T23:08:53.393Z" }, + { url = "https://files.pythonhosted.org/packages/0d/01/add31fe76512642fd6e40e3a3bd21f4b47e242c8ba33efb6809e37076d9b/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cedc4c68178e59a4046f97eca31b148ddcf51e88677de1ef4e78cf06c5376c9a", size = 1885316, upload-time = "2025-09-08T23:08:55.702Z" }, +] + +[[package]] +name = "qp-prob" +version = "1.1.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "tables-io", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/65/c54639ac63667800ee00d74c8851bd104ccaefcae95138073d6e77832d17/qp_prob-1.1.4.tar.gz", hash = "sha256:a68a3a3785d3ca57a2246c0524398338b7b7dffef0aef0722c231c64dde30107", size = 2460490, upload-time = "2026-06-25T20:08:34.63Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/9f/6df94f6a8fc6af157b2479c7ea0f1ef86d06e8ce7f6cba09bfbb004d5de7/qp_prob-1.1.4-py3-none-any.whl", hash = "sha256:e68d193ba0a4eb0a92f9b4861b9defc59194712865fa90e06db57196f8227e30", size = 112941, upload-time = "2026-06-25T20:08:32.68Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs", marker = "sys_platform == 'linux'" }, + { name = "rpds-py", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "regions" +version = "0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astropy", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d3/86/44ff464efa2777247d6addc74416059f1872ab05c88588bbe1aadca27a7c/regions-0.12.tar.gz", hash = "sha256:1c9460770f250ef299e90a9d5c0b35941f7d05bbf879f6ffaa0538250c018ef9", size = 273609, upload-time = "2026-06-24T20:10:17.78Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/a2/fe550b219404e98f2afc742ebb1d1bcd27888a6ad48ce60baf5fa96bde97/regions-0.12-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf5a6d8f1060607eab07f131cf12709ef7c4e9b1918d4aa269572e6c49e68b12", size = 2410524, upload-time = "2026-06-24T20:10:11.56Z" }, + { url = "https://files.pythonhosted.org/packages/92/4c/21bed50e1f86aa9fc5148318057f23d68d81f9ce0c1b31038e6c5c0e9547/regions-0.12-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e3b74d83fc4042430e8f11a0f1a3939ec6f95a7244a7103caeb6ad115d845acf", size = 2426898, upload-time = "2026-06-24T20:10:13.042Z" }, +] + +[[package]] +name = "reproject" +version = "0.21.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astropy", marker = "sys_platform == 'linux'" }, + { name = "astropy-healpix", marker = "sys_platform == 'linux'" }, + { name = "dask", extra = ["array"], marker = "sys_platform == 'linux'" }, + { name = "dask-image", marker = "sys_platform == 'linux'" }, + { name = "fsspec", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "pillow", marker = "sys_platform == 'linux'" }, + { name = "pyavm", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "zarr", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/44/6fd820ba336484277a91a2f4808b60d6ec0b9f033f588c237e778f20fe89/reproject-0.21.0.tar.gz", hash = "sha256:01ede715a1993c29431f52ff74189ef30f5e7b2e8b4dc88c1b002145a971dc1c", size = 1622661, upload-time = "2026-06-25T15:11:34.886Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/5b/8d9b51c754ab014194d374cb4873d0729b397f67997e67721538b643e682/reproject-0.21.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2132fc5d2fa3fbbd57337099f9d47582136e653ac82402783c7ceffa26804816", size = 1776590, upload-time = "2026-06-25T15:11:30.358Z" }, + { url = "https://files.pythonhosted.org/packages/da/2d/f9d76e8e308813978227e1e43a1f25f64cd0f4b030cecaeff8baeec9eeac/reproject-0.21.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8525baaca84e949a69532c02ff491c23993e3bc22a59f694af576e473f2a9d2f", size = 1791889, upload-time = "2026-06-25T15:11:31.741Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi", marker = "sys_platform == 'linux'" }, + { name = "charset-normalizer", marker = "sys_platform == 'linux'" }, + { name = "idna", marker = "sys_platform == 'linux'" }, + { name = "urllib3", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "rfc3339-validator" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/ea/a9387748e2d111c3c2b275ba970b735e04e15cdb1eb30693b6b5708c4dbd/rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b", size = 5513, upload-time = "2021-05-12T16:37:54.178Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa", size = 3490, upload-time = "2021-05-12T16:37:52.536Z" }, +] + +[[package]] +name = "rfc3986-validator" +version = "0.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/88/f270de456dd7d11dcc808abfa291ecdd3f45ff44e3b549ffa01b126464d0/rfc3986_validator-0.1.1.tar.gz", hash = "sha256:3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055", size = 6760, upload-time = "2019-10-28T16:00:19.144Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl", hash = "sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9", size = 4242, upload-time = "2019-10-28T16:00:13.976Z" }, +] + +[[package]] +name = "rfc3987-syntax" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lark", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2c/06/37c1a5557acf449e8e406a830a05bf885ac47d33270aec454ef78675008d/rfc3987_syntax-1.1.0.tar.gz", hash = "sha256:717a62cbf33cffdd16dfa3a497d81ce48a660ea691b1ddd7be710c22f00b4a0d", size = 14239, upload-time = "2025-07-18T01:05:05.015Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/71/44ce230e1b7fadd372515a97e32a83011f906ddded8d03e3c6aafbdedbb7/rfc3987_syntax-1.1.0-py3-none-any.whl", hash = "sha256:6c3d97604e4c5ce9f714898e05401a0445a641cfa276432b0a648c80856f6a3f", size = 8046, upload-time = "2025-07-18T01:05:03.843Z" }, +] + +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, + { name = "pygments", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + +[[package]] +name = "rocket-fft" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numba", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/b0/09cbec3177ecf56e5ceb789a69fbcdcbabe758cbb682e5d36b14aa8c05fb/rocket_fft-0.3.1.tar.gz", hash = "sha256:ddc902a361099aff9fc02fe9ea3b0d036c2f67f93df356c2b754f7627d7fa4f4", size = 74276, upload-time = "2025-12-26T00:30:54.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/18/12d001c33cd7c4e012da59d25086879aec303e0e157d9ec737ecab8e8f58/rocket_fft-0.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46970c0dcc70bd9c365d6a9f4a235edec2e8eaad628097c0e21d920669e25abe", size = 2151682, upload-time = "2025-12-26T00:30:30.315Z" }, + { url = "https://files.pythonhosted.org/packages/c8/92/98bd01cad9e5ceeb383722d744597d066e37cdd0e54742906341aa369c28/rocket_fft-0.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d64c9fc81206f2c77f74f68761f54ac6f1ebe4c8840f0ef9795c28ab229463f", size = 2160197, upload-time = "2025-12-26T00:30:31.945Z" }, + { url = "https://files.pythonhosted.org/packages/48/2c/5bbe8d6b0c4e009f7bc6f9f4eee802a9b556ec2c3b02b9d5e5ccfa8edff1/rocket_fft-0.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5906c6517fc87295a1d142c7e6ebd1c55e057f6ee4c605549bbcecacab83774f", size = 2151627, upload-time = "2025-12-26T00:30:37.177Z" }, + { url = "https://files.pythonhosted.org/packages/55/0d/ac67fb0b2fac382ab5585719bbd1f4927c19ba0ee3bbc28dc8a1ef83e81d/rocket_fft-0.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d830aacc34d65b28daab715aa4a90d98489e89a48c2836039982f18ba1e2db3", size = 2160355, upload-time = "2025-12-26T00:30:38.81Z" }, + { url = "https://files.pythonhosted.org/packages/d5/c6/3ff33ec86e61867c0e36436eb2c9ba258eceba94419594d73ff473460e27/rocket_fft-0.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65a7577d7f3b4e3672de179f91fee8339211b3f5ae13eef1b30cb6a318f0308c", size = 2151746, upload-time = "2025-12-26T00:30:43.66Z" }, + { url = "https://files.pythonhosted.org/packages/bf/d9/def65716cd225954d0cd5f1f6326c72c20601db5dc65557ef4b66157ccea/rocket_fft-0.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:97614dbebb2686a644bbf6c6f8a6198f063052f4c88901d0ca2808536c2e83cc", size = 2160407, upload-time = "2025-12-26T00:30:44.959Z" }, +] + +[[package]] +name = "roman-numerals" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/41dc953bbeb056c17d5f7a519f50fdf010bd0553be2d630bc69d1e022703/roman_numerals-4.1.0.tar.gz", hash = "sha256:1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2", size = 9077, upload-time = "2025-12-17T18:25:34.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7", size = 7676, upload-time = "2025-12-17T18:25:33.098Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.21" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/36/6f65aa9989acdec45d417192d8f4e7921931d8a6cf87ac74bce3eed98a8e/ruff-0.15.21.tar.gz", hash = "sha256:d0cfc841c572283c36548f82664a54ce6565567f1b0d5b4cf2caac693d8b7500", size = 4769401, upload-time = "2026-07-09T20:01:34.005Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/c6/ede15cac6839f3dbce52565c8f5164a8210e669c7bc4decb03e5bdf47d0d/ruff-0.15.21-py3-none-linux_armv6l.whl", hash = "sha256:63ea0e965e5d73c90e95b2434beeafc70820536717f561b32ab6e777cb9bdf5d", size = 10854342, upload-time = "2026-07-09T20:00:53.998Z" }, + { url = "https://files.pythonhosted.org/packages/9a/6f/b4523cc90ba239ede441447a19d0c968846a3012e5a0b0c5b62831a3d5e3/ruff-0.15.21-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01d65b4831c6b2a4ba8ee6faa84049d44d982b7a706e622c4094c509e51673be", size = 10990053, upload-time = "2026-07-09T20:01:02.187Z" }, + { url = "https://files.pythonhosted.org/packages/92/cc/c6a9872a5375f0628875481cf2f66b13d7d865bf3ca2e57f91c7e762d976/ruff-0.15.21-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2c5a913a589120ce67933d5d05fd6ddbcc2481c6a054980ee767f7414c72b4fd", size = 10666096, upload-time = "2026-07-09T20:01:04.299Z" }, + { url = "https://files.pythonhosted.org/packages/ab/97/c621f7a17e097f1790fa3af6374138823b330b2d03fc38337945daca212c/ruff-0.15.21-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef04b681d02ad4dc9620f00f83ac5c22f652d0e9a9cfe431d219b16ad5ccc41", size = 11537011, upload-time = "2026-07-09T20:01:06.771Z" }, + { url = "https://files.pythonhosted.org/packages/ea/51/d928727e476e25ccc57c6f449ffd80241a651a973ad949d39cfb2a771d28/ruff-0.15.21-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16d090c0740916594157e75b80d666eab8e78083b39b3b0e1d698f4670a17b86", size = 12347101, upload-time = "2026-07-09T20:01:08.859Z" }, + { url = "https://files.pythonhosted.org/packages/1e/88/8cd62026802b16018ad06931d87997cf795ba2a6239ab659606c87d96bf0/ruff-0.15.21-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a10e74757dd65004d779b73e2f3c5210156d9980b41224d50d2ebcf1db51e67", size = 11572001, upload-time = "2026-07-09T20:01:11.092Z" }, + { url = "https://files.pythonhosted.org/packages/b2/97/f63084cf55444fc110e8cb985ebfcc592af47f597d44453d778cb81bc156/ruff-0.15.21-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bab0905d2f29e0d9fbc3c373ed23db0095edaa3f71f1f4f519ec15134d9e85c8", size = 11549239, upload-time = "2026-07-09T20:01:13.27Z" }, + { url = "https://files.pythonhosted.org/packages/9d/77/f107da4a2874b7715914b03f09ba9c54424de3ff8a1cc5d015d3ee2ce0ac/ruff-0.15.21-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:00eca240af5789fec6fe7df74c088cc1f9644ed83027113468efba7c92b94075", size = 11535340, upload-time = "2026-07-09T20:01:15.206Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e9/601deb322d3303a7bf212b0100ead6f2ee3f6a044d89c30f2f92bf83c731/ruff-0.15.21-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:262ab31557a75141325e32d3357f3597645a7f084e732b6b054dde428ecd9341", size = 10964048, upload-time = "2026-07-09T20:01:17.723Z" }, + { url = "https://files.pythonhosted.org/packages/ea/2e/0f2176d1e99c15192caea19c8c3a0a955246b4cb4de795042eeb616345cd/ruff-0.15.21-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:659c4e7a4212f83306045ec7c5e5a356d16d9a6ef4ae0c7a4d872914fc655d9d", size = 10667055, upload-time = "2026-07-09T20:01:19.73Z" }, + { url = "https://files.pythonhosted.org/packages/48/60/abd74a02e0c4214f12a68becfd30af7165cfdcb0e661ecdc60bbb949c09a/ruff-0.15.21-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9e866eab611a5f959d36df2d10e446973a3610bc42b0c15b31dc27977d59c233", size = 11242043, upload-time = "2026-07-09T20:01:21.947Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c6/583075d8ccabb4b229345edcaf1545eb3d8d6be90f686a479d7e94088bbf/ruff-0.15.21-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e89bc93c0d3803ba870b55c29671bad9dc6d94bb1eb181b056b52eb05b52854f", size = 11648064, upload-time = "2026-07-09T20:01:24.023Z" }, +] + +[[package]] +name = "sacc" +version = "2.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astropy", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/82/5b603d960279b8eb7fac677ba3550c0903a8390f1bff2eee6fbfa59765da/sacc-2.4.tar.gz", hash = "sha256:a98fb4e59b3c742ddb492a15a10e1f8a4279ba36d56c0d3c35e1cd13cca1167b", size = 424905, upload-time = "2026-07-02T11:29:27.458Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/9f/56288a8aabafa24679d32d01e7db732e8e89e4074cc91aedc0bb7b7c8e46/sacc-2.4-py3-none-any.whl", hash = "sha256:da6b648998738c7c307cbdff644429633726844f8c048642725be5efc1859fa3", size = 50931, upload-time = "2026-07-02T11:29:26.286Z" }, +] + +[[package]] +name = "scipy" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/34/16f10e3042d2f1d6b66e0428308ab52224b6a23049cb2f5c1756f713815f/scipy-1.17.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21", size = 32927842, upload-time = "2026-02-23T00:18:35.367Z" }, + { url = "https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458", size = 35235890, upload-time = "2026-02-23T00:18:49.188Z" }, + { url = "https://files.pythonhosted.org/packages/c5/5c/9d7f4c88bea6e0d5a4f1bc0506a53a00e9fcb198de372bfe4d3652cef482/scipy-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb", size = 35003557, upload-time = "2026-02-23T00:18:54.74Z" }, + { url = "https://files.pythonhosted.org/packages/65/94/7698add8f276dbab7a9de9fb6b0e02fc13ee61d51c7c3f85ac28b65e1239/scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea", size = 37625856, upload-time = "2026-02-23T00:19:00.307Z" }, + { url = "https://files.pythonhosted.org/packages/b4/e0/e58fbde4a1a594c8be8114eb4aac1a55bcd6587047efc18a61eb1f5c0d30/scipy-1.17.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6", size = 32896429, upload-time = "2026-02-23T00:19:35.536Z" }, + { url = "https://files.pythonhosted.org/packages/f5/5f/f17563f28ff03c7b6799c50d01d5d856a1d55f2676f537ca8d28c7f627cd/scipy-1.17.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464", size = 35203952, upload-time = "2026-02-23T00:19:42.259Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a5/9afd17de24f657fdfe4df9a3f1ea049b39aef7c06000c13db1530d81ccca/scipy-1.17.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950", size = 34979063, upload-time = "2026-02-23T00:19:47.547Z" }, + { url = "https://files.pythonhosted.org/packages/8b/13/88b1d2384b424bf7c924f2038c1c409f8d88bb2a8d49d097861dd64a57b2/scipy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369", size = 37598449, upload-time = "2026-02-23T00:19:53.238Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a0/3cb6f4d2fb3e17428ad2880333cac878909ad1a89f678527b5328b93c1d4/scipy-1.17.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e", size = 33019667, upload-time = "2026-02-23T00:20:17.208Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c3/2d834a5ac7bf3a0c806ad1508efc02dda3c8c61472a56132d7894c312dea/scipy-1.17.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475", size = 35264159, upload-time = "2026-02-23T00:20:23.087Z" }, + { url = "https://files.pythonhosted.org/packages/4d/77/d3ed4becfdbd217c52062fafe35a72388d1bd82c2d0ba5ca19d6fcc93e11/scipy-1.17.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50", size = 35102771, upload-time = "2026-02-23T00:20:28.636Z" }, + { url = "https://files.pythonhosted.org/packages/bd/12/d19da97efde68ca1ee5538bb261d5d2c062f0c055575128f11a2730e3ac1/scipy-1.17.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca", size = 37665910, upload-time = "2026-02-23T00:20:34.743Z" }, + { url = "https://files.pythonhosted.org/packages/ef/f2/7cdb8eb308a1a6ae1e19f945913c82c23c0c442a462a46480ce487fdc0ac/scipy-1.17.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350", size = 32957007, upload-time = "2026-02-23T00:21:19.663Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2e/7eea398450457ecb54e18e9d10110993fa65561c4f3add5e8eccd2b9cd41/scipy-1.17.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118", size = 35221333, upload-time = "2026-02-23T00:21:25.278Z" }, + { url = "https://files.pythonhosted.org/packages/d9/77/5b8509d03b77f093a0d52e606d3c4f79e8b06d1d38c441dacb1e26cacf46/scipy-1.17.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068", size = 35042066, upload-time = "2026-02-23T00:21:31.358Z" }, + { url = "https://files.pythonhosted.org/packages/f9/df/18f80fb99df40b4070328d5ae5c596f2f00fffb50167e31439e932f29e7d/scipy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118", size = 37612763, upload-time = "2026-02-23T00:21:37.247Z" }, + { url = "https://files.pythonhosted.org/packages/86/f1/3383beb9b5d0dbddd030335bf8a8b32d4317185efe495374f134d8be6cce/scipy-1.17.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad", size = 33030397, upload-time = "2026-02-23T00:22:01.404Z" }, + { url = "https://files.pythonhosted.org/packages/41/68/8f21e8a65a5a03f25a79165ec9d2b28c00e66dc80546cf5eb803aeeff35b/scipy-1.17.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a", size = 35281163, upload-time = "2026-02-23T00:22:07.024Z" }, + { url = "https://files.pythonhosted.org/packages/84/8d/c8a5e19479554007a5632ed7529e665c315ae7492b4f946b0deb39870e39/scipy-1.17.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4", size = 35116291, upload-time = "2026-02-23T00:22:12.585Z" }, + { url = "https://files.pythonhosted.org/packages/52/52/e57eceff0e342a1f50e274264ed47497b59e6a4e3118808ee58ddda7b74a/scipy-1.17.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2", size = 37682317, upload-time = "2026-02-23T00:22:18.513Z" }, +] + +[[package]] +name = "seaborn" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "pandas", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/59/a451d7420a77ab0b98f7affa3a1d78a313d2f7281a57afb1a34bae8ab412/seaborn-0.13.2.tar.gz", hash = "sha256:93e60a40988f4d65e9f4885df477e2fdaff6b73a9ded434c1ab356dd57eefff7", size = 1457696, upload-time = "2024-01-25T13:21:52.551Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/11/00d3c3dfc25ad54e731d91449895a79e4bf2384dc3ac01809010ba88f6d5/seaborn-0.13.2-py3-none-any.whl", hash = "sha256:636f8336facf092165e27924f223d3c62ca560b1f2bb5dff7ab7fad265361987", size = 294914, upload-time = "2024-01-25T13:21:49.598Z" }, +] + +[[package]] +name = "secretstorage" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography", marker = "sys_platform == 'linux'" }, + { name = "jeepney", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" }, +] + +[[package]] +name = "send2trash" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c5/f0/184b4b5f8d00f2a92cf96eec8967a3d550b52cf94362dad1100df9e48d57/send2trash-2.1.0.tar.gz", hash = "sha256:1c72b39f09457db3c05ce1d19158c2cbef4c32b8bedd02c155e49282b7ea7459", size = 17255, upload-time = "2026-01-14T06:27:36.056Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/78/504fdd027da3b84ff1aecd9f6957e65f35134534ccc6da8628eb71e76d3f/send2trash-2.1.0-py3-none-any.whl", hash = "sha256:0da2f112e6d6bb22de6aa6daa7e144831a4febf2a87261451c4ad849fe9a873c", size = 17610, upload-time = "2026-01-14T06:27:35.218Z" }, +] + +[[package]] +name = "setuptools" +version = "83.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, +] + +[[package]] +name = "shear-psf-leakage" +version = "0.2.1" +source = { git = "https://github.com/CosmoStat/shear_psf_leakage.git?rev=develop#0b3c17523b77760c9259be15ba5bbba99d575d0a" } +dependencies = [ + { name = "camb", marker = "sys_platform == 'linux'" }, + { name = "cs-util", marker = "sys_platform == 'linux'" }, + { name = "emcee", marker = "sys_platform == 'linux'" }, + { name = "getdist", marker = "sys_platform == 'linux'" }, + { name = "gsl", marker = "sys_platform == 'linux'" }, + { name = "jupyter", marker = "sys_platform == 'linux'" }, + { name = "jupyter-server", marker = "sys_platform == 'linux'" }, + { name = "jupyterlab", marker = "sys_platform == 'linux'" }, + { name = "jupytext", marker = "sys_platform == 'linux'" }, + { name = "lenspack", marker = "sys_platform == 'linux'" }, + { name = "lmfit", marker = "sys_platform == 'linux'" }, + { name = "matplotlib", marker = "sys_platform == 'linux'" }, + { name = "notebook", marker = "sys_platform == 'linux'" }, + { name = "pandas", marker = "sys_platform == 'linux'" }, + { name = "pyccl", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "stats", marker = "sys_platform == 'linux'" }, + { name = "swig", marker = "sys_platform == 'linux'" }, + { name = "tornado", marker = "sys_platform == 'linux'" }, + { name = "tqdm", marker = "sys_platform == 'linux'" }, + { name = "treecorr", marker = "sys_platform == 'linux'" }, + { name = "uncertainties", marker = "sys_platform == 'linux'" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "skyproj" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astropy", marker = "sys_platform == 'linux'" }, + { name = "healsparse", marker = "sys_platform == 'linux'" }, + { name = "hpgeom", marker = "sys_platform == 'linux'" }, + { name = "matplotlib", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/cf/5158151ae6fc60459f430451b612a4bf2da07ebff2a9f52a37e73715a5ee/skyproj-2.5.0.tar.gz", hash = "sha256:cb8d5115927ca43cacdb6d92f00bb4fcc8563ee71028f755203ce1752e67c140", size = 8521587, upload-time = "2026-06-26T19:51:15.378Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/b1/1d86ee4ae4ae7a47e31cc770dc83057388ef16311498283ca90046b3bd04/skyproj-2.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cbfcbd4d3d600557ab19bb5cf79b750ec552da054e3be781bce88e996b9955d", size = 4146686, upload-time = "2026-06-26T19:51:04.011Z" }, + { url = "https://files.pythonhosted.org/packages/6e/e9/f2dd235dcb9f6c3b2fec36ef04d7dd36aef4daf9afc4ab4390d6230c96dc/skyproj-2.5.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78618084c9362b62ddbfd336ca1864c94d99e2a1f26d2fbf3466fd19dc702c58", size = 4146630, upload-time = "2026-06-26T19:51:10.552Z" }, +] + +[[package]] +name = "slicerator" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0c/52/f38586b82b2935f8b59a09b0a79c545a22ed062e728c9418bafeb51f61e0/slicerator-1.1.0.tar.gz", hash = "sha256:44010a7f5cd87680c07213b5cabe81d1fb71252962943e5373ee7d14605d6046", size = 38283, upload-time = "2022-04-07T18:54:08.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/ae/fa6cd331b364ad2bbc31652d025f5747d89cbb75576733dfdf8efe3e4d62/slicerator-1.1.0-py3-none-any.whl", hash = "sha256:167668d48c6d3a5ba0bd3d54b2688e81ee267dc20aef299e547d711e6f3c441a", size = 10274, upload-time = "2022-04-07T18:54:07.029Z" }, +] + +[[package]] +name = "smart-open" +version = "7.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/c6/22e7a2acd5d27941e85e0d7ede398da5abe2e4677d2265c924157247c32e/smart_open-7.7.1.tar.gz", hash = "sha256:9414ba5733e28309f29b28a303b0f1054ad23fe0275f1a1b600c80a724f4bd1a", size = 54952, upload-time = "2026-06-26T07:56:35.309Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/c5/14732c140fa0af0e59345949eaf9016c75ad3f076586f2413e3d3566a8da/smart_open-7.7.1-py3-none-any.whl", hash = "sha256:cb62dc45f519bf39b612564d326d2a17556f3a6056e3c3a86a07215d670d45bc", size = 65061, upload-time = "2026-06-26T07:56:33.685Z" }, +] + +[[package]] +name = "smmap" +version = "5.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/ea/49c993d6dfdd7338c9b1000a0f36817ed7ec84577ae2e52f890d1a4ff909/smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c", size = 22506, upload-time = "2026-03-09T03:43:26.1Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, +] + +[[package]] +name = "smokescreen" +version = "1.5.6" +source = { git = "https://github.com/UNIONS-WL/Smokescreen?rev=588a6b9b26560bd5ba3dd5ba342f3c40152644f9#588a6b9b26560bd5ba3dd5ba342f3c40152644f9" } +dependencies = [ + { name = "astropy", marker = "sys_platform == 'linux'" }, + { name = "cryptography", marker = "sys_platform == 'linux'" }, + { name = "jsonargparse", extra = ["signatures"], marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "pyccl", marker = "sys_platform == 'linux'" }, + { name = "sacc", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, +] + +[[package]] +name = "snakemake" +version = "9.23.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "conda-inject", marker = "sys_platform == 'linux'" }, + { name = "configargparse", marker = "sys_platform == 'linux'" }, + { name = "connection-pool", marker = "sys_platform == 'linux'" }, + { name = "docutils", marker = "sys_platform == 'linux'" }, + { name = "dpath", marker = "sys_platform == 'linux'" }, + { name = "gitpython", marker = "sys_platform == 'linux'" }, + { name = "humanfriendly", marker = "sys_platform == 'linux'" }, + { name = "immutables", marker = "sys_platform == 'linux'" }, + { name = "jinja2", marker = "sys_platform == 'linux'" }, + { name = "jsonschema", marker = "sys_platform == 'linux'" }, + { name = "nbformat", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "platformdirs", marker = "sys_platform == 'linux'" }, + { name = "psutil", marker = "sys_platform == 'linux'" }, + { name = "pulp", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "referencing", marker = "sys_platform == 'linux'" }, + { name = "requests", marker = "sys_platform == 'linux'" }, + { name = "smart-open", marker = "sys_platform == 'linux'" }, + { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, + { name = "snakemake-interface-executor-plugins", marker = "sys_platform == 'linux'" }, + { name = "snakemake-interface-logger-plugins", marker = "sys_platform == 'linux'" }, + { name = "snakemake-interface-report-plugins", marker = "sys_platform == 'linux'" }, + { name = "snakemake-interface-scheduler-plugins", marker = "sys_platform == 'linux'" }, + { name = "snakemake-interface-storage-plugins", marker = "sys_platform == 'linux'" }, + { name = "sqlmodel", marker = "sys_platform == 'linux'" }, + { name = "tabulate", marker = "sys_platform == 'linux'" }, + { name = "tenacity", marker = "sys_platform == 'linux'" }, + { name = "throttler", marker = "sys_platform == 'linux'" }, + { name = "wrapt", marker = "sys_platform == 'linux'" }, + { name = "yte", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/c2/45aa858e55edbb7dbc243ed60859f4f3d92d63ecdd67a80632aa3659fe6b/snakemake-9.23.1.tar.gz", hash = "sha256:ef8d698bfce66a6669cc29df7e344b0b367fd90956c725e5ae0a79a556e8e93f", size = 6802587, upload-time = "2026-06-18T09:42:57.653Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/65/2cb1c99117746bea8aff34dc434703f8f23d3876860e2ff3ebeccb754b0f/snakemake-9.23.1-py3-none-any.whl", hash = "sha256:9fdbfff25f1b442cdf7906ca06b465fc24b9bf72e2d51514fb93d83e3f8aeb35", size = 1162133, upload-time = "2026-06-18T09:42:55.541Z" }, +] + +[[package]] +name = "snakemake-interface-common" +version = "1.23.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argparse-dataclass", marker = "sys_platform == 'linux'" }, + { name = "configargparse", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/89/c3/592f832f6e5d2d31f749392e48e8401b7625dec668d3d365d8d28f2b6c30/snakemake_interface_common-1.23.0.tar.gz", hash = "sha256:6ed14531a461417659364a0dd0acc51b786af4e26fc15cc5e00ff3d9fcaffacc", size = 13960, upload-time = "2026-03-08T21:54:29.251Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/90/a4a3652e832667917383f8d34112bdb37b78a6d14efc049e22a2c8192524/snakemake_interface_common-1.23.0-py3-none-any.whl", hash = "sha256:43bb10102d07149f27787dfc65a7342b86ff772973e7f9ca4940c89cb412a11e", size = 17162, upload-time = "2026-03-08T21:54:28.074Z" }, +] + +[[package]] +name = "snakemake-interface-executor-plugins" +version = "9.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argparse-dataclass", marker = "sys_platform == 'linux'" }, + { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, + { name = "throttler", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/54/50/de06b284c45a8e94fb8e4a12d5235065e78b49b8f84329dc10fe39f4b7dd/snakemake_interface_executor_plugins-9.4.0.tar.gz", hash = "sha256:9d4138897beacbaadaedad94b63f948eaeb604b7fc78f9cf65ac57f090f2c066", size = 16549, upload-time = "2026-03-08T17:04:02.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/2c/4422eb14884bb4955ce65819799ff6452dcd3aa5f8e158540a287877bbcf/snakemake_interface_executor_plugins-9.4.0-py3-none-any.whl", hash = "sha256:d335c093185897e4e3654a39bab3a6f1c57dc1a54f998c92bd2d88640ab4a05e", size = 22567, upload-time = "2026-03-08T17:04:01.296Z" }, +] + +[[package]] +name = "snakemake-interface-logger-plugins" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/0c/3fa5d592663c65669a867526604aadc6fbc235fb9284e94b49c0ef59aa41/snakemake_interface_logger_plugins-2.1.0.tar.gz", hash = "sha256:c89a00d2a398490cecd91b6dc6db8049cba93712d82e1d8f3000f3040bf3791c", size = 15917, upload-time = "2026-05-20T15:12:35.259Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/43/cfdaf562303012024c8f48efebdfe258eef3d895665bd7e0000fdeb2411d/snakemake_interface_logger_plugins-2.1.0-py3-none-any.whl", hash = "sha256:601e3e0770dd1b460c1dcb7bd33f4fc2eb8af5016634b80d985cb993f674ef46", size = 14313, upload-time = "2026-05-20T15:12:34.186Z" }, +] + +[[package]] +name = "snakemake-interface-report-plugins" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/d6/6160ed98de665d6871dd356597dbf726688cc786e88668359ca37b7d9f54/snakemake_interface_report_plugins-1.3.0.tar.gz", hash = "sha256:fc9495298bec4e69721ab8afe6d6d88a86966fda2eeb003db56b9a88b86d5934", size = 4283, upload-time = "2025-10-31T10:52:36.55Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/f0/df73f6abc9b5910e43612ae28c7b6f666af80c4edd46a216ef47599ab6cb/snakemake_interface_report_plugins-1.3.0-py3-none-any.whl", hash = "sha256:78da3931f70e79eef51e5645a40b172929e555fe4d86ff45d6b856e521a379db", size = 7251, upload-time = "2025-10-31T10:52:35.474Z" }, +] + +[[package]] +name = "snakemake-interface-scheduler-plugins" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/d9/d480807d2cfc2d132bc760d877d45ec8fbe620a24200ec4d2697c4a26031/snakemake_interface_scheduler_plugins-2.0.2.tar.gz", hash = "sha256:2797e8fa9019d983132c2b403f14d6fcd3c5ad4c8d8a66b984b4740a71cacc46", size = 8642, upload-time = "2025-10-20T13:58:12.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/d0/f4e9894c8aaf37efe3bf1afe15ee3cf0546d82b2713a589e266ee47bf2ef/snakemake_interface_scheduler_plugins-2.0.2-py3-none-any.whl", hash = "sha256:b9ddfa508bd480711de1770dfb24f3b813cfa3cd0f862f0127ef721ae5346915", size = 10766, upload-time = "2025-10-20T13:58:11.898Z" }, +] + +[[package]] +name = "snakemake-interface-storage-plugins" +version = "4.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "humanfriendly", marker = "sys_platform == 'linux'" }, + { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, + { name = "tenacity", marker = "sys_platform == 'linux'" }, + { name = "throttler", marker = "sys_platform == 'linux'" }, + { name = "wrapt", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/6e/f3c5b2d621fd6a6b78d8cfc01fef6b926fe2c277f5ed77c5e4deeacb94eb/snakemake_interface_storage_plugins-4.4.1.tar.gz", hash = "sha256:b2b5bf05318af36955ebf2ce76c921c0fb06904ca98fb30e1657d88b7b7b6945", size = 14924, upload-time = "2026-03-16T11:16:01.075Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/49/1f72d8c05d8588f06a61f1af3deb125a4ff8169d2a41f1b839d62c824d1c/snakemake_interface_storage_plugins-4.4.1-py3-none-any.whl", hash = "sha256:6e075f448543cc3d0cb1ee5e6e5a83af9761e7085f3ce0922011685d098f231e", size = 18256, upload-time = "2026-03-16T11:15:59.582Z" }, +] + +[[package]] +name = "snowballstemmer" +version = "3.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/f8/0a71edf031f03c40db17503cb8ca78a69a171254e568e7db241b0ab57ea1/snowballstemmer-3.1.1.tar.gz", hash = "sha256:e07bbc54a0d798fe6010a12398422e62a8bfbba95c394fd0956ef58cb4d3e260", size = 123314, upload-time = "2026-06-03T00:56:40.194Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl", hash = "sha256:7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752", size = 104164, upload-time = "2026-06-03T00:56:38.614Z" }, +] + +[[package]] +name = "soupsieve" +version = "2.8.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/2c/0a5f6f8ee0d5589e48c7640213ed5175d52cf540a06725b628cc1a45d6ce/soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e", size = 121110, upload-time = "2026-05-24T13:55:57.154Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" }, +] + +[[package]] +name = "sp-validation" +version = "0.6.0" +source = { virtual = "." } +dependencies = [ + { name = "adjusttext", marker = "sys_platform == 'linux'" }, + { name = "astropy", marker = "sys_platform == 'linux'" }, + { name = "camb", marker = "sys_platform == 'linux'" }, + { name = "clmm", marker = "sys_platform == 'linux'" }, + { name = "colorama", marker = "sys_platform == 'linux'" }, + { name = "cosmo-numba", marker = "sys_platform == 'linux'" }, + { name = "cryptography", marker = "sys_platform == 'linux'" }, + { name = "cs-util", marker = "sys_platform == 'linux'" }, + { name = "emcee", marker = "sys_platform == 'linux'" }, + { name = "getdist", marker = "sys_platform == 'linux'" }, + { name = "h5py", marker = "sys_platform == 'linux'" }, + { name = "healpy", marker = "sys_platform == 'linux'" }, + { name = "healsparse", marker = "sys_platform == 'linux'" }, + { name = "importlib-metadata", marker = "sys_platform == 'linux'" }, + { name = "joblib", marker = "sys_platform == 'linux'" }, + { name = "jupyter", marker = "sys_platform == 'linux'" }, + { name = "jupyterlab", marker = "sys_platform == 'linux'" }, + { name = "jupytext", marker = "sys_platform == 'linux'" }, + { name = "lenspack", marker = "sys_platform == 'linux'" }, + { name = "lmfit", marker = "sys_platform == 'linux'" }, + { name = "matplotlib", marker = "sys_platform == 'linux'" }, + { name = "numba", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "opencv-python-headless", marker = "sys_platform == 'linux'" }, + { name = "pandas", marker = "sys_platform == 'linux'" }, + { name = "pyarrow", marker = "sys_platform == 'linux'" }, + { name = "pyccl", marker = "sys_platform == 'linux'" }, + { name = "pymaster", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "regions", marker = "sys_platform == 'linux'" }, + { name = "reproject", marker = "sys_platform == 'linux'" }, + { name = "sacc", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "seaborn", marker = "sys_platform == 'linux'" }, + { name = "shear-psf-leakage", marker = "sys_platform == 'linux'" }, + { name = "skyproj", marker = "sys_platform == 'linux'" }, + { name = "smokescreen", marker = "sys_platform == 'linux'" }, + { name = "statsmodels", marker = "sys_platform == 'linux'" }, + { name = "tqdm", marker = "sys_platform == 'linux'" }, + { name = "treecorr", marker = "sys_platform == 'linux'" }, + { name = "uncertainties", marker = "sys_platform == 'linux'" }, +] + +[package.optional-dependencies] +develop = [ + { name = "myst-parser", marker = "sys_platform == 'linux'" }, + { name = "numpydoc", marker = "sys_platform == 'linux'" }, + { name = "pytest", marker = "sys_platform == 'linux'" }, + { name = "pytest-cov", marker = "sys_platform == 'linux'" }, + { name = "ruff", marker = "sys_platform == 'linux'" }, + { name = "sphinx", marker = "sys_platform == 'linux'" }, + { name = "sphinxawesome-theme", marker = "sys_platform == 'linux'" }, + { name = "sphinxcontrib-bibtex", marker = "sys_platform == 'linux'" }, +] +docs = [ + { name = "myst-parser", marker = "sys_platform == 'linux'" }, + { name = "numpydoc", marker = "sys_platform == 'linux'" }, + { name = "sphinx", marker = "sys_platform == 'linux'" }, + { name = "sphinxawesome-theme", marker = "sys_platform == 'linux'" }, + { name = "sphinxcontrib-bibtex", marker = "sys_platform == 'linux'" }, +] +glass = [ + { name = "cosmology", marker = "sys_platform == 'linux'" }, + { name = "fitsio", marker = "sys_platform == 'linux'" }, + { name = "glass", marker = "sys_platform == 'linux'" }, + { name = "glass-ext-camb", marker = "sys_platform == 'linux'" }, +] +test = [ + { name = "pytest", marker = "sys_platform == 'linux'" }, + { name = "pytest-cov", marker = "sys_platform == 'linux'" }, + { name = "ruff", marker = "sys_platform == 'linux'" }, +] +workflow = [ + { name = "mpi4py", marker = "sys_platform == 'linux'" }, + { name = "snakemake", marker = "sys_platform == 'linux'" }, +] + +[package.metadata] +requires-dist = [ + { name = "adjusttext" }, + { name = "astropy", specifier = ">=5.0" }, + { name = "camb", specifier = ">=1.6" }, + { name = "clmm" }, + { name = "colorama" }, + { name = "cosmo-numba", git = "https://github.com/aguinot/cosmo-numba.git?rev=main" }, + { name = "cosmology", marker = "extra == 'glass'", specifier = "==2022.10.9" }, + { name = "cryptography" }, + { name = "cs-util", git = "https://github.com/CosmoStat/cs_util.git?rev=develop" }, + { name = "emcee" }, + { name = "fitsio", marker = "extra == 'glass'" }, + { name = "getdist", git = "https://github.com/benabed/getdist.git?rev=113cd22a9a0d013b6f72fe734be81f260f3d3be5" }, + { name = "glass", marker = "extra == 'glass'", specifier = "==2025.1" }, + { name = "glass-ext-camb", marker = "extra == 'glass'", specifier = "==2023.6" }, + { name = "h5py" }, + { name = "healpy" }, + { name = "healsparse" }, + { name = "importlib-metadata" }, + { name = "joblib", specifier = ">=0.13" }, + { name = "jupyter" }, + { name = "jupyterlab" }, + { name = "jupytext", specifier = ">=1.15" }, + { name = "lenspack" }, + { name = "lmfit" }, + { name = "matplotlib" }, + { name = "mpi4py", marker = "extra == 'workflow'" }, + { name = "myst-parser", marker = "extra == 'docs'", specifier = ">=4.0" }, + { name = "numba" }, + { name = "numpy", specifier = ">=2.0" }, + { name = "numpydoc", marker = "extra == 'docs'", specifier = ">=1.8" }, + { name = "opencv-python-headless" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "pyccl" }, + { name = "pymaster" }, + { name = "pytest", marker = "extra == 'test'" }, + { name = "pytest-cov", marker = "extra == 'test'" }, + { name = "pyyaml" }, + { name = "regions" }, + { name = "reproject" }, + { name = "ruff", marker = "extra == 'test'" }, + { name = "sacc", specifier = ">=0.12" }, + { name = "scipy", specifier = "<1.18" }, + { name = "seaborn" }, + { name = "shear-psf-leakage", git = "https://github.com/CosmoStat/shear_psf_leakage.git?rev=develop" }, + { name = "skyproj" }, + { name = "smokescreen", git = "https://github.com/UNIONS-WL/Smokescreen?rev=588a6b9b26560bd5ba3dd5ba342f3c40152644f9" }, + { name = "snakemake", marker = "extra == 'workflow'" }, + { name = "sp-validation", extras = ["test", "docs"], marker = "extra == 'develop'" }, + { name = "sphinx", marker = "extra == 'docs'", specifier = ">=8.0" }, + { name = "sphinxawesome-theme", marker = "extra == 'docs'", specifier = ">=5.3,!=6.0.3" }, + { name = "sphinxcontrib-bibtex", marker = "extra == 'docs'", specifier = ">=2.6" }, + { name = "statsmodels" }, + { name = "tqdm" }, + { name = "treecorr", specifier = ">=5.0" }, + { name = "uncertainties" }, +] +provides-extras = ["test", "docs", "glass", "workflow", "develop"] + +[[package]] +name = "sphinx" +version = "9.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "alabaster", marker = "sys_platform == 'linux'" }, + { name = "babel", marker = "sys_platform == 'linux'" }, + { name = "docutils", marker = "sys_platform == 'linux'" }, + { name = "imagesize", marker = "sys_platform == 'linux'" }, + { name = "jinja2", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "pygments", marker = "sys_platform == 'linux'" }, + { name = "requests", marker = "sys_platform == 'linux'" }, + { name = "roman-numerals", marker = "sys_platform == 'linux'" }, + { name = "snowballstemmer", marker = "sys_platform == 'linux'" }, + { name = "sphinxcontrib-applehelp", marker = "sys_platform == 'linux'" }, + { name = "sphinxcontrib-devhelp", marker = "sys_platform == 'linux'" }, + { name = "sphinxcontrib-htmlhelp", marker = "sys_platform == 'linux'" }, + { name = "sphinxcontrib-jsmath", marker = "sys_platform == 'linux'" }, + { name = "sphinxcontrib-qthelp", marker = "sys_platform == 'linux'" }, + { name = "sphinxcontrib-serializinghtml", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/f7/b1884cb3188ab181fc81fa00c266699dab600f927a964df02ec3d5d1916a/sphinx-9.1.0-py3-none-any.whl", hash = "sha256:c84fdd4e782504495fe4f2c0b3413d6c2bf388589bb352d439b2a3bb99991978", size = 3921742, upload-time = "2025-12-31T15:09:25.561Z" }, +] + +[[package]] +name = "sphinxawesome-theme" +version = "6.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4", marker = "sys_platform == 'linux'" }, + { name = "sphinx", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/3f/ba7f8dc2837c0ac362127c6cd491578c23ac4d4e8cc6a37726531be29256/sphinxawesome_theme-6.0.5.tar.gz", hash = "sha256:ed3d82b7f0e30e3d6f38f5245d89775e9472a80a45ffd73d6a0a4b9d05ac5736", size = 343775, upload-time = "2026-06-15T15:09:36.094Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/5c/31521b6a35410cdced12af394b715387537a262bdd1bcf08b2a34a9a4df9/sphinxawesome_theme-6.0.5-py3-none-any.whl", hash = "sha256:c18e60b06f164f181e3ef7540a4a237f1cf4e82d8909a964e5c65e6e8293f51c", size = 376755, upload-time = "2026-06-15T15:09:34.523Z" }, +] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1", size = 20053, upload-time = "2024-07-29T01:09:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5", size = 119300, upload-time = "2024-07-29T01:08:58.99Z" }, +] + +[[package]] +name = "sphinxcontrib-bibtex" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils", marker = "sys_platform == 'linux'" }, + { name = "pybtex", marker = "sys_platform == 'linux'" }, + { name = "pybtex-docutils", marker = "sys_platform == 'linux'" }, + { name = "sphinx", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/15/6a/8e0b2c2420286389e7fed78ff361ec30e2f1d58c8560af8d64df5e7b61e0/sphinxcontrib_bibtex-2.7.0.tar.gz", hash = "sha256:fee700f7aae29bb8f654c62913f00d34ac44fc0b8ca0fa67ac922ff4453addee", size = 120669, upload-time = "2026-05-06T09:29:24.935Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/c0/d28e62407f4733bbe0169287bc012f0ac3b4a2021066b285570654119c8b/sphinxcontrib_bibtex-2.7.0-py3-none-any.whl", hash = "sha256:28cf0ec7a957d1c7548d5749317ed472ce877e1b629f430f88e3789aa51f87b1", size = 40287, upload-time = "2026-05-06T09:29:23.253Z" }, +] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad", size = 12967, upload-time = "2024-07-29T01:09:23.417Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2", size = 82530, upload-time = "2024-07-29T01:09:21.945Z" }, +] + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9", size = 22617, upload-time = "2024-07-29T01:09:37.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705, upload-time = "2024-07-29T01:09:36.407Z" }, +] + +[[package]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8", size = 5787, upload-time = "2019-01-21T16:10:16.347Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071, upload-time = "2019-01-21T16:10:14.333Z" }, +] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab", size = 17165, upload-time = "2024-07-29T01:09:56.435Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb", size = 88743, upload-time = "2024-07-29T01:09:54.885Z" }, +] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d", size = 16080, upload-time = "2024-07-29T01:10:09.332Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.0.51" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "(platform_machine == 'AMD64' and sys_platform == 'linux') or (platform_machine == 'WIN32' and sys_platform == 'linux') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'amd64' and sys_platform == 'linux') or (platform_machine == 'ppc64le' and sys_platform == 'linux') or (platform_machine == 'win32' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/f1/a7a892f18d4d224e6b26f706531eafccc41e37594d37d304786969ee13cb/sqlalchemy-2.0.51.tar.gz", hash = "sha256:804dccd8a4a6242c4e30ad961e540e18a588f6527202f2d6791b01845d59fdc9", size = 9912201, upload-time = "2026-06-15T15:41:20.012Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/1c/71ee0f8a6b9d7316a1ccd30430b4c62b6c2e36adc96017a4e3a72dce49d6/sqlalchemy-2.0.51-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581921d849d6e6f994d560389192955e80e2950e18fcdfe2ccea863e01158e6e", size = 3343835, upload-time = "2026-06-15T16:19:42.613Z" }, + { url = "https://files.pythonhosted.org/packages/2b/7c/7ab9f9aadc5944fdd06612484ed7918fe376ad871a5f50404dc1536e0194/sqlalchemy-2.0.51-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d21ce524ab86c23046e992a5b81cb54c21079c6df6e78b8fc77d77cac70a6b9", size = 3358470, upload-time = "2026-06-15T16:26:38.011Z" }, + { url = "https://files.pythonhosted.org/packages/d0/7d/ff77169fee6186de145a7f2b87006c39638391130abbab2b1f63ac6ea583/sqlalchemy-2.0.51-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c5d98a2709840027f5a347c3af0a7c3d5f6c1ff93af2ca1c54494e23cba8f389", size = 3289874, upload-time = "2026-06-15T16:19:45.212Z" }, + { url = "https://files.pythonhosted.org/packages/6f/3b/6c505903710d781b55bc3141ee34a062bf9745a6b5bc7333305b9ed63b33/sqlalchemy-2.0.51-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1181256e0f16479691b5616d36375dc2620ad8332b25978763c3d206ad3f3f1d", size = 3321692, upload-time = "2026-06-15T16:26:39.747Z" }, + { url = "https://files.pythonhosted.org/packages/17/6b/2dce8369b199cb855110e056032f94a9f66dacc2237d3d39c115a86eac56/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:483b11bd46bf35fc14c52faf338b04300c9e6ce554bce9b11be85bfec3bc3195", size = 3284735, upload-time = "2026-06-15T16:19:46.934Z" }, + { url = "https://files.pythonhosted.org/packages/53/ff/dbc495b8a14da840faffb353857a72d4190113cac33727906fb997047f0f/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1bed1ee8b01da6088210aa9412023326fb98a599ba502e6118308601dcbef77f", size = 3302756, upload-time = "2026-06-15T16:26:41.336Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d5/fde8f4dddcf518ee15ab35a7c6a28acc32c8ba548d1d2aa451f96e6dbb0b/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:72ca54c952107ba5cd58854b67a5a6268631289d21651a1235396f3b98b47400", size = 3232055, upload-time = "2026-06-15T16:19:49.286Z" }, + { url = "https://files.pythonhosted.org/packages/67/d1/43d3a0ac955a58601c24fa23038b1c55ee3a1ec02c0f96ebb1eae2bcf614/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b3e693d15533a45cd5906f0589f9c35090bef6ef45bf1e8195c424aa0ae06a8d", size = 3269850, upload-time = "2026-06-15T16:26:43.017Z" }, + { url = "https://files.pythonhosted.org/packages/23/6b/2e0e38cf75c8780eca78d9b2e78164f8bcfd70125e5caa588ff5cbb9c9f4/sqlalchemy-2.0.51-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c45a496d6bc05dec41dcd4c3a2b183723f47473255c159cd80b503c8f246424d", size = 3282539, upload-time = "2026-06-15T16:19:51.065Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a1/e77854cb5336fd37dc3c6ae3b71de242c98caac5725120be0b526b31cbd0/sqlalchemy-2.0.51-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4004ada0aafe8ae1991b2cd1d99c6d9146126e123bd6f883c260d974aa012e54", size = 3287545, upload-time = "2026-06-15T16:26:44.735Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ab/9e17272fd4dac8df3b83c4fbe52b998a1c9d89a843c8c35ff29b74ff7364/sqlalchemy-2.0.51-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0f6bcad487aee1c638d707235682fc96f741de00663619881ab235400d03289e", size = 3230929, upload-time = "2026-06-15T16:19:52.625Z" }, + { url = "https://files.pythonhosted.org/packages/02/3c/52f408ea701781caee975606beccc48845f2aee8711ac29843d612c0306c/sqlalchemy-2.0.51-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:39a76529db6305693d8d4affa58ad5b5e2e18edd62daea628b29b97930b3513d", size = 3252888, upload-time = "2026-06-15T16:26:46.454Z" }, + { url = "https://files.pythonhosted.org/packages/bf/90/a1bb5c7cbba76b7bc1fbd586d0a5479a7bc9c27b4a8298f22ec9423b2bb3/sqlalchemy-2.0.51-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c6b36ed71f41942bdcd2ad2522be46bfce09d5705be5640ecf19bbc7660e4b7", size = 3566534, upload-time = "2026-06-15T15:58:35.024Z" }, + { url = "https://files.pythonhosted.org/packages/15/4b/481f1fed30e0e9e8dd24aecbb49f29eb57fe7657ece5cf06ee9b84bb97d8/sqlalchemy-2.0.51-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c2c62877097e1a0db401fba5cb4debee33265e5b2a55c4ccb489c02c53b4f72", size = 3535844, upload-time = "2026-06-15T16:02:43.973Z" }, + { url = "https://files.pythonhosted.org/packages/02/71/0aa64aeda645510af0a43f7d9ee70932f0d1dc4263aed34c50ee891d9df3/sqlalchemy-2.0.51-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0378d055e9e8cd6ce4d8dff683bdd3d7d413533c4ee51d67a2b1e0f9eacc0f23", size = 3475355, upload-time = "2026-06-15T15:58:36.592Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/6061db32316446135a3abae5f308d144ab988a34234726042da3e58b1c63/sqlalchemy-2.0.51-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6e46fc36029eff666391e0531e5387b62ce6c4f1d8e50b3fb3099eaca1b42522", size = 3486591, upload-time = "2026-06-15T16:02:45.346Z" }, + { url = "https://files.pythonhosted.org/packages/e2/22/dbf013a12ec759e54a34a119e9e217435b3f71b2dd5c61a7ade0a25dae87/sqlalchemy-2.0.51-py3-none-any.whl", hash = "sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5", size = 1944334, upload-time = "2026-06-15T16:09:22.418Z" }, +] + +[[package]] +name = "sqlmodel" +version = "0.0.37" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic", marker = "sys_platform == 'linux'" }, + { name = "sqlalchemy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fb/26/1d2faa0fd5a765267f49751de533adac6b9ff9366c7c6e7692df4f32230f/sqlmodel-0.0.37.tar.gz", hash = "sha256:d2c19327175794faf50b1ee31cc966764f55b1dedefc046450bc5741a3d68352", size = 85527, upload-time = "2026-02-21T16:39:47.038Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/e1/7c8d18e737433f3b5bbe27b56a9072a9fcb36342b48f1bef34b6da1d61f2/sqlmodel-0.0.37-py3-none-any.whl", hash = "sha256:2137a4045ef3fd66a917a7717ada959a1ceb3630d95e1f6aaab39dd2c0aef278", size = 27224, upload-time = "2026-02-21T16:39:47.781Z" }, +] + +[[package]] +name = "stack-data" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asttokens", marker = "sys_platform == 'linux'" }, + { name = "executing", marker = "sys_platform == 'linux'" }, + { name = "pure-eval", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, +] + +[[package]] +name = "stats" +version = "0.1.2a0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/87/43/f2d8d8e2660740ecca9c0fab85a07bc02da27d1fcc5071ee481082888720/stats-0.1.2a.tar.gz", hash = "sha256:854e59fa99f92ce8319c16149ef8e9939dba3e2237bbb0fb6a6094c60155bd4c", size = 127625, upload-time = "2010-12-31T09:02:15.131Z" } + +[[package]] +name = "statsmodels" +version = "0.14.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "pandas", marker = "sys_platform == 'linux'" }, + { name = "patsy", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/81/e8d74b34f85285f7335d30c5e3c2d7c0346997af9f3debf9a0a9a63de184/statsmodels-0.14.6.tar.gz", hash = "sha256:4d17873d3e607d398b85126cd4ed7aad89e4e9d89fc744cdab1af3189a996c2a", size = 20689085, upload-time = "2025-12-05T23:08:39.522Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/f5/3a73b51e6450c31652c53a8e12e24eac64e3824be816c0c2316e7dbdcb7d/statsmodels-0.14.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8c00a42863e4f4733ac9d078bbfad816249c01451740e6f5053ecc7db6d6368", size = 10058649, upload-time = "2025-12-05T23:10:12.775Z" }, + { url = "https://files.pythonhosted.org/packages/81/68/dddd76117df2ef14c943c6bbb6618be5c9401280046f4ddfc9fb4596a1b8/statsmodels-0.14.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19b58cf7474aa9e7e3b0771a66537148b2df9b5884fbf156096c0e6c1ff0469d", size = 10339446, upload-time = "2025-12-05T23:10:28.503Z" }, + { url = "https://files.pythonhosted.org/packages/56/4a/dce451c74c4050535fac1ec0c14b80706d8fc134c9da22db3c8a0ec62c33/statsmodels-0.14.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:81e7dcc5e9587f2567e52deaff5220b175bf2f648951549eae5fc9383b62bc37", size = 10368705, upload-time = "2025-12-05T23:10:44.339Z" }, + { url = "https://files.pythonhosted.org/packages/ee/77/0ec96803eba444efd75dba32f2ef88765ae3e8f567d276805391ec2c98c6/statsmodels-0.14.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89ee7d595f5939cc20bf946faedcb5137d975f03ae080f300ebb4398f16a5bd4", size = 10060269, upload-time = "2025-12-05T23:11:46.338Z" }, + { url = "https://files.pythonhosted.org/packages/10/b9/fd41f1f6af13a1a1212a06bb377b17762feaa6d656947bf666f76300fc05/statsmodels-0.14.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:730f3297b26749b216a06e4327fe0be59b8d05f7d594fb6caff4287b69654589", size = 10324155, upload-time = "2025-12-05T23:12:01.805Z" }, + { url = "https://files.pythonhosted.org/packages/ee/0f/a6900e220abd2c69cd0a07e3ad26c71984be6061415a60e0f17b152ecf08/statsmodels-0.14.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f1c08befa85e93acc992b72a390ddb7bd876190f1360e61d10cf43833463bc9c", size = 10349765, upload-time = "2025-12-05T23:12:18.018Z" }, + { url = "https://files.pythonhosted.org/packages/c1/98/b0dfb4f542b2033a3341aa5f1bdd97024230a4ad3670c5b0839d54e3dcab/statsmodels-0.14.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e443e7077a6e2d3faeea72f5a92c9f12c63722686eb80bb40a0f04e4a7e267ad", size = 10090802, upload-time = "2025-12-05T23:13:20.653Z" }, + { url = "https://files.pythonhosted.org/packages/34/0e/2408735aca9e764643196212f9069912100151414dd617d39ffc72d77eee/statsmodels-0.14.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3414e40c073d725007a6603a18247ab7af3467e1af4a5e5a24e4c27bc26673b4", size = 10337587, upload-time = "2025-12-05T23:13:37.597Z" }, + { url = "https://files.pythonhosted.org/packages/0f/36/4d44f7035ab3c0b2b6a4c4ebb98dedf36246ccbc1b3e2f51ebcd7ac83abb/statsmodels-0.14.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a518d3f9889ef920116f9fa56d0338069e110f823926356946dae83bc9e33e19", size = 10363350, upload-time = "2025-12-05T23:13:53.08Z" }, +] + +[[package]] +name = "swig" +version = "4.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/99/0556919f96b1c291a7ff83793adb328e2a1a5e6cfb379d5acc637cdec959/swig-4.4.1.tar.gz", hash = "sha256:db9a625653f6454530a6c6553d17cf576d7aa2d253c76dc8cdd5979f0cb15012", size = 25956, upload-time = "2025-12-27T03:25:49.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/4d/860e5475fe38b9c7dc36f61d0c370a62b6cc725d2bd11ada1d22a60ff5f7/swig-4.4.1-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:520fde8805b4775ef3814576929b8c44d666220b06d8da625f292192f9e74fe8", size = 1917511, upload-time = "2025-12-27T03:25:18.064Z" }, + { url = "https://files.pythonhosted.org/packages/84/e1/5cd4de23ff2b1d225279f7c7826191b1dbdb77fc858fe12ee7be2562ddb7/swig-4.4.1-py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:d8cca45a0e8d1e045a1490ea2d05bc1c49e2c266dbbe5954048e2a93d125ee07", size = 2066284, upload-time = "2025-12-27T03:25:19.624Z" }, + { url = "https://files.pythonhosted.org/packages/96/42/d49992f3c39c90d3d46c431a4afea60fc16bae542cf07f394b298a1b21b5/swig-4.4.1-py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:bfe6ce6189fc8b983f65f5ed3db8e1d8be03accb0ec3869f986a8343f014e4d9", size = 1912492, upload-time = "2025-12-27T03:25:21.843Z" }, + { url = "https://files.pythonhosted.org/packages/10/2e/4baa50fcf932f9d58bc8394b211c7daf3b7ec1e53c08d560142f1bc99cb9/swig-4.4.1-py3-none-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9c98d7b882b6debe3cbb87e6503858e104162d7ce5ab4cfc23425da2e26e51b5", size = 1979872, upload-time = "2025-12-27T03:25:23.48Z" }, + { url = "https://files.pythonhosted.org/packages/fa/7b/e3a14d053fa18b0d2e14efcc21883816964ddbe52a0c43018e195a99aba2/swig-4.4.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ae3da2bf679a4c942a2c100789395d4d167e7da8286018124e4665f5eff43e31", size = 1895896, upload-time = "2025-12-27T03:25:24.98Z" }, + { url = "https://files.pythonhosted.org/packages/f0/a6/c961a911385a4cc4ae10b4dcbcf5dede0407530e59870ce6e4f7bacce47d/swig-4.4.1-py3-none-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7945f153f7fc93f1ab280f44a3740d6bbfeec3829f954784ba83990fc61a11a8", size = 1912119, upload-time = "2025-12-27T03:25:27.661Z" }, + { url = "https://files.pythonhosted.org/packages/86/4b/a08a2673a2fd3185932db110de67a9f6fdfcece6bf9220e18502bf745f7e/swig-4.4.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:5a9c74b87151d8247d802bda33f78cab9f27c759f0605b760a8f9c9893b5606b", size = 1885810, upload-time = "2025-12-27T03:25:29.562Z" }, + { url = "https://files.pythonhosted.org/packages/7e/38/efa343b506221a9d43bb1132f08b29fcb07d4113165f598d8a1ea733a1e4/swig-4.4.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a95ae66aebb8ccce4c50febd496e6927b888d98aba272dd5642fe9312bf61a3b", size = 2950985, upload-time = "2025-12-27T03:25:31.586Z" }, + { url = "https://files.pythonhosted.org/packages/de/0b/5b46e61798982f3cb0ed54fc30a30bad5fa6e3bdd1d9177e439976f7a7e5/swig-4.4.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a8f06227e6472832dad3ae34d5bd88710b7bbb109f2927e14a21e6725146e944", size = 2781782, upload-time = "2025-12-27T03:25:33.534Z" }, + { url = "https://files.pythonhosted.org/packages/17/e6/a15efd9cbf39b1289ee7c1acb8d18824a0c2532b85cdbd93acfdf15ee000/swig-4.4.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:98d61b45215982dd3a71745dfc31599176ed79d286b58bc712c10ed031432fdf", size = 3272993, upload-time = "2025-12-27T03:25:35.862Z" }, + { url = "https://files.pythonhosted.org/packages/d6/fa/516b95f9966c57acf1da9f2eb22e4e63104452d6e28686d57b216171ee7c/swig-4.4.1-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:b6909a8bd68037ea0df061a52c664431b062dbdfb1eb90768719518c120139d1", size = 3189093, upload-time = "2025-12-27T03:25:38.158Z" }, + { url = "https://files.pythonhosted.org/packages/0c/03/88ca92d3d8e9dd765a66985202db23745bd26a368570a8d8db9a218bfe90/swig-4.4.1-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:6a6a6331a5c7759ba5a910aa83aa5d33022b8e691b6cb466635c7fc2238facd2", size = 2967789, upload-time = "2025-12-27T03:25:40.24Z" }, + { url = "https://files.pythonhosted.org/packages/bf/97/5ca3d750f294048c983edfad1f2ad093a83480494fccb2f2e2496304f965/swig-4.4.1-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:2d076358cded172d2f44b3c617923c593dbae664b3fa45051641f028f7f74010", size = 3203253, upload-time = "2025-12-27T03:25:41.953Z" }, + { url = "https://files.pythonhosted.org/packages/97/44/96586955c0f08b0b9b6c8b2b68ecb43a815da94d5362cfb6691ede3b9fdf/swig-4.4.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:43a0a876b4660f6addb6ae72d14a72324dc1388e1c168ed38bb00fd123048749", size = 3060721, upload-time = "2025-12-27T03:25:44.322Z" }, +] + +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + +[[package]] +name = "tables" +version = "3.11.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "blosc2", marker = "sys_platform == 'linux'" }, + { name = "numexpr", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "py-cpuinfo", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/a3/d213ebe7376d48055bd55a29cd9f99061afa0dcece608f94a5025d797b0a/tables-3.11.1.tar.gz", hash = "sha256:78abcf413091bc7c1e4e8c10fbbb438d1ac0b5a87436c5b972c3e8253871b6fb", size = 4790533, upload-time = "2026-03-01T11:43:36.036Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/a3/ec228901fca4c996306b17f5c60a4105144df0bbd07b3a4a816f91f37b4a/tables-3.11.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56bf6fb9132ead989b7e76695d7613d6d08f071a8019038d6565ba90c66b9f3e", size = 6903733, upload-time = "2026-03-01T11:42:58.349Z" }, + { url = "https://files.pythonhosted.org/packages/99/29/c2dc674ea70fa9a4819417289a9c0d3e4780835beeed573eb66964cfb763/tables-3.11.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e78fe190fdeb4afe430b79651bae2a4f341904eb85aa8dbafe5f1caee1c7f67", size = 7241357, upload-time = "2026-03-01T11:43:03.938Z" }, + { url = "https://files.pythonhosted.org/packages/59/3c/f4a694aa744d2b14d536e172c28dd70c84445f4787083a82d6d44a39e39f/tables-3.11.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a9c35f87fcb6a48c79fbc4e3ab15ca8f6053c4ce13063d6ca2ec36cbb58f40f", size = 7014135, upload-time = "2026-03-01T11:43:22.359Z" }, + { url = "https://files.pythonhosted.org/packages/45/82/94d4320d6c0fe5bd55230eec90cd142d58cda37b7cce00a318ac2a6abd93/tables-3.11.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4cf3218b76ba78d156d6ee75c19fb757d50682f6c7b4905370441afbfc9d77f3", size = 7349293, upload-time = "2026-03-01T11:43:27.569Z" }, +] + +[[package]] +name = "tables-io" +version = "1.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click", marker = "sys_platform == 'linux'" }, + { name = "deprecated", marker = "sys_platform == 'linux'" }, + { name = "h5py", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "pandas", marker = "sys_platform == 'linux'" }, + { name = "tables", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/1b/61bed3c55ce4602be5379362ce939b71a2e707e856ee497c1d1e72f8ff3d/tables_io-1.1.2.tar.gz", hash = "sha256:ea27eac8b22ff4e3500afd92140794bd41161f768a2c84520b91b3b58e945182", size = 91394, upload-time = "2026-06-18T16:21:20.896Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/68/025307ebd3c54c65e540d2a4ced11d0e5aa370c8ddb6d0e83066e5a39054/tables_io-1.1.2-py3-none-any.whl", hash = "sha256:4361e04dfcc10471bf9a3940d9ee6a1f903ac76ece51d32ef7fec7ea572b20c0", size = 39206, upload-time = "2026-06-18T16:21:19.577Z" }, +] + +[[package]] +name = "tabulate" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, +] + +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + +[[package]] +name = "terminado" +version = "0.18.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ptyprocess", marker = "os_name != 'nt' and sys_platform == 'linux'" }, + { name = "tornado", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/11/965c6fd8e5cc254f1fe142d547387da17a8ebfd75a3455f637c663fb38a0/terminado-0.18.1.tar.gz", hash = "sha256:de09f2c4b85de4765f7714688fff57d3e75bad1f909b589fde880460c753fd2e", size = 32701, upload-time = "2024-03-12T14:34:39.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl", hash = "sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0", size = 14154, upload-time = "2024-03-12T14:34:36.569Z" }, +] + +[[package]] +name = "threadpoolctl" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, +] + +[[package]] +name = "throttler" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/3f/47baf510c31e0e52ac0d80d9071e5e166ca069167fee4a6c13841f9d5f5f/throttler-1.2.3.tar.gz", hash = "sha256:d2f5b0b499d62f1fc984dcac8043450b606549b0097753a9c8a707f7427c27e1", size = 11792, upload-time = "2026-01-27T00:48:11.552Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/70/42d8796acc57c8bcd9ae395b1a6a0bbc833f738492a8ed192a44ccd58035/throttler-1.2.3-py3-none-any.whl", hash = "sha256:241ea3e97438dec4dc2f31ddc56dbd96262787a9b1d0598adfcc0bada1134b66", size = 9704, upload-time = "2026-01-27T00:48:09.544Z" }, +] + +[[package]] +name = "tifffile" +version = "2026.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b7/38/5e2ecef5af2f4fd4a89bb8d6240de9458bab4d51a4cbd97aeb3a0cd618e2/tifffile-2026.6.1.tar.gz", hash = "sha256:626c892c0e899d959b9438e7c0e1491dc154a7fead1f1f37a991724a50eceba9", size = 429694, upload-time = "2026-05-31T23:57:12.165Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/59/208f71d70ddc6184f79b8c6d87d46eb7d7b12c19186a817dec9c9c3f3693/tifffile-2026.6.1-py3-none-any.whl", hash = "sha256:0d7382d2769b855b81ce358528e2b40c16d48aa39031746efa81215205332a8d", size = 267108, upload-time = "2026-05-31T23:57:10.597Z" }, +] + +[[package]] +name = "tinycss2" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "webencodings", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/ae/2ca4913e5c0f09781d75482874c3a95db9105462a92ddd303c7d285d3df2/tinycss2-1.5.1.tar.gz", hash = "sha256:d339d2b616ba90ccce58da8495a78f46e55d4d25f9fd71dfd526f07e7d53f957", size = 88195, upload-time = "2025-11-23T10:29:10.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/45/c7b5c3168458db837e8ceab06dc77824e18202679d0463f0e8f002143a97/tinycss2-1.5.1-py3-none-any.whl", hash = "sha256:3415ba0f5839c062696996998176c4a3751d18b7edaaeeb658c9ce21ec150661", size = 28404, upload-time = "2025-11-23T10:29:08.676Z" }, +] + +[[package]] +name = "toolz" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/d6/114b492226588d6ff54579d95847662fc69196bdeec318eb45393b24c192/toolz-1.1.0.tar.gz", hash = "sha256:27a5c770d068c110d9ed9323f24f1543e83b2f300a687b7891c1a6d56b697b5b", size = 52613, upload-time = "2025-10-17T04:03:21.661Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/12/5911ae3eeec47800503a238d971e51722ccea5feb8569b735184d5fcdbc0/toolz-1.1.0-py3-none-any.whl", hash = "sha256:15ccc861ac51c53696de0a5d6d4607f99c210739caf987b5d2054f3efed429d8", size = 58093, upload-time = "2025-10-17T04:03:20.435Z" }, +] + +[[package]] +name = "tornado" +version = "6.5.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/64/24/95ec527ad67b76d59299e5465b3935d05e4294b7e0290a3924b7487df30b/tornado-6.5.7.tar.gz", hash = "sha256:66c513a76cda70d53907bc27cf1447557699c2e95aa48ba27a442ff61c3ddfc2", size = 519252, upload-time = "2026-06-08T17:34:51.232Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/d8/ef374952fd5da67d4463122c2b8e5a96536ec10b4b339254c6dcde81d01c/tornado-6.5.7-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8d759e71906ee783f8867b93bf26a265743da4c1e2f4a018464c1ba019862972", size = 449774, upload-time = "2026-06-08T17:34:41.204Z" }, + { url = "https://files.pythonhosted.org/packages/35/37/d434c73f4c6e014b745b9b37085f34f40c022f007efff3d7fe65991899f3/tornado-6.5.7-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a46347a18f23fb92b396beebe0fb78f61dda0cc302445202c16203d8a18848b", size = 450745, upload-time = "2026-06-08T17:34:42.531Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2b/56b9aff361d7f1ab728a805ec7d7ea835f8807afa9f5cc690ea0e630efb9/tornado-6.5.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7778b30bef919231265e91c69963ce0f49a1e9c07ac900bbe75b19ce2575ba92", size = 450578, upload-time = "2026-06-08T17:34:43.787Z" }, + { url = "https://files.pythonhosted.org/packages/02/30/a7444fb23aa76860a14198fab96ac79f1866b0a6e19e26c4381b0938e50f/tornado-6.5.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e726f0c75da7726eec023aa62751ff8878bd2737e34fbdd33b1ae5897d2200f5", size = 449985, upload-time = "2026-06-08T17:34:45.326Z" }, +] + +[[package]] +name = "tqdm" +version = "4.68.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/5f/57ff8b434839e70dab45601284ea413e947a63799891b7553e5960a793a8/tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520", size = 792418, upload-time = "2026-07-07T09:58:18.369Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2", size = 676612, upload-time = "2026-07-07T09:58:16.256Z" }, +] + +[[package]] +name = "traitlets" +version = "5.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/a9/a2584b8313b89f94869ddb3c4074617a691de1812a614d2d50e32ca5a7a6/traitlets-5.15.1.tar.gz", hash = "sha256:7b1c07854fe25acb39e009bae49f11b79ff6cbb2f27999104e9110e7a6b53722", size = 163344, upload-time = "2026-06-03T12:26:06.181Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/8d/1080ee4c231f361b6ce4470d556c8c435b67c7e0753aaa641497ee92f88b/traitlets-5.15.1-py3-none-any.whl", hash = "sha256:770a53705f84b81ac107e83a1b3328ff2dae16094d8fc3cfc004e4b22dfd8e92", size = 85858, upload-time = "2026-06-03T12:26:04.395Z" }, +] + +[[package]] +name = "transformcl" +version = "2026.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "array-api-compat", marker = "sys_platform == 'linux'" }, + { name = "flt", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/ce/ae8373b67259c3ea6033425161f7d195387b1e68b17d4d00eee1b68e5edd/transformcl-2026.1.tar.gz", hash = "sha256:14551c33d164f3db24b9141f1f77553182a4b9c789f883476eb33d87ea4d8f13", size = 4850, upload-time = "2026-03-02T19:18:21.221Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/98/de7a05005238e03bdc57cd2885c27ad9140539bbaddf2b6ad2b2a0aa9863/transformcl-2026.1-py3-none-any.whl", hash = "sha256:fe9c380074ef783a4f854e0d326eadac677d6a35bc683e3b429d5b49a8e57223", size = 3597, upload-time = "2026-03-02T19:18:19.832Z" }, +] + +[[package]] +name = "treecorr" +version = "5.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lsstdesc-coord", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "pybind11", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "setuptools", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/95/ff399cb37b9e3d7c758f8cf0d5757f06a38093f223f2a65d7cc5b50313d5/treecorr-5.1.3.tar.gz", hash = "sha256:d921ccace064325a8ebb93f371c624f13fcc9559373e3b37cc216cb94ae81ed2", size = 1146147, upload-time = "2026-03-18T18:38:03.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/74/1cc590e4fe1afb69ec2f10a67ee8e0a92ecafd36512052755362ff302d9c/treecorr-5.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:596b6f5eea3115a9e211dc83ec6465887983f41b24e484e08fc47170c6cf4e26", size = 20941767, upload-time = "2026-03-18T18:37:37.874Z" }, + { url = "https://files.pythonhosted.org/packages/6f/bb/5ba5aa1021fce0cf459fa264cb0d298dfd095ddb48d63fb6589b90fdefac/treecorr-5.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:660dacd47a9af18790de279bcb6e7f375369096ad77e7cc64bf21a9b78f6c430", size = 19785388, upload-time = "2026-03-18T18:37:40.573Z" }, + { url = "https://files.pythonhosted.org/packages/be/92/be01211263a4fa488e768c9288c6f9b0f4402870423a88de4d60031b3384/treecorr-5.1.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73c938c6c31e1bad4d5338ad4519d1d78eedf3b7ef579b1e3a19d983e220ed93", size = 20940568, upload-time = "2026-03-18T18:37:44.966Z" }, + { url = "https://files.pythonhosted.org/packages/dc/a3/69e5a16aa33600ba22d28a47c6d8de3b3118292e79f9ae77dc69a3df7c77/treecorr-5.1.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc17b1f623bae6769148f57cac8c3a295e26e5d6215698117bd97e14c8bb0d6b", size = 19781991, upload-time = "2026-03-18T18:37:47.468Z" }, +] + +[[package]] +name = "typeshed-client" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-resources", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/54/5d/97d6fa8c204b0d42be931e7a568b306cda43f93ff45f8b47537ee61390de/typeshed_client-2.12.0.tar.gz", hash = "sha256:54dcfa25fcff82cedcb1b51c03c1b3c51a614097aab8fe49e4316aff6f79d9c7", size = 529457, upload-time = "2026-06-02T04:00:51.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/98/3aef1587ec9b3a5cd062b8d8910545a0ee3715c31fd65b1f2386b27b94af/typeshed_client-2.12.0-py3-none-any.whl", hash = "sha256:da3ef54a0e60c0f45f59006b73a3cb20b1acbf7a784e982476d8193e8828ddf5", size = 796929, upload-time = "2026-06-02T04:00:49.643Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, +] + +[[package]] +name = "uncertainties" +version = "3.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/0c/cb09f33b26955399c675ab378e4063ed7e48422d3d49f96219ab0be5eba9/uncertainties-3.2.3.tar.gz", hash = "sha256:76a5653e686f617a42922d546a239e9efce72e6b35411b7750a1d12dcba03031", size = 160492, upload-time = "2025-04-21T19:58:28.63Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/5e/f1e1dd319e35e962a4e00b33150a8868b6329cc1d19fd533436ba5488f09/uncertainties-3.2.3-py3-none-any.whl", hash = "sha256:313353900d8f88b283c9bad81e7d2b2d3d4bcc330cbace35403faaed7e78890a", size = 60118, upload-time = "2025-04-21T19:58:26.864Z" }, +] + +[[package]] +name = "uri-template" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/31/c7/0336f2bd0bcbada6ccef7aaa25e443c118a704f828a0620c6fa0207c1b64/uri-template-1.3.0.tar.gz", hash = "sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7", size = 21678, upload-time = "2023-06-21T01:49:05.374Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl", hash = "sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363", size = 11140, upload-time = "2023-06-21T01:49:03.467Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "vos" +version = "3.6.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aenum", marker = "sys_platform == 'linux'" }, + { name = "cadcutils", marker = "sys_platform == 'linux'" }, + { name = "html2text", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3e/6c/445bbedf2f90e71c09ff890aa8b409af5d074b1578eef383117cbcdfd09a/vos-3.6.4.tar.gz", hash = "sha256:67e4e24689ee881124b6670d7d2fc6527cc404687fc527fffc20cf5b1bd4c0bd", size = 103445, upload-time = "2025-12-12T00:20:56.398Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/21/e2d351883aa11b9175223a78d3a77b0fedf86461596cdbfb019893ca9260/vos-3.6.4-py3-none-any.whl", hash = "sha256:4a7b2248bad0f50da465a47ab1c819dff8378c189c184c2d8e2c17a714c62f17", size = 153404, upload-time = "2025-12-12T00:20:53.232Z" }, +] + +[[package]] +name = "wcwidth" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/74/c6428f875774288bec1396f5bfcbc2d925700a4dad61727fd5f2b12f249d/wcwidth-0.8.2.tar.gz", hash = "sha256:91fbef97204b96a3d4d421609b80340b760cf33e26da123ff243d76b1fda8dda", size = 1466253, upload-time = "2026-06-29T18:11:11.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl", hash = "sha256:d63947694a0539a1d51e01eda7caf800c291020e6cdd7e28ad7b14dd33ad4f85", size = 323166, upload-time = "2026-06-29T18:11:09.888Z" }, +] + +[[package]] +name = "webcolors" +version = "25.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/7a/eb316761ec35664ea5174709a68bbd3389de60d4a1ebab8808bfc264ed67/webcolors-25.10.0.tar.gz", hash = "sha256:62abae86504f66d0f6364c2a8520de4a0c47b80c03fc3a5f1815fedbef7c19bf", size = 53491, upload-time = "2025-10-31T07:51:03.977Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl", hash = "sha256:032c727334856fc0b968f63daa252a1ac93d33db2f5267756623c210e57a4f1d", size = 14905, upload-time = "2025-10-31T07:51:01.778Z" }, +] + +[[package]] +name = "webencodings" +version = "0.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47/webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923", size = 9721, upload-time = "2017-04-05T20:21:34.189Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", size = 11774, upload-time = "2017-04-05T20:21:32.581Z" }, +] + +[[package]] +name = "websocket-client" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, +] + +[[package]] +name = "widgetsnbextension" +version = "4.0.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/f4/c67440c7fb409a71b7404b7aefcd7569a9c0d6bd071299bf4198ae7a5d95/widgetsnbextension-4.0.15.tar.gz", hash = "sha256:de8610639996f1567952d763a5a41af8af37f2575a41f9852a38f947eb82a3b9", size = 1097402, upload-time = "2025-11-01T21:15:55.178Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl", hash = "sha256:8156704e4346a571d9ce73b84bee86a29906c9abfd7223b7228a28899ccf3366", size = 2196503, upload-time = "2025-11-01T21:15:53.565Z" }, +] + +[[package]] +name = "wrapt" +version = "2.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/a4/282c8e64300a59fc834518a54bf0afabb4ff9218b5fa76958b450459a844/wrapt-2.2.2.tar.gz", hash = "sha256:0788e321027c999bf221b667bd4a54aaefd1a36283749a860ac3eb77daed0302", size = 129068, upload-time = "2026-06-20T23:49:44.49Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/b3/7e6e9fcf4fe7e1b69a49fe6cc5a44e8224bab6283c5233c97e132f14908e/wrapt-2.2.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e096c9d39a59b35b63c9aacfbbbec2088ff51ff1fc31051acc60a07f42f273a", size = 169828, upload-time = "2026-06-20T23:48:02.719Z" }, + { url = "https://files.pythonhosted.org/packages/0b/43/894f132d857ed5a9904d937baf368badcbe5ea9e436e2f1930fe21c9f1f0/wrapt-2.2.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d1a6050405bf334be33bf66296f113563622972a34900ae6fa60fd283a1a900", size = 171544, upload-time = "2026-06-20T23:48:04.266Z" }, + { url = "https://files.pythonhosted.org/packages/29/de/3c833e03725b477e9ea34028224dd21a48781830101e4e036f77e8b6b102/wrapt-2.2.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10adb01371408c6de504a6658b9886480f1a4919a83752748a387a504a21df79", size = 160663, upload-time = "2026-06-20T23:48:05.708Z" }, + { url = "https://files.pythonhosted.org/packages/33/be/27edce350b24e3054d9d047f65f16d4c4d4c1f3f31c4278a1f8a95c723c8/wrapt-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3442eee2a5798f9b451f1b2cd7518ce8b7e28a2a364696c414460a0e295c012a", size = 169387, upload-time = "2026-06-20T23:48:07.243Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c4/9fd9679af8bf38e146652c7f47b6b352c3e5795b4ad1c0b7f94e15ac2aa7/wrapt-2.2.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6c99012a22f735a85eed7c4b86a3e99c30fdd57d9e115b2b45f796264b58d0bf", size = 158849, upload-time = "2026-06-20T23:48:08.91Z" }, + { url = "https://files.pythonhosted.org/packages/bc/c2/aa6c0c2206803068c6859dabe01f8c84c43744da93d4c67b8946d21655ee/wrapt-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b686cfc008776a3952d6213cb296ed7f45d782a8453936406faa89eac0835ab", size = 168147, upload-time = "2026-06-20T23:48:10.374Z" }, + { url = "https://files.pythonhosted.org/packages/15/ff/63ad96f98eb58a742b1a20d80f21da88924405910149950b912368150468/wrapt-2.2.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb18fc51e813df0d9c98049e3bf2298a5495a648602040e21fa3c7329371159e", size = 167882, upload-time = "2026-06-20T23:48:18.764Z" }, + { url = "https://files.pythonhosted.org/packages/20/1f/8bb62d8933df7acf3247194e6e9fc68edf9d2fa203252c89c94b319dd472/wrapt-2.2.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94b00b00f806eb3ef2abe9049ed45994a81ee9284884d96e6b8314927c6cea3d", size = 167411, upload-time = "2026-06-20T23:48:20.315Z" }, + { url = "https://files.pythonhosted.org/packages/17/09/8789dcb09ee1de715727db7521aabbb68ffa68dfade3a49468440cfced49/wrapt-2.2.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:62415fd095bc590b842b6d092f2b5d9ccbaeb7e0b28535c03dcea2718b48636b", size = 158607, upload-time = "2026-06-20T23:48:21.728Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/66e02562d53ee67d841f175e38e3c993c2d78a3e104c576cad61c028b43c/wrapt-2.2.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a41e758d80dc0ab8c210f641ac892009d356cf1f955d97db544c8dd317b4d14c", size = 166367, upload-time = "2026-06-20T23:48:23.177Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a3/832ac4e41222fb263b3042d42c2f08d305db7d0f0c9b1d3a271a9eede8f6/wrapt-2.2.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b84cd4058001c9727b0e9980b7a9e66325b5ca748b1b578e822cade1bc6b304f", size = 157176, upload-time = "2026-06-20T23:48:24.711Z" }, + { url = "https://files.pythonhosted.org/packages/b7/01/1bd5e4d2df9c0178989ac8da9186543465388588ee2ef153e2591accebef/wrapt-2.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26fc73a1b15e0946d2942b9a4426d162b51676338327dc067ccd8d2d76385f94", size = 167025, upload-time = "2026-06-20T23:48:26.118Z" }, + { url = "https://files.pythonhosted.org/packages/18/1e/f060df47755e87b57684cee7bfc1362b204df55fac96ffebc0631b697b79/wrapt-2.2.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0065a3b657cec06813b4241d2462ccec287f6863103d7445b725fb3a889736f9", size = 203399, upload-time = "2026-06-20T23:48:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/c4/de/2316a757a1abb6453700b79d83e532146dcef2611348282d4d8889792161/wrapt-2.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30f7424af5c5c345b7f26490e097f74a2ef45b3d08b664dc33571aee3bd3b56c", size = 210461, upload-time = "2026-06-20T23:48:36.569Z" }, + { url = "https://files.pythonhosted.org/packages/ed/29/d1160785ae18ca2495a6d82a21154103d74f656c9fd457fb35f6b11b965a/wrapt-2.2.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:07fdcb012821859168641acf68afad61ef9783cf37100af85f152550e9677194", size = 195313, upload-time = "2026-06-20T23:48:38.175Z" }, + { url = "https://files.pythonhosted.org/packages/f5/2d/7caa9598ae61a9cf0989cc501739cbeeb7d650ab3193cca1407b9af0c6ab/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f90038ab58fafb584801ca62d72384d7d5225d93c76f7b773c22fae545bd8066", size = 206116, upload-time = "2026-06-20T23:48:39.804Z" }, + { url = "https://files.pythonhosted.org/packages/ac/02/281ea1088b8650d865f311b35cf86fd21df89128e2909714f1161e01c9d0/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:c5d7825491bfa2d08b97e9557768987952c7b9ae687d06c3320b40a37ccb7f20", size = 192668, upload-time = "2026-06-20T23:48:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/be/7d/976e2d5b4b5c5babda40974edd54d0a5585cb60132ed86b46f4b80239b16/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ad520e6daa9bbf136f14de735474dbec7dcc0891f718e1d274ce8dc92e645af", size = 198891, upload-time = "2026-06-20T23:48:43.056Z" }, + { url = "https://files.pythonhosted.org/packages/74/eb/df7b7f0b631dbbc750f39be27d8b55f65777d8ac86da80e12be41a644c4b/wrapt-2.2.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0abfd648dac9ac9c5b3aa9b523d27f1789046640b58dcd5652a720ddb325e1fc", size = 167713, upload-time = "2026-06-20T23:48:52.598Z" }, + { url = "https://files.pythonhosted.org/packages/4d/9a/d1bd36f6d088c8e652a9383cabbd49af30b8c576302a7eccddbab6963e3f/wrapt-2.2.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f4bfd8d1eb438153eff8b8cfe87f032ba65731e1ce06138b5090f745a33f6f95", size = 166779, upload-time = "2026-06-20T23:48:54.33Z" }, + { url = "https://files.pythonhosted.org/packages/4c/ae/24ffacd4187fac2740a1972093929e836dea092d42c87d728cd98fee11a6/wrapt-2.2.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c427c9d06d859848a69f0d928fe28b5c33a941b2265d10a0e1f15cd244f1ee33", size = 158407, upload-time = "2026-06-20T23:48:55.944Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ed/974427668249a356051e8d67d47fa54ef6c777f0fcf3bae9d292c047d4b6/wrapt-2.2.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4250b43d1a129d947e083c4dc6baf333c9bb34edd26f912d5b0457841fc858ab", size = 166594, upload-time = "2026-06-20T23:48:57.617Z" }, + { url = "https://files.pythonhosted.org/packages/fb/5f/e1d7c6e4523f78db2fbd7826babd0348da1d5e0834c4f918b9ab5757dfae/wrapt-2.2.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:173e5bb5ca350a6e0abab60b7ec7cdd7992a814cb14b4de670a28f067f105663", size = 157068, upload-time = "2026-06-20T23:48:59.171Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c1/7ebd1027f00700c0b0233b20aceef2b4784294ed64971424c4a78e069e34/wrapt-2.2.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aa14b01804bce36c6d63d7b6a4f55df390f29f8648cc13a1f40b166f4d54680d", size = 166470, upload-time = "2026-06-20T23:49:00.737Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ea/1a89e6d3b7a83c3affe5c09cde77792c947e63e4bc85ad84cd5bb9abb0d8/wrapt-2.2.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:45c2f2768e790c9f8db90f239ef23a2af8e7570f25a35619ef902df4a738447f", size = 203362, upload-time = "2026-06-20T23:49:09.811Z" }, + { url = "https://files.pythonhosted.org/packages/19/d8/3b58763d9863b5a73771c0d97110f9595d248db454009e07e1535ee905a4/wrapt-2.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbf00ee0cb55ec24e2b0995a71942b85b21a066db8f3f46e1dbfdb9433ffba81", size = 210449, upload-time = "2026-06-20T23:49:11.521Z" }, + { url = "https://files.pythonhosted.org/packages/2d/6f/17fd9e053103d8be148d20d5d7505facc72d5fe1f9127973904ceaed79cf/wrapt-2.2.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2252f77663651b89255895f58cc6ac08fcb206d4371813e5af61bb62d4f7689c", size = 195349, upload-time = "2026-06-20T23:49:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/ef/04/d0d1ccaaa12cb7dccf28a23f0279a608ba498f71e81d949d5ed54bcfd5c1/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cd7181ab1c31192ff5219269830744b5a62020b3a6d433588c4f1c95b8f8bff", size = 206099, upload-time = "2026-06-20T23:49:15.051Z" }, + { url = "https://files.pythonhosted.org/packages/44/b3/e8aa07b619890a2aa6cde1931b1887abb08820721b564a5f80b7ca3f3aa0/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:6fe35fd51b74867d8b80174c277bd6bbf6a73e443f908129dc531c4b688a20d5", size = 192728, upload-time = "2026-06-20T23:49:16.854Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f0/1819fb50f0d3c9bd758d8a83b56f1b470dee8b5b8eac8702b7c137cea9d4/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:11d95fc2fbad3163596c39d440e6f21ca9fccece74b56e30a37ac2fca786a07c", size = 198842, upload-time = "2026-06-20T23:49:18.504Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, +] + +[[package]] +name = "yte" +version = "1.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argparse-dataclass", marker = "sys_platform == 'linux'" }, + { name = "dpath", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/f5/7e44620e6e077bfe624b9a17c329b8e0d0159e176e1f1a93c2790428ab2c/yte-1.9.4.tar.gz", hash = "sha256:86a47e6d722cec9419a7ac88be57d0d6c4ce28f02860393b71a66f2c674069f6", size = 8101, upload-time = "2025-11-27T12:55:00.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/63/6a44729fdc60eb255a7b156a84e7552290174a9bf151e3b6c18e83d6fbfa/yte-1.9.4-py3-none-any.whl", hash = "sha256:5dac63303d3e6bc2ebadc36ece3c3fb09343772fe6e25e9356d9baf8f9dfaf6d", size = 10618, upload-time = "2025-11-27T12:55:01.685Z" }, +] + +[[package]] +name = "zarr" +version = "3.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "donfig", marker = "sys_platform == 'linux'" }, + { name = "google-crc32c", marker = "sys_platform == 'linux'" }, + { name = "numcodecs", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/8d/aeb164004f87543b06ef54f885d02c342c31ceb274e2bbec470a98927621/zarr-3.2.1.tar.gz", hash = "sha256:71565b738a0e7e8ed226f0516eba8c6bb53440ad7669a8c48ebb3534a161d035", size = 675161, upload-time = "2026-05-05T12:37:22.383Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/0a/469e2bd01be1490336e6c8707386845655d59261543315778a3ccc7e8019/zarr-3.2.1-py3-none-any.whl", hash = "sha256:f78cdd3d9687ad0e9f9cba2c5683b64f0c52589c19f685eeabe872e93cc0d2c7", size = 319617, upload-time = "2026-05-05T12:37:20.66Z" }, +] + +[[package]] +name = "zipp" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, +] + +[[package]] +name = "zope-interface" +version = "8.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/dc/50550cfcbb2ea3cbca5f1d7ed05c8aa840f831a0f2d63aec0a953f7c590e/zope_interface-8.5.tar.gz", hash = "sha256:7a3ba1c5877f0f3e3906b02ddf793abed2becc2948116414ce0e1dd820b68d6d", size = 257957, upload-time = "2026-05-26T06:50:14.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/c3/005032195ff3b210c139b7c560ed5c534e844b0907d8e44d2b3d8919305e/zope_interface-8.5-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:e6347b8d8d12c5eca6502450a92be30079b7acfade2c4f693efa0deb8871b06e", size = 265296, upload-time = "2026-05-26T06:49:29.741Z" }, + { url = "https://files.pythonhosted.org/packages/c5/66/1036543d6a66bc04c19df3cf650f3ad938a002ab0a443c24e23e8de5e8b9/zope_interface-8.5-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5e970dabea777a24b0b0bbf9dae3ab75ce8b2d8e948edf4875627034b21f3560", size = 270689, upload-time = "2026-05-26T06:49:31.767Z" }, + { url = "https://files.pythonhosted.org/packages/30/4c/8b56259558cace4414e753ca6740396a1f59d4a95ddb55b4658600408670/zope_interface-8.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f0b48ccadaa9839e09ff81e969703cecb3f402c813bfe8b958652e699bea69f5", size = 270280, upload-time = "2026-05-26T06:49:33.489Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a5/692a2b8d70f78e848793231d5fae5fecbf8d0cccd73430fdc34802a6d3c1/zope_interface-8.5-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:7849ad8fa90763cc1087f4dda78ca3a233e950b3e08fac7079297c9cafbbd7bb", size = 265191, upload-time = "2026-05-26T06:49:43.449Z" }, + { url = "https://files.pythonhosted.org/packages/70/8d/454a9cfc7a050c394ab4f11b3371f7897828b7415e096afff724637e65e0/zope_interface-8.5-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5578c9421ca409a1f39f153d6f7803e4cde01da592ec75a9ac5e1b777d18d33b", size = 270626, upload-time = "2026-05-26T06:49:45.425Z" }, + { url = "https://files.pythonhosted.org/packages/51/8c/db8409cfa3575b8e9b4800babd7d49f8228433cd1f0c56814bd0ada49c33/zope_interface-8.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e1bd7d96b4ca5fa311f54c9eac16dce4886b428c1531dbe06067763ccdf123b4", size = 270444, upload-time = "2026-05-26T06:49:47.025Z" }, + { url = "https://files.pythonhosted.org/packages/32/67/bc8b4f465d388039255003e230c284a175cedf1203c692f23cb7bff64efe/zope_interface-8.5-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:3090e3a663d20194756a59a272e0c8508b889341e31d5894223331fe6b4f9b21", size = 266827, upload-time = "2026-05-26T06:49:54.873Z" }, + { url = "https://files.pythonhosted.org/packages/a7/eb/37d05b935ede53d79690fecc8d201440084418e590bcfc05f384451c7593/zope_interface-8.5-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9342fb74e2afefdb081bf1df727d209ea56995c6e13f5a0540e6d7aff4beafb8", size = 270139, upload-time = "2026-05-26T06:49:57.116Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0b/fd0c54579e2ce8dc6cf1a757903f3374bc6fbda929a46af9e0f53cb0e5f0/zope_interface-8.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6c54725d818f1b57a7efb8b16528326e1f3c257b602b32393fd255c45af8799d", size = 270338, upload-time = "2026-05-26T06:49:58.698Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/d97430abd5ae9677e8b9295b58720c0064a5b557dbb6b8bf5928484cf0d8/zope_interface-8.5-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:14b0e9799351d4c34fe99afd67f0cdd76e55ba15c66a98699d5fc22ea8241e08", size = 294905, upload-time = "2026-05-26T06:50:07.384Z" }, + { url = "https://files.pythonhosted.org/packages/41/ec/a0f8f3dad6e74992f4654bdd94802be0929eabca7b871cac3b6fbb5e961b/zope_interface-8.5-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cd6a732ac84b94eb1ef9222a117347a27efd294ee16810ffdf7ecd307677ed5", size = 300885, upload-time = "2026-05-26T06:50:08.997Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/6881b48803a0ee8d23eb5efa30fce3ed218a2bd9de5758ce489d224fee81/zope_interface-8.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:798b7c87d0e59a7d5d086d642208d0d8700ff0d55c4029134b3c479c3bfb110f", size = 304672, upload-time = "2026-05-26T06:50:10.563Z" }, +] From 8fc6fcb70e6797e27b1d688f27ca1a1b68c14346 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 15:12:34 +0200 Subject: [PATCH 023/160] chore(ci): restore develop's lint.yml (PR #264 server-side ruff autofix) lost in stale merge --- .github/workflows/lint.yml | 209 ++++++++++++++++++++++++++++++------- 1 file changed, 174 insertions(+), 35 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c2be73bd..a0f36765 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,21 +1,34 @@ name: Lint -# Lint gate for `develop` — warn locally, account here. +# Lint gate for `develop` — warn locally, fix-and-account here. # -# The model (settled with Cail + Sacha, 2026-06-23/30): +# The model (settled with Cail + Sacha, 2026-06-23/30; autofix added 2026-07-10): # * Locally, ruff auto-applies safe fixes and only WARNS on the rest (see # .pre-commit-config.yaml) — you commit freely. # * Getting into `develop` is gated. This job runs the FULL ruff policy -# (`ruff check .` + `ruff format --check .`, region-aware per pyproject.toml) -# and on any failure it (1) fails the job RED so the check blocks the merge, -# and (2) tells the author what to fix — in the surface that fits the event: -# - On a PR → posts/updates a COMMENT on the PR itself, with the full -# violation list (and ruff annotations in the run's Checks view). The -# author sees it where they already are; no disconnected issue. The -# comment turns green when ruff passes. -# - On a direct push to develop → there's no PR to comment on, so it opens -# (or updates) ONE lint-debt issue for the committer, @-mentioning and -# assigning them. Auto-closes when their next push is clean. +# (`ruff check .` + `ruff format --check .`, region-aware per pyproject.toml). +# What happens next depends on the event: +# +# - On a same-repo PR (head branch lives in THIS repo, not a fork) → the gate +# doesn't just report; it FIXES. If the first pass isn't clean it runs +# `ruff check --fix-only` + `ruff format`, commits the diff as the +# github-actions bot, and pushes it back to the PR branch. Then it re-runs +# ruff on the fixed tree IN THE SAME RUN and gates on THAT: if formatting + +# safe fixes cleaned everything → job GREEN, comment says autofix was +# pushed; if anything survives (unsafe/judgement lint — undefined names, +# unused vars) → job RED, comment lists ONLY the residual (the mechanical +# stuff is already fixed). Contributors mostly never touch ruff by hand. +# +# - On a fork PR (read-only token, can't push to the fork's branch) → the old +# behaviour: run the checks, and on failure (1) fail RED so the check blocks +# the merge, (2) post/update a COMMENT on the PR with the full violation +# list. The comment turns green when ruff passes. +# +# - On a direct push to develop → there's no PR to comment on, so on failure +# it opens (or updates) ONE lint-debt issue for the committer, @-mentioning +# and assigning them. Auto-closes when their next push is clean. (No autofix +# here — pushing a bot commit onto develop would be its own event.) +# # * If ruff itself can't run (network, bad version), the job goes red but says # nothing — that's infra, not the committer's lint debt. # @@ -23,17 +36,25 @@ name: Lint # branches stay quiet (too noisy otherwise). `workflow_dispatch` is for manual # testing of the gate itself. # -# Why `pull_request_target` for PRs: commenting on a PR needs a write token, and -# PRs from forks (e.g. sachaguer/) get a read-only token under the plain -# `pull_request` event. `pull_request_target` runs this workflow from the BASE -# branch (so the workflow definition is trusted) with a write token, while we -# check out the PR head ONLY to lint it. Two hardening measures make running -# tooling over untrusted PR code safe here: ruff is a static analyzer (it parses -# files, never imports/executes them), and `uvx --no-config` makes uv ignore any -# `uv.toml`/`[tool.uv]` in the PR tree, so a malicious PR can't redirect ruff's -# download to a trojaned index. The checkout also drops its git credentials. -# (A PR can still edit `[tool.ruff]` to weaken its own policy, but that's visible -# in the diff and reviewed like any other change.) +# Why `pull_request_target` for PRs: commenting on (and pushing to) a PR needs a +# write token, and PRs from forks (e.g. sachaguer/) get a read-only token under +# the plain `pull_request` event. `pull_request_target` runs this workflow from +# the BASE branch (so the workflow definition is trusted) with a write token, +# while we check out the PR head ONLY to lint it. Two hardening measures make +# running tooling over untrusted PR code safe here: ruff is a static analyzer (it +# parses files, never imports/executes them), and `uvx --no-config` makes uv +# ignore any `uv.toml`/`[tool.uv]` in the PR tree, so a malicious PR can't +# redirect ruff's download to a trojaned index. The checkout also drops its git +# credentials. (A PR can still edit `[tool.ruff]` to weaken its own policy, but +# that's visible in the diff and reviewed like any other change.) +# +# Why the autofix push is safe under pull_request_target: we run ONLY ruff over +# the untrusted tree (static, never executes PR code), and we push back ONLY the +# diff ruff itself produced — no PR-authored script runs with our write token. +# The push uses the workflow token explicitly (the checkout keeps +# persist-credentials: false), and a GITHUB_TOKEN push does NOT trigger a new +# workflow run — so no recursion, but also no fresh CI on the bot commit, which +# is exactly why we re-lint and gate in THIS run rather than waiting for a rerun. on: push: @@ -48,7 +69,7 @@ concurrency: cancel-in-progress: true permissions: - contents: read + contents: write # push ruff autofix commit back to a same-repo PR branch issues: write # develop-push lint-debt issue pull-requests: write # PR lint comment @@ -67,6 +88,19 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v3 + # Is this a PR whose head branch lives in THIS repo (not a fork)? Only then + # can we push an autofix commit back to it with the workflow token. + - name: Decide whether autofix can push + id: mode + shell: bash + run: | + if [ "${{ github.event_name }}" = "pull_request_target" ] && \ + [ "${{ github.event.pull_request.head.repo.full_name }}" = "${{ github.repository }}" ]; then + echo "autofix=true" >> "$GITHUB_OUTPUT" + else + echo "autofix=false" >> "$GITHUB_OUTPUT" + fi + # Run the checks WITHOUT failing the step — we post feedback before turning # the job red. `ruff@` matches the pre-commit version; `--no-config` # neutralizes any uv config in the (untrusted) PR tree. @@ -102,20 +136,109 @@ jobs: echo "passed=false" >> "$GITHUB_OUTPUT" fi + # ── Autofix (same-repo PRs only) ────────────────────────────────────────── + # The first pass found something on a branch we can push to: apply ruff's + # own fixes (safe lint fixes + formatting), commit the diff as the bot, and + # push it back. SAFE under pull_request_target: only ruff runs over the PR + # tree (static, never executes it), and only ruff's own diff is pushed — no + # PR-authored code touches our write token. Then re-lint the FIXED tree in + # this same run: a GITHUB_TOKEN push doesn't trigger a new workflow, so the + # residual pass/fail we compute here is what the gate reports. + - name: Ruff autofix + push (same-repo PR) + id: autofix + if: >- + steps.mode.outputs.autofix == 'true' && + steps.ruff.outputs.tool_error == 'false' && + steps.ruff.outputs.passed == 'false' + shell: bash + run: | + # `check --fix-only` exits 1 when unfixable violations remain even + # after applying every safe fix, so don't let that abort the step. + uvx --no-config ruff@0.15.18 check --fix-only . || true + uvx --no-config ruff@0.15.18 format . || true + + set -e + if git diff --quiet; then + # ruff couldn't fix anything (all issues are unsafe/judgement calls): + # nothing to push. The gate falls back to the first-pass result and + # the original report already lists these — no autofix comment. + echo "pushed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add -A + git commit -m "ruff autofix (format + safe lint fixes)" \ + -m "Pushed by the lint gate." + + # Push back to the PR HEAD branch. The checkout kept + # persist-credentials: false, so authenticate the push explicitly with + # the workflow token via the remote URL. + BRANCH='${{ github.event.pull_request.head.ref }}' + REPO='${{ github.event.pull_request.head.repo.full_name }}' + git push "https://x-access-token:${{ github.token }}@github.com/${REPO}.git" "HEAD:${BRANCH}" + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "pushed=true" >> "$GITHUB_OUTPUT" + + # Re-lint the FIXED tree — this is what the gate reports (no rerun comes). + set +e + uvx --no-config ruff@0.15.18 check . --output-format=concise > check2.txt 2>&1; check_rc=$? + uvx --no-config ruff@0.15.18 format --check . > format2.txt 2>&1; fmt_rc=$? + set -e + { + echo "### Residual \`ruff check .\` (after autofix)" + if [ "$check_rc" -eq 0 ]; then echo; echo "✅ clean"; else echo; echo '```'; cat check2.txt; echo '```'; fi + echo + echo "### Residual \`ruff format --check .\` (after autofix)" + if [ "$fmt_rc" -eq 0 ]; then echo; echo "✅ clean"; else echo; echo '```'; cat format2.txt; echo '```'; fi + } > residual.md + + if [ "$check_rc" -eq 0 ] && [ "$fmt_rc" -eq 0 ]; then + echo "residual_passed=true" >> "$GITHUB_OUTPUT" + else + echo "residual_passed=false" >> "$GITHUB_OUTPUT" + fi + + # Resolve the outcome the gate reports. For a same-repo PR that got an + # autofix push, the residual pass/fail on the FIXED tree supersedes the + # first pass (that's what's now on the branch); otherwise the first pass + # stands. `pushed`/`residual` are surfaced so the comment can say so. + - name: Resolve gate outcome + id: gate + if: steps.ruff.outputs.tool_error == 'false' + shell: bash + run: | + if [ "${{ steps.autofix.outputs.pushed }}" = "true" ]; then + echo "passed=${{ steps.autofix.outputs.residual_passed }}" >> "$GITHUB_OUTPUT" + echo "autofixed=true" >> "$GITHUB_OUTPUT" + echo "sha=${{ steps.autofix.outputs.sha }}" >> "$GITHUB_OUTPUT" + echo "report_file=residual.md" >> "$GITHUB_OUTPUT" + else + echo "passed=${{ steps.ruff.outputs.passed }}" >> "$GITHUB_OUTPUT" + echo "autofixed=false" >> "$GITHUB_OUTPUT" + echo "report_file=report.md" >> "$GITHUB_OUTPUT" + fi + # Feedback is a side effect — never let it red a clean run. - name: Tell the author (PR comment) or record it (develop-push issue) if: steps.ruff.outputs.tool_error == 'false' continue-on-error: true uses: actions/github-script@v7 env: - PASSED: ${{ steps.ruff.outputs.passed }} + PASSED: ${{ steps.gate.outputs.passed }} + AUTOFIXED: ${{ steps.gate.outputs.autofixed }} + AUTOFIX_SHA: ${{ steps.gate.outputs.sha }} + REPORT_FILE: ${{ steps.gate.outputs.report_file }} with: script: | const fs = require('fs'); const passed = process.env.PASSED === 'true'; + const autofixed = process.env.AUTOFIXED === 'true'; + const autofixSha = (process.env.AUTOFIX_SHA || '').slice(0, 7); const { owner, repo } = context.repo; const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; - const report = passed ? '' : fs.readFileSync('report.md', 'utf8'); + const report = passed ? '' : fs.readFileSync(process.env.REPORT_FILE, 'utf8'); // ---- PR: speak on the PR itself (comment, auto-updating) ---------- if (context.eventName === 'pull_request_target') { @@ -128,22 +251,35 @@ jobs: const mine = comments.find(c => c.body && c.body.includes(MARKER)); if (passed) { - // Only update an existing comment to green; don't post on a PR + // Clean now. If we got here by pushing an autofix, say so (the + // push is why the branch changed under the author). Otherwise + // only update an existing comment to green — don't post on a PR // that was never dirty. + const body = autofixed + ? `🤖 **autofix pushed \`${autofixSha}\`, ruff is clean** — formatting and safe lint fixes were applied for you; nothing else to do. ${MARKER}` + : `✅ **ruff is clean** — nothing to fix here. ${MARKER}`; if (mine) { - await github.rest.issues.updateComment({ - owner, repo, comment_id: mine.id, - body: `✅ **ruff is clean** — nothing to fix here. ${MARKER}`, - }); + await github.rest.issues.updateComment({ owner, repo, comment_id: mine.id, body }); + } else if (autofixed) { + await github.rest.issues.createComment({ owner, repo, issue_number: pr.number, body }); } core.info('PR clean.'); return; } + const intro = autofixed + ? [ + `### 🔴 ruff — residual issues after autofix`, + ``, + `@${author} — I pushed \`${autofixSha}\` with the formatting and safe lint fixes, but these need a human and still block the merge into \`develop\`:`, + ] + : [ + `### 🔴 ruff found lint / format issues`, + ``, + `@${author} — these block the merge into \`develop\`. Full list below (also surfaced as annotations in the CI run):`, + ]; const body = [ - `### 🔴 ruff found lint / format issues`, - ``, - `@${author} — these block the merge into \`develop\`. Full list below (also surfaced as annotations in the CI run):`, + ...intro, ``, report, ``, @@ -233,8 +369,11 @@ jobs: # Red → blocks the merge. `always()` so a hiccup in the feedback step above # can't suppress the red on genuine lint debt; a tooling error also reds. + # A tooling error means `gate` was skipped (its outputs are empty), so test + # it first; otherwise the resolved gate outcome (post-autofix on same-repo + # PRs, first-pass elsewhere) decides. - name: Fail the job if the gate didn't pass - if: always() && steps.ruff.outputs.passed == 'false' + if: always() && (steps.ruff.outputs.tool_error == 'true' || steps.gate.outputs.passed == 'false') run: | if [ "${{ steps.ruff.outputs.tool_error }}" = "true" ]; then echo "::error::ruff could not run (network / version) — gate inconclusive, blocking." From 9ee6eb9e3bb80f3e2ea16079e1e3251a5bb8c07a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 15:41:11 +0200 Subject: [PATCH 024/160] feat(blinding): wire blind-at-birth custody into the SACC migration DAG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Author the Snakemake rules that realise sp_validation.blinding's three-verb custody surface on the migrated cosmo_val workflow (issues #247/#252). - rule blind_init (once per catalogue version): draws the seed, publishes commitment.json + the encrypted seed bundle. Refuses to overwrite state. - rule blind_part (generic over the three blindable stems — reporting ξ±, integration ξ±, analysis pseudo-Cℓ): conceals the part at birth, escrows the true vector beside the blinded output. The plaintext part is a temp() output of its producer on a data run, so only its blinded sibling persists. - common.py: run-type-aware path helpers (blindable_part / maybe_temp / blind_state_paths / version_of) so a data run binds ξ-derived consumers to the blinded siblings and a mock run bypasses blinding entirely — the whole blinding subgraph appears or vanishes with RUN_TYPE. - assemble_sacc: assert_consistent_blind across parts (assembly-time commitment check of #252), stamping the shared blind on the terminal file. - Constrain the npatch wildcard to \d+ so a producer's ξ± output cannot absorb the _blinded suffix (which made rule xi ambiguous with blind_part). The cosmo_val assemble DAG dry-runs: blind_init x2, blind_part x5, assemble consuming the blinded ξ± + pseudo-Cℓ parts. Born-blinded COSEBIs/pure-E/B and the ρ/τ concealed pass-through follow in the next commit. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD --- workflow/Snakefile | 4 ++ workflow/common.py | 97 ++++++++++++++++++++++++++++++- workflow/rules/blinding.smk | 74 +++++++++++++++++++++++ workflow/rules/cosmo_val.smk | 13 ++++- workflow/rules/twopoint.smk | 14 ++++- workflow/scripts/assemble_sacc.py | 13 +++++ workflow/scripts/blind_init.py | 12 ++++ workflow/scripts/blind_part.py | 19 ++++++ 8 files changed, 240 insertions(+), 6 deletions(-) create mode 100644 workflow/rules/blinding.smk create mode 100644 workflow/scripts/blind_init.py create mode 100644 workflow/scripts/blind_part.py diff --git a/workflow/Snakefile b/workflow/Snakefile index 6f59827b..0a1f3258 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -37,6 +37,10 @@ wildcard_constraints: # Compute rules (infrastructure — raw outputs, no evidence.json) include: "rules/twopoint.smk" +# Smokescreen blind-at-birth custody (blind_init / blind_part). Generic over the +# blindable parts twopoint.smk produces; dormant unless a data run requests a +# blinded part. Included before covariance/cosmo_val so their consumers resolve. +include: "rules/blinding.smk" include: "rules/covariance.smk" include: "rules/inference.smk" include: "rules/masks.smk" diff --git a/workflow/common.py b/workflow/common.py index 1a168041..530b9212 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -5,6 +5,8 @@ import re from pathlib import Path +from snakemake.io import temp + # Absolute path to the generic workflow's scripts, anchored on this module's own # location (common.py lives in workflow/, is `from common import *`'d into every # Snakefile, and so resolves to the generic workflow dir of the running checkout @@ -51,6 +53,10 @@ # glass-mock A/B/C variant, not Smokescreen blinding — see BLINDS above. "blind": r"[ABC]", "nbins": r"\d+", + # Constrained so a producer's ξ± output pattern cannot greedily absorb the + # "_blinded" suffix into npatch (which would make it ambiguous with the + # blind_part rule's {stem}_blinded output). npatch is always an integer. + "npatch": r"\d+", "min_sep": r"[0-9.]+", "max_sep": r"[0-9.]+", "gaussian": r"(g|ng)", @@ -65,16 +71,22 @@ DEFAULT_MASK_SUFFIX = "" CATALOG_CONFIG = None PLANCK18 = None +# Run type gates Smokescreen blind-at-birth (see the blind custody section +# below): "data" blinds the three blindable parts and binds ξ-derived consumers +# to the blinded siblings; "mock" bypasses blinding entirely. Set from +# config["cosmo_val"]["type"] in configure(); "data" is the production default. +RUN_TYPE = "data" def configure(workflow_config): """Install config-derived values after Snakemake has loaded configfiles.""" - global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, FIDUCIAL, PLANCK18 + global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, FIDUCIAL, PLANCK18, RUN_TYPE CATALOG_CONFIG = workflow_config FIDUCIAL = workflow_config["fiducial"] DEFAULT_MASK_SUFFIX = ( "_masked" if workflow_config["covariance"].get("default_masked", False) else "" ) + RUN_TYPE = workflow_config.get("cosmo_val", {}).get("type", "data") with open(COSMOLOGY_PARAMS) as f: PLANCK18 = json.load(f) @@ -196,6 +208,89 @@ def get_shear_catalog(wildcards): return str(Path(subdir) / shear_path) +# --------------------------------------------------------------------------- +# Smokescreen blind-at-birth custody (issues #247/#252, PR #253) +# --------------------------------------------------------------------------- +# Distinct from the glass-mock A/B/C `blind` wildcard above: this is Smokescreen +# concealment (the concealed=True SACC stamp). Blinding is per part, at birth. +# The three blindable parts (reporting ξ±, integration ξ±, pseudo-Cℓ) are each +# concealed the moment they are computed, so only blinded parts persist on disk. +# A `data` run binds every ξ-derived consumer (the terminal assemble, and the +# born-blinded COSEBIs / pure-E/B) to the *_blinded parts, which pulls the +# blind_part → blind_init subgraph into the DAG. A `mock` run bypasses blinding +# entirely and binds to the plaintext parts, so the subgraph never appears. +# RUN_TYPE is the single switch that flips which files exist. +# +# The path helpers below MIRROR sp_validation.blinding.init_paths / part_paths +# by hand rather than importing blinding (which pulls in numpy + smokescreen) at +# DAG-build time. test_blinding_wiring asserts the two stay in lockstep. + + +def is_data_run(): + """True when blinding is active (production data runs); False for mocks.""" + return RUN_TYPE == "data" + + +def blind_state_dir(version): + """Per-version blind-init custody directory (commitment + encrypted seed).""" + return str(COSMO_VAL / "blind" / version) + + +def blind_state_paths(version): + """The fixed custody-state files blind_init writes for a version. + + Mirrors sp_validation.blinding.init_paths(blind_state_dir(version)). + """ + d = blind_state_dir(version) + return { + "commitment": os.path.join(d, "commitment.json"), + "bundle": os.path.join(d, "blind_seed.encrpt"), + "key": os.path.join(d, "blind_seed.key"), + } + + +def blinded_path(part_path): + """The *_blinded sibling blind_part writes beside a plaintext part. + + Mirrors sp_validation.blinding.part_paths(part_path)["blinded"]. + """ + stem, ext = os.path.splitext(str(part_path)) + return f"{stem}_blinded{ext or '.fits'}" + + +def version_of(stem): + """Extract the catalogue version embedded in a blindable part's stem. + + Every blindable stem carries the version (as {version}_xi_… or + pseudo_cl_{version}_…); blind_part needs it to locate the version's blind + state. Matches the shared `version` wildcard pattern. + """ + m = re.search(WILDCARD_CONSTRAINTS["version"], stem) + if m is None: + raise ValueError(f"no catalogue version found in part stem {stem!r}") + return m.group(0) + + +def blindable_part(part_path): + """On-disk path a run persists for one blindable part. + + Data run -> the blinded sibling (binding it pulls blind_part + blind_init + into the DAG); mock run -> the plaintext part (blinding bypassed). + """ + return blinded_path(part_path) if is_data_run() else str(part_path) + + +def maybe_temp(part_path): + """Wrap a producer's blindable plaintext part temp() on data runs. + + On a data run the plaintext part's only consumer is blind_part, which + escrows the true vector before Snakemake removes the temp file — so no + plaintext blindable part persists. On a mock run the part is the real + product downstream binds to, so it is left persistent. + """ + return temp(str(part_path)) if is_data_run() else str(part_path) + + # --------------------------------------------------------------------------- # CosmologyValidation diagnostic suite (cosmo_val.py) # --------------------------------------------------------------------------- diff --git a/workflow/rules/blinding.smk b/workflow/rules/blinding.smk new file mode 100644 index 00000000..bf79433f --- /dev/null +++ b/workflow/rules/blinding.smk @@ -0,0 +1,74 @@ +# Smokescreen blind-at-birth custody rules (issues #247/#252, PR #253). +# +# Two rules realise the three-verb custody surface of sp_validation.blinding on +# the DAG. They only enter the graph on a `data` run, and only when a consumer +# binds to a *_blinded part through common.blindable_part — a `mock` run never +# requests a blinded file, so blind_part and blind_init stay dormant. +# +# blind_init (once per catalogue version) draws the seed, publishes +# commitment.json + the encrypted seed bundle. +# blind_part (once per blindable part, at birth) conceals the part, escrows +# the true vector beside the blinded output, and lets Snakemake +# remove the plaintext (temp()) once it is the sole consumer. +# +# The terminal assemble_sacc rule (cosmo_val.smk) asserts the shared commitment +# across parts — the assembly-time custody check of #252. + +# The three blindable stems: reporting ξ± (rule xi), integration ξ± (xi_highres), +# and the analysis pseudo-Cℓ (pseudo_cl). None contains "_blinded", so the +# generic blind_part rule can never blind its own output twice. The version +# pattern is the shared one from common.WILDCARD_CONSTRAINTS. +_V = WILDCARD_CONSTRAINTS["version"] +BLINDABLE_STEM = ( + rf"(?:{_V}_xi_reporting_minsep=[0-9.]+_maxsep=[0-9.]+_nbins=\d+_npatch=\d+" + rf"|{_V}_xi_integration" + rf"|pseudo_cl_{_V}_blind=[ABC]_[a-z]+_nbins=\d+)" +) + + +rule blind_init: + """Fix the blind for one catalogue version (blind-init). + + Draws an OS-entropy seed, writes the repo-committable commitment.json + (sha256(seed) + config digest) and the Fernet-encrypted seed bundle. Runs + once per version and refuses to overwrite existing state — a blind is a + one-shot custody event. + """ + output: + commitment=str(COSMO_VAL / "blind" / "{version}" / "commitment.json"), + bundle=str(COSMO_VAL / "blind" / "{version}" / "blind_seed.encrpt"), + key=str(COSMO_VAL / "blind" / "{version}" / "blind_seed.key"), + params: + blind_dir=lambda w: blind_state_dir(w.version), + resources: + runtime=5, + script: + "../scripts/blind_init.py" + + +rule blind_part: + """Blind one intermediate part SACC at birth (blind-part). + + Conceals the plaintext part through its matching theory backend, escrows the + true vector into a per-part encrypted bundle beside the blinded output, and + leaves the plaintext for Snakemake to remove (it is a temp() output of the + producing rule, and this is its only consumer on a data run). Generic over + the three blindable stems. + """ + input: + part=str(COSMO_VAL / "{stem}.sacc"), + commitment=lambda w: blind_state_paths(version_of(w.stem))["commitment"], + bundle=lambda w: blind_state_paths(version_of(w.stem))["bundle"], + key=lambda w: blind_state_paths(version_of(w.stem))["key"], + output: + blinded=str(COSMO_VAL / "{stem}_blinded.sacc"), + escrow=str(COSMO_VAL / "{stem}_escrow.encrpt"), + escrow_key=str(COSMO_VAL / "{stem}_escrow.key"), + wildcard_constraints: + stem=BLINDABLE_STEM, + params: + blind_dir=lambda w: blind_state_dir(version_of(w.stem)), + resources: + runtime=10, + script: + "../scripts/blind_part.py" diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 73ec22c7..7a7e62c6 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -482,8 +482,15 @@ def cv_assemble_inputs(version): fiducial harmonic tag). pseudo_cl (+ its cov) is included only when the config toggles the harmonic-space BB into the analysis. """ + # blindable_part binds the raw-signal parts (reporting/integration ξ±, + # analysis pseudo-Cℓ) to their blinded siblings on a data run and to the + # plaintext on a mock run. COSEBIs and pure-E/B are born blinded (their + # writers derive them from the blinded integration ξ± and stamp + # concealed=True), so they bind by their own name in both cases; ρ/τ is a + # diagnostic carrying no cosmological vector but is stamped concealed + # pass-through so the fail-closed load gate admits it on a data run. parts = dict( - xi_reporting=cv_xi_reporting_sacc(version), + xi_reporting=blindable_part(cv_xi_reporting_sacc(version)), cosebis=cv_cosebis_sacc(version), pure_eb=cv_pure_eb_sacc(version), rho_tau=cv_rho_tau_sacc(version), @@ -493,9 +500,9 @@ def cv_assemble_inputs(version): # fiducial version's terminal file alone; other versions' {version}.sacc omit # the integration rows rather than trigger a job with no output to bind. if version == config["fiducial"]["version"]: - parts["xi_integration"] = cv_xi_integration_sacc(version) + parts["xi_integration"] = blindable_part(cv_xi_integration_sacc(version)) if CV.get("include_pseudo_cl", False): - parts["pseudo_cl"] = cv_pseudo_cl_analysis_sacc(version) + parts["pseudo_cl"] = blindable_part(cv_pseudo_cl_analysis_sacc(version)) parts["pseudo_cl_cov"] = cv_pseudo_cl_cov(version) return parts diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 9eb90c64..f4a644ee 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -14,7 +14,9 @@ rule xi: # rule to share one wildcard set, and it keeps the reporting .sacc name # self-describing so requesting it binds the xi job unambiguously. txt=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), - xi_reporting=str(COSMO_VAL / "{version}_xi_reporting_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), + # Blindable part: temp() on a data run so only its blinded sibling + # persists (blind_part escrows the true vector first). See common.maybe_temp. + xi_reporting=maybe_temp(str(COSMO_VAL / "{version}_xi_reporting_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc")), threads: 24 params: ver="{version}", @@ -42,7 +44,8 @@ rule xi_highres: container: None output: txt=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.txt"), - xi_integration=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_integration.sacc"), + # Blindable part: temp() on a data run (see rule xi / common.maybe_temp). + xi_integration=maybe_temp(str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_integration.sacc")), resources: tasks=30, cpus_per_task=12, @@ -116,6 +119,13 @@ rule pseudo_cl: concealed=True stamp is a separate axis on the SACC file. """ output: + # This generic rule produces every pseudo-Cℓ variant — the analysis part + # (blind=A, powspace, nbins=32) folded into {version}.sacc, plus the fine + # (COSEBIS) and glass-mock variants. Only the analysis part is a terminal + # blindable, and a data run blinds it via a requested _blinded sibling + # (blind_part reads this plaintext); the fine/mock variants are B-mode / + # validation intermediates left untouched here. The output is therefore + # not temp()'d — see the PR note on residual unblinded pseudo-Cℓ. pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.sacc"), wildcard_constraints: blind="[ABC]", # glass-mock variant, not Smokescreen blinding diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 4eec725d..84e5c97d 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -187,7 +187,20 @@ def assemble_sacc( ) if not parts: raise ValueError(f"no parts found for {version}: {part_paths}") + # Assembly-time custody assertion (#252): every blindable part (ξ± / pseudo-Cℓ + # EE) must share one blind commitment + config digest, or assembly fails + # closed — mixed blinded/plaintext parts and divergent-seed parts both raise. + # ρ/τ and covariance-only parts are exempt. Returns the shared blind stamp to + # carry onto the assembled file (or None for a fully-mock plaintext assembly). + from sp_validation import blinding + + shared = blinding.assert_consistent_blind(parts) s = assemble_analysis_sacc(nz, metadata, parts) + if shared is not None: + # Stamp the assembled file with the shared blind so it, too, reads as + # concealed (its parts already carried the stamp into `metadata` above; + # this makes the custody state explicit and authoritative on the union). + s.metadata.update(shared) # Assembly preserves its parts' provenance: every part was written by # sacc_io.save and therefore carries the type=data|mock stamp in its # metadata (copied into the assembled file above). diff --git a/workflow/scripts/blind_init.py b/workflow/scripts/blind_init.py new file mode 100644 index 00000000..f9e4b682 --- /dev/null +++ b/workflow/scripts/blind_init.py @@ -0,0 +1,12 @@ +"""Rule blind_init: fix the blind for one catalogue version. + +Thin wrapper over :func:`sp_validation.blinding.blind_init`. Draws the seed, +writes commitment.json + the encrypted seed bundle into the version's blind +directory. The plaintext seed is never written (the encryptor deletes it). +""" + +from snakemake.script import snakemake + +from sp_validation import blinding + +blinding.blind_init(snakemake.params["blind_dir"]) diff --git a/workflow/scripts/blind_part.py b/workflow/scripts/blind_part.py new file mode 100644 index 00000000..b93313fb --- /dev/null +++ b/workflow/scripts/blind_part.py @@ -0,0 +1,19 @@ +"""Rule blind_part: blind one intermediate part SACC at birth. + +Thin wrapper over :func:`sp_validation.blinding.blind_part`. Conceals the part, +escrows the true vector beside the blinded output, and leaves the plaintext in +place: it is a temp() output of the producing rule, so Snakemake removes it once +this (its only consumer on a data run) finishes. keep_input=True hands that +lifecycle to Snakemake rather than deleting inside the blind step, which keeps +the blinded output and its temp input in one consistent DAG accounting. +""" + +from snakemake.script import snakemake + +from sp_validation import blinding + +blinding.blind_part( + snakemake.input["part"], + snakemake.params["blind_dir"], + keep_input=True, +) From be38cd5df43f8d6ec9cacb733c740600fb74e879 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 16:03:27 +0200 Subject: [PATCH 025/160] test(blinding): wiring tests for the blind-at-birth DAG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - common.py path helpers mirror sp_validation.blinding init_paths/part_paths (drift guard), version_of on all three blindable stems, run-type switch. - Data-run fail-closed assembly: assemble_sacc refuses an unblinded type=data part, passes when every part is concealed under one commitment, and refuses a blinded/plaintext mix and divergent commitments. - Candide-only: the blinding subgraph (blind_init/blind_part) resolves in the cosmo_val assemble dry-run and binds the blinded ξ± + pseudo-Cℓ siblings. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD --- .../tests/test_blinding_wiring.py | 263 ++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 src/sp_validation/tests/test_blinding_wiring.py diff --git a/src/sp_validation/tests/test_blinding_wiring.py b/src/sp_validation/tests/test_blinding_wiring.py new file mode 100644 index 00000000..77beb7ee --- /dev/null +++ b/src/sp_validation/tests/test_blinding_wiring.py @@ -0,0 +1,263 @@ +"""Tests for the Snakemake blind-at-birth wiring (issues #247/#252, PR #253). + +Two seams are covered here, both independent of a live cluster: + +1. **The path helpers in ``workflow/common.py``** must stay in lockstep with + ``sp_validation.blinding`` — common mirrors ``init_paths`` / ``part_paths`` by + hand (to keep the DAG build from importing the heavy blinding module), so a + drift between them would silently mis-wire ``blind_part``. These tests are the + guard. +2. **The data-run fail-closed assembly**: ``assemble_sacc`` must refuse an + unblinded ``type='data'`` part and succeed once every part is concealed under + one commitment — the terminal custody gate of #252. + +A candide-only test additionally asserts the blinding subgraph resolves in the +cosmo_val DAG dry-run. +""" + +import importlib.util +import os +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest + +from sp_validation import blinding +from sp_validation import sacc_io as sio +from sp_validation.cosmo_val import sacc_writers as sw + + +def _repo_root(): + return next( + p for p in Path(__file__).resolve().parents if (p / "pyproject.toml").exists() + ) + + +def _load_module(rel_path, name): + """Import a workflow module/script by file path (off the package path).""" + path = _repo_root() / rel_path + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +common = _load_module("workflow/common.py", "wf_common") +asm = _load_module("workflow/scripts/assemble_sacc.py", "assemble_sacc") + + +# --------------------------------------------------------------------------- # +# 1. common.py path helpers mirror sp_validation.blinding (drift guard) +# --------------------------------------------------------------------------- # +_STEMS = [ + "SP_v1.4.6.3_xi_reporting_minsep=1.0_maxsep=250.0_nbins=20_npatch=100", + "SP_v1.4.6.3_leak_corr_xi_integration", + "pseudo_cl_SP_v1.4.6.3_blind=A_powspace_nbins=32", + "pseudo_cl_SP_v1.4.6.3_leak_corr_blind=A_powspace_nbins=32", +] + + +@pytest.mark.parametrize("stem", _STEMS) +def test_blinded_path_mirrors_blinding_part_paths(stem): + part = f"/out/{stem}.sacc" + assert common.blinded_path(part) == blinding.part_paths(part)["blinded"] + + +def test_blind_state_paths_mirror_blinding_init_paths(): + version = "SP_v1.4.6.3_leak_corr" + common_paths = common.blind_state_paths(version) + ref = blinding.init_paths(common.blind_state_dir(version)) + assert common_paths == ref + + +@pytest.mark.parametrize( + "stem,expected", + [ + (_STEMS[0], "SP_v1.4.6.3"), + (_STEMS[1], "SP_v1.4.6.3_leak_corr"), + (_STEMS[2], "SP_v1.4.6.3"), + (_STEMS[3], "SP_v1.4.6.3_leak_corr"), + ], +) +def test_version_of_extracts_catalogue_version(stem, expected): + assert common.version_of(stem) == expected + + +def test_version_of_raises_without_version(): + with pytest.raises(ValueError, match="no catalogue version"): + common.version_of("cosebis_no_version_here") + + +def test_blindable_part_switches_on_run_type(monkeypatch): + part = "/out/SP_v1.4.6.3_xi_integration.sacc" + monkeypatch.setattr(common, "RUN_TYPE", "data") + assert common.blindable_part(part) == common.blinded_path(part) + monkeypatch.setattr(common, "RUN_TYPE", "mock") + assert common.blindable_part(part) == part + + +# --------------------------------------------------------------------------- # +# 2. Data-run fail-closed assembly (#252 terminal custody gate) +# --------------------------------------------------------------------------- # +META = {"catalogue_version": "vSYNTH", "npatch": 1} +# Two arbitrary-but-consistent hex stamps standing in for a real blind's +# sha256(seed) / config digest; the assembly only checks they agree across parts. +_COMMIT = "a" * 64 +_DIGEST = "b" * 64 + + +def _nz(): + return np.linspace(0.01, 2.0, 40), np.random.default_rng(0).uniform(0.1, 1.0, 40) + + +def _spd(n, seed): + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _data_parts(tmp_path, *, conceal, one_plaintext=False): + """Write the five per-statistic parts as ``type='data'``. + + ``conceal`` stamps every part with the shared blind (concealed=True). With + ``one_plaintext`` the ξ± reporting part is left unconcealed — a blinded / + plaintext mix the assembly must refuse. + """ + nz = {0: _nz()} + theta = np.geomspace(1.0, 100.0, 6) + + xi = sw.xi_to_sacc( + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" + ) + xi.add_covariance(_spd(len(xi.mean), 1)) + co = sw.cosebis_to_sacc( + nz, + META, + { + "En": np.arange(1, 6) * 1e-6, + "Bn": np.arange(1, 6) * 1e-7, + "cov": _spd(10, 3), + }, + (1.0, 100.0), + ) + eb_arrays = {k: np.arange(6) * (i + 1) * 1e-6 for i, k in enumerate(sio.PURE_KEYS)} + eb = sw.pure_eb_to_sacc(nz, META, theta, eb_arrays, covariance=_spd(36, 4)) + rho = {"theta": theta} + tau = {"theta": theta} + rng = np.random.default_rng(5) + for k in sw.RHO_K: + for s in ("p", "m"): + rho[f"rho_{k}_{s}"] = rng.normal(size=6) * 1e-6 + rho[f"varrho_{k}_{s}"] = rng.uniform(1e-14, 1e-13, 6) + for k in sw.TAU_K: + for s in ("p", "m"): + tau[f"tau_{k}_{s}"] = rng.normal(size=6) * 1e-6 + tau[f"vartau_{k}_{s}"] = rng.uniform(1e-14, 1e-13, 6) + rt = sw.rho_tau_to_sacc(nz, META, rho, tau) + + parts = {"xi_reporting": xi, "cosebis": co, "pure_eb": eb, "rho_tau": rt} + paths = {} + for name, part in parts.items(): + if conceal and not (one_plaintext and name == "xi_reporting"): + blinding._stamp_provenance(part, _COMMIT, "A", _DIGEST) + p = tmp_path / f"{name}.sacc" + sio.save(part, str(p), type="data") + paths[name] = str(p) + return paths + + +def test_data_assemble_fails_closed_on_unblinded_part(tmp_path): + """A data run refuses to assemble an unconcealed real part (fail closed).""" + paths = _data_parts(tmp_path, conceal=False) + with pytest.raises(ValueError, match="refusing to load an unblinded"): + asm.assemble_sacc( + "vSYNTH", paths, str(tmp_path / "vSYNTH.sacc"), placeholder_var=1.0 + ) + + +def test_data_assemble_passes_on_blinded_parts(tmp_path): + """With every part concealed under one blind, the data-run assembly succeeds + and stamps the shared commitment on the terminal file.""" + paths = _data_parts(tmp_path, conceal=True) + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + assert s.metadata["concealed"] is True + assert s.metadata["blind_commitment"] == _COMMIT + assert s.metadata["blind_config_digest"] == _DIGEST + # Round-trips through the fail-closed load gate without an escape hatch. + assert sio.load(str(out)).metadata["concealed"] is True + + +def test_data_assemble_refuses_blinded_plaintext_mix(tmp_path): + """A concealed ξ± beside a plaintext one is a custody violation — refuse.""" + paths = _data_parts(tmp_path, conceal=True, one_plaintext=True) + # The plaintext ξ± reporting part fails the load gate first (data + not + # concealed), so the mix can never even reach assembly. + with pytest.raises(ValueError, match="refusing to load an unblinded"): + asm.assemble_sacc( + "vSYNTH", paths, str(tmp_path / "vSYNTH.sacc"), placeholder_var=1.0 + ) + + +def test_assert_consistent_blind_rejects_divergent_commitments(tmp_path): + """Two ξ± parts blinded under different commitments must never combine.""" + nz = {0: _nz()} + theta = np.geomspace(1.0, 100.0, 6) + a = sw.xi_to_sacc( + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" + ) + b = sw.xi_to_sacc( + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="integration" + ) + blinding._stamp_provenance(a, _COMMIT, "A", _DIGEST) + blinding._stamp_provenance(b, "c" * 64, "A", _DIGEST) + with pytest.raises(ValueError, match="different blind commitments"): + blinding.assert_consistent_blind([a, b]) + + +# --------------------------------------------------------------------------- # +# 3. The blinding subgraph resolves in the cosmo_val DAG (candide-only) +# --------------------------------------------------------------------------- # +requires_candide_data = pytest.mark.skipif( + not Path("/n17data/cdaley/unions").exists(), + reason="candide-local workflow config/data (/n17data) absent — off-cluster", +) + + +@requires_candide_data +def test_blinding_subgraph_in_cosmo_val_dry_run(): + """A data-run cosmo_val assemble pulls blind_init + blind_part, and binds the + ξ± / pseudo-Cℓ parts to their *_blinded siblings.""" + env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} + env.pop("SNAKEMAKE_PROFILE", None) + result = subprocess.run( + [ + sys.executable, + "-m", + "snakemake", + "assemble_sacc_all", + "--dry-run", + "--cores", + "1", + "--configfile", + "config/config.yaml", + ], + cwd=_repo_root() / "papers/cosmo_val", + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=180, + check=False, + ) + assert result.returncode == 0, result.stdout + out = result.stdout + assert "rule blind_init:" in out, out + assert "rule blind_part:" in out, out + # assemble consumes the blinded ξ± reporting + integration and pseudo-Cℓ. + assert ( + "_xi_reporting_minsep=1.0_maxsep=250.0_nbins=20_npatch=100_blinded.sacc" in out + ) + assert "_xi_integration_blinded.sacc" in out + assert "_blind=A_powspace_nbins=32_blinded.sacc" in out From 4f7dc210a6e9f99bcbf67f55d936edfcc76f2a97 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 16:32:43 +0200 Subject: [PATCH 026/160] revert(sacc): drop xi_integration from terminal file per #247 ruling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep the integration-grid ξ± as its own per-part intermediate ({version}_xi_integration.sacc) rather than folding it into the terminal {version}.sacc. Per the #247 ruling (comment 5033716753), the terminal file carries the analysis vector only; COSEBIs/pure-E/B consume the integration part directly, and Snakemake provenance covers its traceability. This keeps the terminal file at tens-of-MB scale. Removes xi_integration from CANONICAL, the cv_xi_integration_sacc helper, and the fiducial-gated assemble input. Keeps the fail-closed allow_unblinded gate, glass A/B/C comments, and dependency restorations from the prior rework. The integration part stays blinded at birth on data runs (per #253). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD --- src/sp_validation/cosmo_val/sacc_writers.py | 14 +++++----- workflow/rules/cosmo_val.smk | 29 ++++++--------------- workflow/rules/twopoint.smk | 10 +++---- workflow/scripts/assemble_sacc.py | 27 ++++++------------- workflow/scripts/run_2pcf_highres.py | 8 +++--- 5 files changed, 32 insertions(+), 56 deletions(-) diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index b59db0c2..de5fddf3 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -12,11 +12,12 @@ *not* ``sacc.concatenate_data_sets``, whose unvalidated block-diagonal the contract rules out). -The integration-grid ``{version}_xi_integration.sacc`` is one more part -(:func:`xi_to_sacc` with ``grid="integration"`` and a ``DiagonalCovariance`` from -TreeCorr ``varxip``/``varxim``); COSEBIs and pure-E/B consume it, and -:func:`assemble_analysis_sacc` folds its integration-grid ξ± rows (tagged -``grid="integration"``) into the single terminal ``{version}.sacc``. +The integration-grid ``{version}_xi_integration.sacc`` is an intermediate +per-part file (:func:`xi_to_sacc` with ``grid="integration"`` and a +``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim``); COSEBIs and pure-E/B +consume it. It is blinded at birth on data runs (per PR #253) but does not join +the terminal ``{version}.sacc`` — Snakemake provenance covers its traceability +(see #247 ruling). Everything here is single-bin today (``bins=(0, 0)``); the interface is tomography-native so a future round supplies real bin pairs unchanged. @@ -223,8 +224,7 @@ def assemble_analysis_sacc(nz, metadata, parts): Each part is a single-statistic Sacc (from a ``*_to_sacc`` writer, loaded from disk) carrying its own covariance = its block. This re-adds every part's data points into one Sacc in the order the parts are given — which - must be the canonical order (ξ± reporting, ξ± integration, pseudo-Cℓ, - COSEBIs, pure-E/B, ρ, τ) + must be the canonical order (ξ± reporting, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ) — and assembles a single ``BlockDiagonalCovariance`` from the per-part covariance blocks. Point insertion order and block order therefore agree by construction, which ``sacc_io.assemble_covariance`` validates (contiguous, diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 73ec22c7..3536ed11 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -169,16 +169,6 @@ def cv_xi_reporting_sacc(version): ) -def cv_xi_integration_sacc(version): - """Integration-grid ξ± SACC part the xi_highres rule (run_2pcf_highres.py) writes. - - A per-statistic part (grid='integration', its own DiagonalCovariance from - TreeCorr varxip/varxim), not a terminal file: assemble_sacc folds its rows - into {version}.sacc. The name matches xi_highres's fiducial-keyed output. - """ - return str(COSMO_VAL / f"{version}_xi_integration.sacc") - - def cv_analysis_sacc(version): """Terminal assembled analysis file {version}.sacc.""" return str(COSMO_VAL / f"{version}.sacc") @@ -452,14 +442,17 @@ rule cv_summarize_bmodes: # --------------------------------------------------------------------------- # Terminal analysis file: assemble the per-statistic SACC parts into {version}.sacc # --------------------------------------------------------------------------- -# The born-as-SACC parts (xi_reporting, xi_integration, pseudo_cl, cosebis, -# pure_eb, rho_tau) are each written by their own rule carrying its own covariance -# block, except ξ± reporting and pseudo-Cℓ which are born cov-less by design. The -# integration-grid ξ± is fiducial-only, so it joins the fiducial version's file -# alone (see cv_assemble_inputs). assemble_sacc.py +# The five born-as-SACC parts (xi_reporting, pseudo_cl, cosebis, pure_eb, rho_tau) +# are each written by their own rule carrying its own covariance block, except +# ξ± reporting and pseudo-Cℓ which are born cov-less by design. assemble_sacc.py # loads the parts in canonical order and rebuilds one {version}.sacc with a # single BlockDiagonalCovariance (point-insertion order = block order). # +# The integration-grid ξ± (grid='integration') is deliberately NOT gathered here: +# it persists as its own per-part intermediate {version}_xi_integration.sacc, +# consumed by COSEBIs/pure-E/B, with Snakemake provenance covering traceability +# (see #247 ruling). The terminal file carries the analysis vector only. +# # The pseudo-Cℓ part is the TAGGED, blinded inference product (blind=A, powspace, # nbins=32) — the same pseudo-Cℓ today's cosmosis_fitting.py consumes — so the # analysis file stays byte-comparable against it (PR-3's converter). Its real @@ -488,12 +481,6 @@ def cv_assemble_inputs(version): pure_eb=cv_pure_eb_sacc(version), rho_tau=cv_rho_tau_sacc(version), ) - # The integration-grid ξ± is a fiducial-only product (the 10k-bin MPI run in - # xi_highres emits only {fiducial}_xi_integration.sacc), so it folds into the - # fiducial version's terminal file alone; other versions' {version}.sacc omit - # the integration rows rather than trigger a job with no output to bind. - if version == config["fiducial"]["version"]: - parts["xi_integration"] = cv_xi_integration_sacc(version) if CV.get("include_pseudo_cl", False): parts["pseudo_cl"] = cv_pseudo_cl_analysis_sacc(version) parts["pseudo_cl_cov"] = cv_pseudo_cl_cov(version) diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 9eb90c64..07e36372 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -33,11 +33,11 @@ rule xi: rule xi_highres: """High-resolution xi for COSEBIS integration. - Born-as-SACC part: {version}_xi_integration.sacc (a DiagonalCovariance from - TreeCorr varxip/varxim). COSEBIs and pure-E/B consume it, and rule - assemble_sacc folds its integration-grid ξ± rows (grid='integration') into - the single terminal {version}.sacc. The raw .txt dump is kept as a - convergence byproduct. + Intermediate born-as-SACC part: {version}_xi_integration.sacc (a + DiagonalCovariance from TreeCorr varxip/varxim). COSEBIs and pure-E/B consume + it; it stays a standalone per-part file and does not join the terminal + {version}.sacc (see #247 ruling). The raw .txt dump is kept as a convergence + byproduct. """ container: None output: diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 4eec725d..cfad6a8f 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -6,8 +6,7 @@ Each per-statistic ``*.sacc`` *part* (written born-as-SACC by the mixins and the run_2pcf / generate_pseudo_cl scripts) holds one statistic. The assembler loads -them in canonical order — ξ± reporting, ξ± integration, pseudo-Cℓ, COSEBIs, -pure-E/B, ρ/τ — and +them in canonical order — ξ± reporting, pseudo-Cℓ, COSEBIs, pure-E/B, ρ/τ — and calls :func:`sacc_writers.assemble_analysis_sacc`, which rebuilds one Sacc with a single ``BlockDiagonalCovariance`` (point-insertion order = block order, validated by ``sacc_io.assemble_covariance``). @@ -15,10 +14,9 @@ Covariance sourcing (the part-by-part decision) ----------------------------------------------- ``assemble_analysis_sacc`` REQUIRES every part to carry its own covariance block. -The ξ± integration, COSEBIs, pure-E/B and ρ/τ parts already do (their writers -attach it — ξ± integration a ``DiagonalCovariance`` from TreeCorr varxip/varxim). -The ξ± reporting and pseudo-Cℓ parts are born cov-less by design; this script -injects their blocks before assembly: +The COSEBIs, pure-E/B and ρ/τ parts already do (their writers attach it). The +ξ± reporting and pseudo-Cℓ parts are born cov-less by design; this script injects +their blocks before assembly: * **ξ± reporting** — the CosmoCov theory covariance ``.txt`` (``--xi-cov``). For the single-bin round it is already ``[ξ+; ξ−]``-ordered (CosmoCov / covdat_to_fits: @@ -52,17 +50,8 @@ _CL_ORDER = ("EE", "BB", "EB") # Canonical part order — the order assemble_analysis_sacc inserts points in, which -# must match the covariance block order. Missing parts are simply skipped. The -# integration-grid ξ± part (grid='integration', its own DiagonalCovariance) sits -# next to the reporting ξ±: both are ξ± sections distinguished only by grid tag. -CANONICAL = ( - "xi_reporting", - "xi_integration", - "pseudo_cl", - "cosebis", - "pure_eb", - "rho_tau", -) +# must match the covariance block order. Missing parts are simply skipped. +CANONICAL = ("xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau") def _pseudo_cl_cov_block(cov_fits, hdu): @@ -89,8 +78,8 @@ def _pseudo_cl_cov_block(cov_fits, hdu): def _attach_cov(part, name, xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var): """Ensure ``part`` carries a covariance, injecting the xi/pseudo-Cℓ block. - ``part`` is mutated in place. xi_integration/cosebis/pure_eb/rho_tau parts - already carry their covariance and pass straight through. Raises loudly if a required xi / + ``part`` is mutated in place. cosebis/pure_eb/rho_tau parts already carry + their covariance and pass straight through. Raises loudly if a required xi / pseudo-Cℓ block is missing and no placeholder was requested. """ if part.covariance is not None: diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py index d76ae7b6..9fcab0d0 100644 --- a/workflow/scripts/run_2pcf_highres.py +++ b/workflow/scripts/run_2pcf_highres.py @@ -206,10 +206,10 @@ def compute_patch_centers(ra, dec): def write_xi_integration_sacc(gg): """Write the integration-grid ξ± SACC part (``{version}_xi_integration.sacc``). - This is a per-statistic part — COSEBIs and pure-E/B consume it, and - ``rule assemble_sacc`` folds its integration-grid ξ± rows into the single - terminal ``{version}.sacc``. It carries a ``DiagonalCovariance`` from TreeCorr - ``varxip``/``varxim`` + This is an intermediate per-statistic part — COSEBIs and pure-E/B consume it. + It stays a standalone per-part file and does not join the terminal + ``{version}.sacc`` (see #247 ruling). It carries a ``DiagonalCovariance`` from + TreeCorr ``varxip``/``varxim`` (npatch=1 leaves shot-noise variance as the only covariance estimate). Both run paths land here: in-container this uses the full SACC stack; on the bare-host MPI run only ``sacc_io`` + the n(z) file are needed (no healpy). From b60fcb9ba44e52c66f779bd94e41e13ec67a37cf Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 16:50:33 +0200 Subject: [PATCH 027/160] chore: drop files resurrected from a stale base (deleted on develop) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "Rebuild PR4" base re-added the cosmo_inference/ analysis tree that develop removed in #236 (Clean up cosmo_inference folder); the develop merge kept them because a delete-vs-readd does not conflict. Remove the 80 resurrected files (notebooks, cosmosis_config .ini set, cosmo_inference scripts, get_chi2 notebooks, pipeline shells) to match develop. None is PR4 scope — PR4 is the SACC migration (workflow/ rules + scripts, sacc-related src + tests). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD --- cosmo_inference/cfis_pipeline.sh | 66 - cosmo_inference/cosmocov_config/cosmocov.ini | 80 - .../cosmosis_config/cosmosis_pipeline.ini | 89 - .../cosmosis_pipeline_A_ia.ini | 108 -- .../cosmosis_pipeline_A_ia_cell.ini | 105 -- .../cosmosis_pipeline_A_psf.ini | 114 -- ..._minsep=1_maxsep=250_nbins=20_npatch=1.ini | 124 -- ..._minsep=1_maxsep=250_nbins=20_npatch=1.ini | 124 -- ...sep=1.0_maxsep=250.0_nbins=20_npatch=1.ini | 124 -- ....0_maxsep=250.0_nbins=20_npatch=1_cell.ini | 113 -- .../cosmosis_pipeline_SP_v1.4.6.3_A_cell.ini | 114 -- ..._pipeline_SP_v1.4.6.3_leak_corr_A_cell.ini | 111 -- ...v1.4.6.3_leak_corr_HMCode_nobar_A_cell.ini | 114 -- ...ne_SP_v1.4.6.3_leak_corr_OneCov_A_cell.ini | 114 -- ...e_SP_v1.4.6.3_leak_corr_halofit_A_cell.ini | 114 -- ..._leak_corr_include_large_scales_A_cell.ini | 114 -- ...SP_v1.4.6.3_leak_corr_kmax=1Mpc_A_cell.ini | 114 -- ...SP_v1.4.6.3_leak_corr_kmax=3Mpc_A_cell.ini | 114 -- ...SP_v1.4.6.3_leak_corr_kmax=5Mpc_A_cell.ini | 114 -- ...v1.4.6.3_leak_corr_large_scales_A_cell.ini | 114 -- ...v1.4.6.3_leak_corr_small_scales_A_cell.ini | 114 -- .../cosmosis_pipeline_SP_v1.4.6.3_B_cell.ini | 111 -- ..._pipeline_SP_v1.4.6.3_leak_corr_B_cell.ini | 111 -- ...v1.4.6.3_leak_corr_HMCode_nobar_B_cell.ini | 111 -- ...ne_SP_v1.4.6.3_leak_corr_OneCov_B_cell.ini | 111 -- ...e_SP_v1.4.6.3_leak_corr_halofit_B_cell.ini | 111 -- ..._leak_corr_include_large_scales_B_cell.ini | 111 -- ...SP_v1.4.6.3_leak_corr_kmax=1Mpc_B_cell.ini | 111 -- ...SP_v1.4.6.3_leak_corr_kmax=3Mpc_B_cell.ini | 111 -- ...SP_v1.4.6.3_leak_corr_kmax=5Mpc_B_cell.ini | 111 -- ...v1.4.6.3_leak_corr_large_scales_B_cell.ini | 111 -- ...v1.4.6.3_leak_corr_small_scales_B_cell.ini | 111 -- .../cosmosis_pipeline_SP_v1.4.6.3_C_cell.ini | 114 -- ..._pipeline_SP_v1.4.6.3_leak_corr_C_cell.ini | 111 -- ...v1.4.6.3_leak_corr_HMCode_nobar_C_cell.ini | 114 -- ...ne_SP_v1.4.6.3_leak_corr_OneCov_C_cell.ini | 114 -- ...e_SP_v1.4.6.3_leak_corr_halofit_C_cell.ini | 114 -- ..._leak_corr_include_large_scales_C_cell.ini | 114 -- ...SP_v1.4.6.3_leak_corr_kmax=1Mpc_C_cell.ini | 114 -- ...SP_v1.4.6.3_leak_corr_kmax=3Mpc_C_cell.ini | 114 -- ...SP_v1.4.6.3_leak_corr_kmax=5Mpc_C_cell.ini | 114 -- ...v1.4.6.3_leak_corr_large_scales_C_cell.ini | 114 -- ...v1.4.6.3_leak_corr_small_scales_C_cell.ini | 114 -- cosmo_inference/cosmosis_config/priors.ini | 11 - .../cosmosis_config/priors_mock.ini | 15 - .../cosmosis_config/priors_mock_cell.ini | 11 - .../priors_mock_cell_no_sys.ini | 5 - .../cosmosis_config/priors_psf.ini | 15 - cosmo_inference/cosmosis_config/values.ini | 27 - .../cosmosis_config/values_empty.ini | 27 - cosmo_inference/cosmosis_config/values_ia.ini | 26 - .../cosmosis_config/values_ia_no_sys.ini | 26 - .../cosmosis_config/values_ia_test.ini | 27 - .../cosmosis_config/values_psf.ini | 30 - .../cosmosis_config/values_template.ini | 23 - cosmo_inference/get_chi2.ipynb | 1391 --------------- cosmo_inference/get_chi2_cell.ipynb | 1527 ---------------- .../S8_om_sigma8_whisker.ipynb | 645 ------- .../best_fit_xipm.ipynb | 607 ------- .../contours.ipynb | 950 ---------- .../get_chi2.ipynb | 690 -------- .../get_chi2_glass_mock.ipynb | 565 ------ .../get_prior_psf_leakage.ipynb | 261 --- .../glass_mock_hist.ipynb | 586 ------- .../masking.ipynb | 132 -- .../nonlin_k_analysis.ipynb | 174 -- .../unblinding_party_plots.py | 894 ---------- .../2D_cosmic_shear_unblinding/utils.py | 442 ----- cosmo_inference/notebooks/cfis_analysis.ipynb | 1065 ------------ cosmo_inference/notebooks/cfis_mcmc.ipynb | 1546 ----------------- .../notebooks/get_prior_psf_leakage.ipynb | 269 --- cosmo_inference/pipeline.sh | 131 -- cosmo_inference/scripts/2pt_like_xi_sys.py | 614 ------- cosmo_inference/scripts/cosmocov_process.py | 81 - cosmo_inference/scripts/masking.py | 319 ---- cosmo_inference/scripts/matching.py | 40 - cosmo_inference/scripts/nz_writeout.py | 26 - cosmo_inference/scripts/slurm.sh | 22 - cosmo_inference/scripts/treecorr_calc.py | 107 -- cosmo_inference/scripts/xi_sys_psf.py | 53 - 80 files changed, 18150 deletions(-) delete mode 100644 cosmo_inference/cfis_pipeline.sh delete mode 100644 cosmo_inference/cosmocov_config/cosmocov.ini delete mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline.ini delete mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini delete mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_cell.ini delete mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_A_psf.ini delete mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini delete mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini delete mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini delete mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_A_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_A_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_A_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_A_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_A_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_A_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_A_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_A_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_A_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_A_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_A_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_B_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_B_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_B_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_B_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_B_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_B_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_B_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_B_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_B_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_B_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_B_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_C_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_C_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_C_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_C_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_C_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_C_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_C_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_C_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_C_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_C_cell.ini delete mode 100755 cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_C_cell.ini delete mode 100644 cosmo_inference/cosmosis_config/priors.ini delete mode 100644 cosmo_inference/cosmosis_config/priors_mock.ini delete mode 100644 cosmo_inference/cosmosis_config/priors_mock_cell.ini delete mode 100644 cosmo_inference/cosmosis_config/priors_mock_cell_no_sys.ini delete mode 100644 cosmo_inference/cosmosis_config/priors_psf.ini delete mode 100644 cosmo_inference/cosmosis_config/values.ini delete mode 100644 cosmo_inference/cosmosis_config/values_empty.ini delete mode 100644 cosmo_inference/cosmosis_config/values_ia.ini delete mode 100644 cosmo_inference/cosmosis_config/values_ia_no_sys.ini delete mode 100644 cosmo_inference/cosmosis_config/values_ia_test.ini delete mode 100644 cosmo_inference/cosmosis_config/values_psf.ini delete mode 100644 cosmo_inference/cosmosis_config/values_template.ini delete mode 100644 cosmo_inference/get_chi2.ipynb delete mode 100644 cosmo_inference/get_chi2_cell.ipynb delete mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/S8_om_sigma8_whisker.ipynb delete mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/best_fit_xipm.ipynb delete mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/contours.ipynb delete mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2.ipynb delete mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2_glass_mock.ipynb delete mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_prior_psf_leakage.ipynb delete mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/glass_mock_hist.ipynb delete mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/masking.ipynb delete mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/nonlin_k_analysis.ipynb delete mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_unblinding/unblinding_party_plots.py delete mode 100644 cosmo_inference/notebooks/2D_cosmic_shear_unblinding/utils.py delete mode 100644 cosmo_inference/notebooks/cfis_analysis.ipynb delete mode 100644 cosmo_inference/notebooks/cfis_mcmc.ipynb delete mode 100644 cosmo_inference/notebooks/get_prior_psf_leakage.ipynb delete mode 100755 cosmo_inference/pipeline.sh delete mode 100644 cosmo_inference/scripts/2pt_like_xi_sys.py delete mode 100644 cosmo_inference/scripts/cosmocov_process.py delete mode 100644 cosmo_inference/scripts/masking.py delete mode 100644 cosmo_inference/scripts/matching.py delete mode 100644 cosmo_inference/scripts/nz_writeout.py delete mode 100644 cosmo_inference/scripts/slurm.sh delete mode 100644 cosmo_inference/scripts/treecorr_calc.py delete mode 100644 cosmo_inference/scripts/xi_sys_psf.py diff --git a/cosmo_inference/cfis_pipeline.sh b/cosmo_inference/cfis_pipeline.sh deleted file mode 100644 index dadab6fb..00000000 --- a/cosmo_inference/cfis_pipeline.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -read -p 'SHEAR CATALOGUE: ' shear_cat -# read -p 'NZ CATALOGUE: ' nz_cat -read -p 'DATA ROOT: ' root -read -p 'OUT ROOT: ' out_root -# read -p 'BLIND:' blind -mkdir -p data/${root} - -# #################STEP 0: RUN NOTEBOOK TO ANALYSE CATALOGUE; DERIVE PLOTS################## -# #File: cfis_analysis.ipynb - -# ##################STEP 1: CALCULATE XIP/XIM (OUTPUTS TREECORR FITS CATALOG)############### -python treecorr_calc.py $shear_cat $root - -echo -e "2PCF's calculated!\n" - -# # ##################STEP 2: WRITE NZ's###################################################### -# python nz_writeout.py $nz_cat $root $blind - -# echo -e "nz's written out!\n" - -# # # # ##################STEP 3: ESTIMATE COVMATS################################################ - -# # #edit ini file -# mkdir -p data/${root}/covs - -# nz_file="data/${root}/nz_shapepipe_A.txt" - -# # sed -i "/shear_REDSHIFT_FILE/c shear_REDSHIFT_FILE : $nz_file" cosmocov.ini -# # sed -i "/clustering_REDSHIFT_FILE/c clustering_REDSHIFT_FILE : $nz_file" cosmocov.ini -# # sed -i "/outdir/c outdir : data/$root/covs/" cosmocov.ini - -# echo -e "Running CosmoCov...\n" - -# ##run cosmocov -# for i in {1..3}; -# do ../CosmoCov/covs/cov $i cosmocov.ini; -# done - -# # do postprocessing (plot covmat and write into txt file) -# f="data/${root}/covs/cov_${root}"; cat data/${root}/covs/out_cov* > $f; python cosmocov_process.py $f - -# # # # # ##################STEP 4: COMBINE########################################################## -# xip_cat="data/${root}/xiplus_${root}.fits" -# xim_cat="data/${root}/ximinus_${root}.fits" -# covmat="data/${root}/covs/cov_${root}.txt" - -# out_file="$PWD/data/${root}/cosmosis_${root}.fits" - -python cosmosis_fitting.py /n23data1/n06data/lgoh/scratch/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/data/SP_v1.4_A/xiplus_SP_v1.4_A.fits /n23data1/n06data/lgoh/scratch/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/data/SP_v1.4_A/ximinus_SP_v1.4_A.fits /n23data1/n06data/lgoh/scratch/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/data/SP_v1.4_A/covs/cov_SP_v1.4.txt /n23data1/n06data/lgoh/scratch/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/data/nz/nz_shapepipe_A.txt /n23data1/n06data/lgoh/scratch/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/data/SP_v1.4_A/cosmosis_SP_v1.4_A.fits - -# # # # ##################STEP 5: RUN COSMOSIS##################################################### -# echo -e "Running CosmoSIS...\n" - -# sed -i "/SCRATCH = /c SCRATCH = $WORK/UNIONS/chains/${out_root}/" cosmosis_config/cosmosis_pipeline.ini -# sed -i "/FITS_FILE = /c FITS_FILE = ${out_file}" cosmosis_config/cosmosis_pipeline.ini -# sed -i "/filename = /c filename = %(SCRATCH)s/samples_${out_root}.txt" cosmosis_config/cosmosis_pipeline.ini - -# #submit cosmosis job to run on cluster - -# sbatch -J cfis_${root} --output=$WORK/UNIONS/cfis_${out_root}.log slurm.sh - -# echo -e "-------------PIPELINE END----------------" - -# # ##################STEP 6: RUN NOTEBOOK TO ANALYSE CONTOURS (WITH GETDIST)################## -# #File: CFIS_plotting.ipynb \ No newline at end of file diff --git a/cosmo_inference/cosmocov_config/cosmocov.ini b/cosmo_inference/cosmocov_config/cosmocov.ini deleted file mode 100644 index 0afbdc64..00000000 --- a/cosmo_inference/cosmocov_config/cosmocov.ini +++ /dev/null @@ -1,80 +0,0 @@ -# -# Cosmological parameters -# -Omega_m : 0.25 -Omega_v : 0.75 -sigma_8 : 0.8 -n_spec : 0.95 -w0 : -1 -wa : 0 -omb : 0.044 -h0 : 0.7 - - -# Survey and galaxy parameters -# -# area in degrees -# n_gal,lens_n_gal in gals/arcmin^2 - -#FOR LENSFIT -#area : 2138 -#sourcephotoz : multihisto -#lensphotoz : multihisto -#source_tomobins : 1 -#lens_tomobins : 1 -#sigma_e : 0.41016433003564806 -#source_n_gal : 10.78 - -#FOR SHAPEPIPE -; area : 3218.19 -; sourcephotoz : multihisto -; lensphotoz : multihisto -; source_tomobins : 1 -; lens_tomobins : 1 -; sigma_e : 0.491712 -; source_n_gal : 8.42 - -#FOR SHAPEPIPE 1500 -; area : 1453 -; sourcephotoz : multihisto -; lensphotoz : multihisto -; source_tomobins : 1 -; lens_tomobins : 1 -; sigma_e : 0.4808326112068524 -; source_n_gal : 7.92 - -#FOR SHAPEPIPE v1.3/v1.4 -area : 2782 -sourcephotoz : multihisto -lensphotoz : multihisto -source_tomobins : 1 -lens_tomobins : 1 -sigma_e : 0.4370966656902571 -; source_n_gal: 7.6 #v1.3 -source_n_gal : 7.18 # v1.4.1 -lens_n_gal : 7.18 - -c_footprint_file: - - -# IA parameters -IA : 1 -A_ia : 0.0 -eta_ia : 0.0 - - -# Covariance paramters -# -# tmin,tmax in arcminutes -tmin : 0.1 -tmax : 250 -ntheta : 20 -ng : 1 -cng : 1 - - -#mkdir before running! -filename : out_cov -ss : true -ls : false -ll : false \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline.ini deleted file mode 100644 index f6b11411..00000000 --- a/cosmo_inference/cosmosis_config/cosmosis_pipeline.ini +++ /dev/null @@ -1,89 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -# Specify the directory of your cosmological CosmoSIS library -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -modules = consistency camb load_nz_fits linear_alignment projection 2pt_shear add_xi_sys 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 -timing = T -debug = T - -[runtime] -sampler = metropolis -resume = T -verbosity = debug - -[output] -format = text -lock = F - -[metropolis] -samples = 10000000 - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=all -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=takahashi -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -get_kernel_peaks = F -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[add_xi_sys] -file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py -data_file=%(FITS FILE)s -rho_stats_name=RHO_STATS - -[tau_from_rho] -file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py -data_file=%(FITS_FILE)s - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS -like_name=2pt_like -add_xi_sys=T - -angle_range_XI_PLUS_1_1= 1.0 200.0 -angle_range_XI_MINUS_1_1= 1.0 200.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini deleted file mode 100644 index eb3ab166..00000000 --- a/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini +++ /dev/null @@ -1,108 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -COSMOSIS_DIR = /n23data1/n06data/lgoh/scratch/cosmosis-standard-library_lisa - - -[pipeline] -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[polychord] -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[test] - -[output] -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_xi_plus shear_xi_minus -verbose = F - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -data_sets=XI_PLUS XI_MINUS -like_name=2pt_like - -angle_range_XI_PLUS_1_1= 10.0 200.0 -angle_range_XI_MINUS_1_1= 20.0 200.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_cell.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_cell.ini deleted file mode 100644 index 87f06064..00000000 --- a/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_cell.ini +++ /dev/null @@ -1,105 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] - - -[polychord] -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_psf.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_psf.ini deleted file mode 100644 index f9f4da51..00000000 --- a/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_psf.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias add_xi_sys tau_from_rho 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] - -[polychord] -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_xi_plus shear_xi_minus -verbose = F - -[add_xi_sys] -file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py -data_file=%(FITS_FILE)s -rho_stats_name=RHO_STATS - -[tau_from_rho] -file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py -data_file=%(FITS_FILE)s - -[2pt_like] -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_XI_PLUS_1_1= 12.0 83.0 -angle_range_XI_MINUS_1_1= 12.0 83.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini deleted file mode 100644 index efcf75f7..00000000 --- a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini +++ /dev/null @@ -1,124 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -FITS_FILE = data/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1/cosmosis_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.fits -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1 -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -priors = cosmosis_config/priors_psf.ini -values = cosmosis_config/values_psf.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear add_xi_sys tau_from_rho 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/test_new_pipeline - -[polychord] -polychord_outfile_root = SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1 -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1/samples_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_xi_plus shear_xi_minus -verbose = F - -[add_xi_sys] -file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py -data_file=%(FITS_FILE)s -rho_stats_name=RHO_STATS - -[tau_from_rho] -file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py -data_file=%(FITS_FILE)s - -[2pt_like] -add_xi_sys=T -data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_XI_PLUS_1_1= 3.0 150.0 -angle_range_XI_MINUS_1_1= 10.0 200.0 diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini deleted file mode 100644 index 0af5b180..00000000 --- a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini +++ /dev/null @@ -1,124 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -FITS_FILE = data/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1/cosmosis_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.fits -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1 -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -priors = cosmosis_config/priors_psf.ini -values = cosmosis_config/values_psf.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear add_xi_sys tau_from_rho 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/test_new_pipeline - -[polychord] -polychord_outfile_root = SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1 -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1/samples_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_xi_plus shear_xi_minus -verbose = F - -[add_xi_sys] -file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py -data_file=%(FITS_FILE)s -rho_stats_name=RHO_STATS - -[tau_from_rho] -file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py -data_file=%(FITS_FILE)s - -[2pt_like] -add_xi_sys=T -data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_XI_PLUS_1_1= 3.0 150.0 -angle_range_XI_MINUS_1_1= 10.0 200.0 diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini deleted file mode 100644 index d64ac64f..00000000 --- a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini +++ /dev/null @@ -1,124 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 -FITS_FILE = data/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/cosmosis_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_psf.ini -priors = cosmosis_config/priors_psf.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias add_xi_sys tau_from_rho 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 - -[polychord] -polychord_outfile_root = SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/samples_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_xi_plus shear_xi_minus -verbose = F - -[add_xi_sys] -file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py -data_file=%(FITS_FILE)s -rho_stats_name=RHO_STATS - -[tau_from_rho] -file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py -data_file=%(FITS_FILE)s - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py -data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS -add_xi_sys=T -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_XI_PLUS_1_1= 3.0 150.0 -angle_range_XI_MINUS_1_1= 10.0 200.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini deleted file mode 100644 index 34a7209f..00000000 --- a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini +++ /dev/null @@ -1,113 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 -FITS_FILE = data/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/cosmosis_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_psf.ini -priors = cosmosis_config/priors_psf.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell - - -[polychord] -polychord_outfile_root = SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/samples_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT_CELL -cut_zeros=F -like_name=2pt_like \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_A_cell.ini deleted file mode 100755 index 70404b87..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_A_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_A -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_A/cosmosis_SP_v1.4.6.3_A.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_A_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_A/samples_SP_v1.4.6.3_A_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_A_cell.ini deleted file mode 100755 index dad17f3d..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_A_cell.ini +++ /dev/null @@ -1,111 +0,0 @@ -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_A -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_A/cosmosis_SP_v1.4.6.3_leak_corr_A.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_A_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_A/samples_SP_v1.4.6.3_leak_corr_A_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode = power -lmax = 2508 -feedback = 0 -do_reionization = F -kmin = 1e-5 -kmax = 20.0 -nk = 200 -zmax = 5.0 -zmax_background = 5.0 -nz_background = 500 -halofit_version = mead2020_feedback -nonlinear = pk -neutrino_hierarchy = normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file = %(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear = T -position-shear = F -perbin = F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets = CELL_EE -data_file = %(FITS_FILE)s -gaussian_covariance = F -covmat_name = COVMAT -cut_zeros = F -like_name = 2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 - diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_A_cell.ini deleted file mode 100755 index e8c48898..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_A_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_HMCode_nobar_A -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_HMCode_nobar_A/cosmosis_SP_v1.4.6.3_leak_corr_HMCode_nobar_A.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_HMCode_nobar_A_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_HMCode_nobar_A/samples_SP_v1.4.6.3_leak_corr_HMCode_nobar_A_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020 -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_A_cell.ini deleted file mode 100755 index e686f245..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_A_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_OneCov_A -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_OneCov_A/cosmosis_SP_v1.4.6.3_leak_corr_OneCov_A.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_OneCov_A_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_OneCov_A/samples_SP_v1.4.6.3_leak_corr_OneCov_A_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_A_cell.ini deleted file mode 100755 index dd46e3e7..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_A_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_halofit_A -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_halofit_A/cosmosis_SP_v1.4.6.3_leak_corr_halofit_A.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_halofit_A_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_halofit_A/samples_SP_v1.4.6.3_leak_corr_halofit_A_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=takahashi -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_A_cell.ini deleted file mode 100755 index 507b2f9a..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_A_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_include_large_scales_A -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_include_large_scales_A/cosmosis_SP_v1.4.6.3_leak_corr_include_large_scales_A.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_include_large_scales_A_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_include_large_scales_A/samples_SP_v1.4.6.3_leak_corr_include_large_scales_A_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 0.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_A_cell.ini deleted file mode 100755 index 028e875c..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_A_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=1Mpc_A -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=1Mpc_A/cosmosis_SP_v1.4.6.3_leak_corr_kmax=1Mpc_A.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=1Mpc_A_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=1Mpc_A/samples_SP_v1.4.6.3_leak_corr_kmax=1Mpc_A_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 500.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_A_cell.ini deleted file mode 100755 index 32c45ad5..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_A_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=3Mpc_A -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=3Mpc_A/cosmosis_SP_v1.4.6.3_leak_corr_kmax=3Mpc_A.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=3Mpc_A_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=3Mpc_A/samples_SP_v1.4.6.3_leak_corr_kmax=3Mpc_A_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 1800.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_A_cell.ini deleted file mode 100755 index 92d61b20..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_A_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=5Mpc_A -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=5Mpc_A/cosmosis_SP_v1.4.6.3_leak_corr_kmax=5Mpc_A.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=5Mpc_A_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=5Mpc_A/samples_SP_v1.4.6.3_leak_corr_kmax=5Mpc_A_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 2048.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_A_cell.ini deleted file mode 100755 index aa3784c2..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_A_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_large_scales_A -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_large_scales_A/cosmosis_SP_v1.4.6.3_leak_corr_large_scales_A.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_large_scales_A_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_large_scales_A/samples_SP_v1.4.6.3_leak_corr_large_scales_A_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 800.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_A_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_A_cell.ini deleted file mode 100755 index 1aa71c5f..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_A/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_A_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_small_scales_A -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_small_scales_A/cosmosis_SP_v1.4.6.3_leak_corr_small_scales_A.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_small_scales_A_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_small_scales_A/samples_SP_v1.4.6.3_leak_corr_small_scales_A_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 800.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_B_cell.ini deleted file mode 100755 index db67430b..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_B_cell.ini +++ /dev/null @@ -1,111 +0,0 @@ -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_B -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_B/cosmosis_SP_v1.4.6.3_B.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_B_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_B/samples_SP_v1.4.6.3_B_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode = power -lmax = 2508 -feedback = 0 -do_reionization = F -kmin = 1e-5 -kmax = 20.0 -nk = 200 -zmax = 5.0 -zmax_background = 5.0 -nz_background = 500 -halofit_version = mead2020_feedback -nonlinear = pk -neutrino_hierarchy = normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file = %(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear = T -position-shear = F -perbin = F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets = CELL_EE -data_file = %(FITS_FILE)s -gaussian_covariance = F -covmat_name = COVMAT -cut_zeros = F -like_name = 2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 - diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_B_cell.ini deleted file mode 100755 index d1b15a8f..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_B_cell.ini +++ /dev/null @@ -1,111 +0,0 @@ -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_B -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_B/cosmosis_SP_v1.4.6.3_leak_corr_B.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_B_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_B/samples_SP_v1.4.6.3_leak_corr_B_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode = power -lmax = 2508 -feedback = 0 -do_reionization = F -kmin = 1e-5 -kmax = 20.0 -nk = 200 -zmax = 5.0 -zmax_background = 5.0 -nz_background = 500 -halofit_version = mead2020_feedback -nonlinear = pk -neutrino_hierarchy = normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file = %(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear = T -position-shear = F -perbin = F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets = CELL_EE -data_file = %(FITS_FILE)s -gaussian_covariance = F -covmat_name = COVMAT -cut_zeros = F -like_name = 2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 - diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_B_cell.ini deleted file mode 100755 index ccc62f9f..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_B_cell.ini +++ /dev/null @@ -1,111 +0,0 @@ -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_HMCode_nobar_B -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_HMCode_nobar_B/cosmosis_SP_v1.4.6.3_leak_corr_HMCode_nobar_B.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_HMCode_nobar_B_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_HMCode_nobar_B/samples_SP_v1.4.6.3_leak_corr_HMCode_nobar_B_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode = power -lmax = 2508 -feedback = 0 -do_reionization = F -kmin = 1e-5 -kmax = 20.0 -nk = 200 -zmax = 5.0 -zmax_background = 5.0 -nz_background = 500 -halofit_version = mead2020 -nonlinear = pk -neutrino_hierarchy = normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file = %(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear = T -position-shear = F -perbin = F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets = CELL_EE -data_file = %(FITS_FILE)s -gaussian_covariance = F -covmat_name = COVMAT -cut_zeros = F -like_name = 2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 - diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_B_cell.ini deleted file mode 100755 index 20b637ea..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_B_cell.ini +++ /dev/null @@ -1,111 +0,0 @@ -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_OneCov_B -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_OneCov_B/cosmosis_SP_v1.4.6.3_leak_corr_OneCov_B.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_OneCov_B_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_OneCov_B/samples_SP_v1.4.6.3_leak_corr_OneCov_B_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode = power -lmax = 2508 -feedback = 0 -do_reionization = F -kmin = 1e-5 -kmax = 20.0 -nk = 200 -zmax = 5.0 -zmax_background = 5.0 -nz_background = 500 -halofit_version = mead2020_feedback -nonlinear = pk -neutrino_hierarchy = normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file = %(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear = T -position-shear = F -perbin = F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets = CELL_EE -data_file = %(FITS_FILE)s -gaussian_covariance = F -covmat_name = COVMAT -cut_zeros = F -like_name = 2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 - diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_B_cell.ini deleted file mode 100755 index c3d99809..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_B_cell.ini +++ /dev/null @@ -1,111 +0,0 @@ -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_halofit_B -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_halofit_B/cosmosis_SP_v1.4.6.3_leak_corr_halofit_B.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_halofit_B_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_halofit_B/samples_SP_v1.4.6.3_leak_corr_halofit_B_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode = power -lmax = 2508 -feedback = 0 -do_reionization = F -kmin = 1e-5 -kmax = 20.0 -nk = 200 -zmax = 5.0 -zmax_background = 5.0 -nz_background = 500 -halofit_version = takahashi -nonlinear = pk -neutrino_hierarchy = normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file = %(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear = T -position-shear = F -perbin = F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets = CELL_EE -data_file = %(FITS_FILE)s -gaussian_covariance = F -covmat_name = COVMAT -cut_zeros = F -like_name = 2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 - diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_B_cell.ini deleted file mode 100755 index d724c837..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_B_cell.ini +++ /dev/null @@ -1,111 +0,0 @@ -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_include_large_scales_B -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_include_large_scales_B/cosmosis_SP_v1.4.6.3_leak_corr_include_large_scales_B.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_include_large_scales_B_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_include_large_scales_B/samples_SP_v1.4.6.3_leak_corr_include_large_scales_B_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode = power -lmax = 2508 -feedback = 0 -do_reionization = F -kmin = 1e-5 -kmax = 20.0 -nk = 200 -zmax = 5.0 -zmax_background = 5.0 -nz_background = 500 -halofit_version = mead2020_feedback -nonlinear = pk -neutrino_hierarchy = normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file = %(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear = T -position-shear = F -perbin = F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets = CELL_EE -data_file = %(FITS_FILE)s -gaussian_covariance = F -covmat_name = COVMAT -cut_zeros = F -like_name = 2pt_like -angle_range_CELL_EE_1_1 = 0.0 1600.0 - diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_B_cell.ini deleted file mode 100755 index 5991d198..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_B_cell.ini +++ /dev/null @@ -1,111 +0,0 @@ -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=1Mpc_B -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=1Mpc_B/cosmosis_SP_v1.4.6.3_leak_corr_kmax=1Mpc_B.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=1Mpc_B_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=1Mpc_B/samples_SP_v1.4.6.3_leak_corr_kmax=1Mpc_B_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode = power -lmax = 2508 -feedback = 0 -do_reionization = F -kmin = 1e-5 -kmax = 20.0 -nk = 200 -zmax = 5.0 -zmax_background = 5.0 -nz_background = 500 -halofit_version = mead2020_feedback -nonlinear = pk -neutrino_hierarchy = normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file = %(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear = T -position-shear = F -perbin = F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets = CELL_EE -data_file = %(FITS_FILE)s -gaussian_covariance = F -covmat_name = COVMAT -cut_zeros = F -like_name = 2pt_like -angle_range_CELL_EE_1_1 = 300.0 500.0 - diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_B_cell.ini deleted file mode 100755 index fb347172..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_B_cell.ini +++ /dev/null @@ -1,111 +0,0 @@ -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=3Mpc_B -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=3Mpc_B/cosmosis_SP_v1.4.6.3_leak_corr_kmax=3Mpc_B.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=3Mpc_B_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=3Mpc_B/samples_SP_v1.4.6.3_leak_corr_kmax=3Mpc_B_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode = power -lmax = 2508 -feedback = 0 -do_reionization = F -kmin = 1e-5 -kmax = 20.0 -nk = 200 -zmax = 5.0 -zmax_background = 5.0 -nz_background = 500 -halofit_version = mead2020_feedback -nonlinear = pk -neutrino_hierarchy = normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file = %(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear = T -position-shear = F -perbin = F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets = CELL_EE -data_file = %(FITS_FILE)s -gaussian_covariance = F -covmat_name = COVMAT -cut_zeros = F -like_name = 2pt_like -angle_range_CELL_EE_1_1 = 300.0 1800.0 - diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_B_cell.ini deleted file mode 100755 index 4005c10b..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_B_cell.ini +++ /dev/null @@ -1,111 +0,0 @@ -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=5Mpc_B -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=5Mpc_B/cosmosis_SP_v1.4.6.3_leak_corr_kmax=5Mpc_B.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=5Mpc_B_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=5Mpc_B/samples_SP_v1.4.6.3_leak_corr_kmax=5Mpc_B_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode = power -lmax = 2508 -feedback = 0 -do_reionization = F -kmin = 1e-5 -kmax = 20.0 -nk = 200 -zmax = 5.0 -zmax_background = 5.0 -nz_background = 500 -halofit_version = mead2020_feedback -nonlinear = pk -neutrino_hierarchy = normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file = %(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear = T -position-shear = F -perbin = F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets = CELL_EE -data_file = %(FITS_FILE)s -gaussian_covariance = F -covmat_name = COVMAT -cut_zeros = F -like_name = 2pt_like -angle_range_CELL_EE_1_1 = 300.0 2048.0 - diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_B_cell.ini deleted file mode 100755 index 95009a6a..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_B_cell.ini +++ /dev/null @@ -1,111 +0,0 @@ -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_large_scales_B -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_large_scales_B/cosmosis_SP_v1.4.6.3_leak_corr_large_scales_B.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_large_scales_B_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_large_scales_B/samples_SP_v1.4.6.3_leak_corr_large_scales_B_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode = power -lmax = 2508 -feedback = 0 -do_reionization = F -kmin = 1e-5 -kmax = 20.0 -nk = 200 -zmax = 5.0 -zmax_background = 5.0 -nz_background = 500 -halofit_version = mead2020_feedback -nonlinear = pk -neutrino_hierarchy = normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file = %(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear = T -position-shear = F -perbin = F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets = CELL_EE -data_file = %(FITS_FILE)s -gaussian_covariance = F -covmat_name = COVMAT -cut_zeros = F -like_name = 2pt_like -angle_range_CELL_EE_1_1 = 300.0 800.0 - diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_B_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_B_cell.ini deleted file mode 100755 index fd6d8990..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_B/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_B_cell.ini +++ /dev/null @@ -1,111 +0,0 @@ -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_small_scales_B -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_small_scales_B/cosmosis_SP_v1.4.6.3_leak_corr_small_scales_B.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_small_scales_B_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_small_scales_B/samples_SP_v1.4.6.3_leak_corr_small_scales_B_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode = power -lmax = 2508 -feedback = 0 -do_reionization = F -kmin = 1e-5 -kmax = 20.0 -nk = 200 -zmax = 5.0 -zmax_background = 5.0 -nz_background = 500 -halofit_version = mead2020_feedback -nonlinear = pk -neutrino_hierarchy = normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file = %(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear = T -position-shear = F -perbin = F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets = CELL_EE -data_file = %(FITS_FILE)s -gaussian_covariance = F -covmat_name = COVMAT -cut_zeros = F -like_name = 2pt_like -angle_range_CELL_EE_1_1 = 800.0 1600.0 - diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_C_cell.ini deleted file mode 100755 index 584a5fbb..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_C_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_C -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_C/cosmosis_SP_v1.4.6.3_C.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_C_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_C/samples_SP_v1.4.6.3_C_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_C_cell.ini deleted file mode 100755 index 341a4b7f..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_C_cell.ini +++ /dev/null @@ -1,111 +0,0 @@ -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_C -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_C/cosmosis_SP_v1.4.6.3_leak_corr_C.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_C_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_C/samples_SP_v1.4.6.3_leak_corr_C_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode = power -lmax = 2508 -feedback = 0 -do_reionization = F -kmin = 1e-5 -kmax = 20.0 -nk = 200 -zmax = 5.0 -zmax_background = 5.0 -nz_background = 500 -halofit_version = mead2020_feedback -nonlinear = pk -neutrino_hierarchy = normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file = %(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear = T -position-shear = F -perbin = F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets = CELL_EE -data_file = %(FITS_FILE)s -gaussian_covariance = F -covmat_name = COVMAT -cut_zeros = F -like_name = 2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 - diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_C_cell.ini deleted file mode 100755 index e2b478dd..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_HMCode_nobar_C_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_HMCode_nobar_C -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_HMCode_nobar_C/cosmosis_SP_v1.4.6.3_leak_corr_HMCode_nobar_C.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_HMCode_nobar_C_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_HMCode_nobar_C/samples_SP_v1.4.6.3_leak_corr_HMCode_nobar_C_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020 -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_C_cell.ini deleted file mode 100755 index 66e4a192..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_OneCov_C_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_OneCov_C -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_OneCov_C/cosmosis_SP_v1.4.6.3_leak_corr_OneCov_C.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_OneCov_C_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_OneCov_C/samples_SP_v1.4.6.3_leak_corr_OneCov_C_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_C_cell.ini deleted file mode 100755 index 0be94bbd..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_halofit_C_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_halofit_C -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_halofit_C/cosmosis_SP_v1.4.6.3_leak_corr_halofit_C.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_halofit_C_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_halofit_C/samples_SP_v1.4.6.3_leak_corr_halofit_C_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=takahashi -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_C_cell.ini deleted file mode 100755 index 718ce25d..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_include_large_scales_C_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_include_large_scales_C -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_include_large_scales_C/cosmosis_SP_v1.4.6.3_leak_corr_include_large_scales_C.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_include_large_scales_C_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_include_large_scales_C/samples_SP_v1.4.6.3_leak_corr_include_large_scales_C_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 0.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_C_cell.ini deleted file mode 100755 index 69f977d0..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=1Mpc_C_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=1Mpc_C -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=1Mpc_C/cosmosis_SP_v1.4.6.3_leak_corr_kmax=1Mpc_C.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=1Mpc_C_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=1Mpc_C/samples_SP_v1.4.6.3_leak_corr_kmax=1Mpc_C_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 500.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_C_cell.ini deleted file mode 100755 index 3ef25704..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=3Mpc_C_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=3Mpc_C -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=3Mpc_C/cosmosis_SP_v1.4.6.3_leak_corr_kmax=3Mpc_C.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=3Mpc_C_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=3Mpc_C/samples_SP_v1.4.6.3_leak_corr_kmax=3Mpc_C_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 1800.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_C_cell.ini deleted file mode 100755 index 0b82ecbf..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_kmax=5Mpc_C_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_kmax=5Mpc_C -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_kmax=5Mpc_C/cosmosis_SP_v1.4.6.3_leak_corr_kmax=5Mpc_C.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_kmax=5Mpc_C_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_kmax=5Mpc_C/samples_SP_v1.4.6.3_leak_corr_kmax=5Mpc_C_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 2048.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_C_cell.ini deleted file mode 100755 index 3382193d..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_large_scales_C_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_large_scales_C -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_large_scales_C/cosmosis_SP_v1.4.6.3_leak_corr_large_scales_C.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_large_scales_C_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_large_scales_C/samples_SP_v1.4.6.3_leak_corr_large_scales_C_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 300.0 800.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_C_cell.ini b/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_C_cell.ini deleted file mode 100755 index 4bc8fe6d..00000000 --- a/cosmo_inference/cosmosis_config/harmonic_space_fiducial_C/cosmosis_pipeline_SP_v1.4.6.3_leak_corr_small_scales_C_cell.ini +++ /dev/null @@ -1,114 +0,0 @@ -#parameters used elsewhere in this file -[DEFAULT] -SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6.3_leak_corr_small_scales_C -FITS_FILE = /n17data/sguerrini/sp_validation/cosmo_inference/data/SP_v1.4.6.3_leak_corr_small_scales_C/cosmosis_SP_v1.4.6.3_leak_corr_small_scales_C.fits -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library - - -[pipeline] -values = cosmosis_config/values_ia.ini -priors = cosmosis_config/priors.ini -modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like -likelihoods = 2pt_like -extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m -timing = T -debug = T - -[runtime] -sampler = polychord -verbosity = debug - -[test] -save_dir = %(SCRATCH)s/best_fit/ - - -[polychord] -polychord_outfile_root = SP_v1.4.6.3_leak_corr_small_scales_C_cell -live_points = 192 -feedback = 3 -resume = T -base_dir = %(SCRATCH)s/polychord - -[output] -filename = %(SCRATCH)s/SP_v1.4.6.3_leak_corr_small_scales_C/samples_SP_v1.4.6.3_leak_corr_small_scales_C_cell.txt -format = text -lock = F - -[consistency] -file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py -verbose = F - -[sample_S8] -file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py - -[camb] -file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py -mode=power -lmax=2508 -feedback=0 -do_reionization=F -kmin=1e-5 -kmax=20.0 -nk=200 -zmax=5.0 -zmax_background=5.0 -nz_background=500 -halofit_version=mead2020_feedback -nonlinear=pk -neutrino_hierarchy=normal -kmax_extrapolate = 500.0 - -[load_nz_fits] -file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py -nz_file =%(FITS_FILE)s -data_sets = SOURCE - -[photoz_bias] -file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py -mode = additive -sample = nz_source -bias_section = nofz_shifts -interpolation = cubic -output_deltaz_section_name = delta_z_out - -[linear_alignment] -file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py -method = bk_corrected - -[projection] -file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py -ell_min_logspaced = 1.0 -ell_max_logspaced = 25000.0 -n_ell_logspaced = 400 -shear-shear = source-source -shear-intrinsic = source-source -intrinsic-intrinsic = source-source -get_kernel_peaks = F -verbose = F - -[add_intrinsic] -file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py -shear-shear=T -position-shear=F -perbin=F - -[shear_m_bias] -file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py -m_per_bin = True -; Despite the parameter name, this can operate on xi as well as C_ell. -cl_section = shear_cl -verbose = F - -[2pt_shear] -file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so -corr_type = 0 ; shear_cl -> shear_xi - -[2pt_like] -file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE -data_file=%(FITS_FILE)s -gaussian_covariance=F -covmat_name=COVMAT -cut_zeros=F -like_name=2pt_like -angle_range_CELL_EE_1_1 = 800.0 1600.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/priors.ini b/cosmo_inference/cosmosis_config/priors.ini deleted file mode 100644 index e1e4a50e..00000000 --- a/cosmo_inference/cosmosis_config/priors.ini +++ /dev/null @@ -1,11 +0,0 @@ -[intrinsic_alignment_parameters] -A = gaussian 0.83 0.7 - -[cosmological_parameters] -ombh2 = gaussian 0.0244 0.00038 - -[shear_calibration_parameters] -m1 = gaussian -0.057 0.014 - -[nofz_shifts] -bias_1 = gaussian -0.030 0.018 diff --git a/cosmo_inference/cosmosis_config/priors_mock.ini b/cosmo_inference/cosmosis_config/priors_mock.ini deleted file mode 100644 index 5a42d2bb..00000000 --- a/cosmo_inference/cosmosis_config/priors_mock.ini +++ /dev/null @@ -1,15 +0,0 @@ -[psf_leakage_parameters] -alpha = gaussian 0.0 0.022 -beta = gaussian 0.0 0.1148 - -[intrinsic_alignment_parameters] -A = gaussian 0.0 0.7 - -[cosmological_parameters] -ombh2 = gaussian 0.0244 0.00038 - -[shear_calibration_parameters] -m1 = gaussian 0.0 0.01 - -[nofz_shifts] -bias_1 = gaussian 0.0 0.018 diff --git a/cosmo_inference/cosmosis_config/priors_mock_cell.ini b/cosmo_inference/cosmosis_config/priors_mock_cell.ini deleted file mode 100644 index 6381417d..00000000 --- a/cosmo_inference/cosmosis_config/priors_mock_cell.ini +++ /dev/null @@ -1,11 +0,0 @@ -[intrinsic_alignment_parameters] -A = gaussian 0.0 0.7 - -[cosmological_parameters] -ombh2 = gaussian 0.0244 0.00038 - -[shear_calibration_parameters] -m1 = gaussian 0.0 0.01 - -[nofz_shifts] -bias_1 = gaussian 0.0 0.018 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/priors_mock_cell_no_sys.ini b/cosmo_inference/cosmosis_config/priors_mock_cell_no_sys.ini deleted file mode 100644 index 781ae8f7..00000000 --- a/cosmo_inference/cosmosis_config/priors_mock_cell_no_sys.ini +++ /dev/null @@ -1,5 +0,0 @@ -[cosmological_parameters] -ombh2 = gaussian 0.0244 0.00038 - -[nofz_shifts] -bias_1 = gaussian 0.0 0.013 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/priors_psf.ini b/cosmo_inference/cosmosis_config/priors_psf.ini deleted file mode 100644 index 37b0e029..00000000 --- a/cosmo_inference/cosmosis_config/priors_psf.ini +++ /dev/null @@ -1,15 +0,0 @@ -[psf_leakage_parameters] -alpha = gaussian 0.0051 0.022 -beta = gaussian 0.8098 0.1148 - -[intrinsic_alignment_parameters] -A = gaussian 0.83 0.7 - -[cosmological_parameters] -ombh2 = gaussian 0.0244 0.00038 - -[shear_calibration_parameters] -m1 = gaussian -0.0057 0.014 - -[nofz_shifts] -bias_1 = gaussian -0.030 0.018 diff --git a/cosmo_inference/cosmosis_config/values.ini b/cosmo_inference/cosmosis_config/values.ini deleted file mode 100644 index 39d1ec22..00000000 --- a/cosmo_inference/cosmosis_config/values.ini +++ /dev/null @@ -1,27 +0,0 @@ -[cosmological_parameters] -tau = 0.0544 -w = -1.0 -mnu = 0.06 -omega_k = 0.0 -wa = 0.0 -omch2 = 0.10496564589028712 -h0 = 0.7703672811295145 -ombh2 = 0.024364520846452055 -n_s = 1.0378056660322685 -s_8_input = 0.867063981537897 - -[halo_model_parameters] - -[intrinsic_alignment_parameters] -a = 1.2105355520872163 - -[shear_calibration_parameters] -m1 = -0.0019669717507113187 - -[nofz_shifts] -bias_1 = -0.05240296008081707 - -[psf_leakage_parameters] -alpha = 0.017313482956287624 -beta = 1.0863942321436328 - diff --git a/cosmo_inference/cosmosis_config/values_empty.ini b/cosmo_inference/cosmosis_config/values_empty.ini deleted file mode 100644 index 019ede84..00000000 --- a/cosmo_inference/cosmosis_config/values_empty.ini +++ /dev/null @@ -1,27 +0,0 @@ -[cosmological_parameters] -omch2 = 0.12565412726665712 -ombh2 = 0.022190866236551653 -h0 = 0.7030358726770478 -n_s = 0.9289664070330077 -tau = 0.11917908882774889 -s_8_input = 0.8305214945570943 - -[halo_model_parameters] -logt_agn = 7.557408270897992 - -[intrinsic_alignment_parameters] -a = 1.1275705902391073 - -[shear_calibration_parameters] -m1 = -0.055917714943670135 - -[nofz_shifts] -bias_1 = -0.0043115110715487015 - -[psf_leakage_parameters] -alpha = 0.004915858299931063 -beta = 0.811523998498723 - -[planck] -a_planck = 0.9998087705267193 - diff --git a/cosmo_inference/cosmosis_config/values_ia.ini b/cosmo_inference/cosmosis_config/values_ia.ini deleted file mode 100644 index 33389111..00000000 --- a/cosmo_inference/cosmosis_config/values_ia.ini +++ /dev/null @@ -1,26 +0,0 @@ -[cosmological_parameters] -omch2 = 0.051 0.120 0.255 -h0 = 0.64 0.7 0.82 -ombh2 = 0.019 0.023 0.026 -n_s = 0.84 0.96 1.1 -S_8_input = 0.1 0.8 1.3 - -tau = 0.0544 -w = -1.0 -mnu = 0.06 -omega_k = 0.0 -wa = 0.0 - -[halo_model_parameters] -logT_AGN = 7.3 7.5 8.0 - -[intrinsic_alignment_parameters] -A = -5.0 1.0 5.0 - -[shear_calibration_parameters] -m1 = -0.1 0.0 0.1 - -[nofz_shifts] -bias_1 = -0.1 0.0 0.1 - - diff --git a/cosmo_inference/cosmosis_config/values_ia_no_sys.ini b/cosmo_inference/cosmosis_config/values_ia_no_sys.ini deleted file mode 100644 index da390032..00000000 --- a/cosmo_inference/cosmosis_config/values_ia_no_sys.ini +++ /dev/null @@ -1,26 +0,0 @@ -[cosmological_parameters] -omch2 = 0.051 0.120 0.255 -h0 = 0.64 0.7 0.82 -ombh2 = 0.019 0.023 0.026 -n_s = 0.84 0.96 1.1 -S_8_input = 0.1 0.8 1.3 - -tau = 0.0544 -w = -1.0 -mnu = 0.06 -omega_k = 0.0 -wa = 0.0 - -[halo_model_parameters] -logT_AGN = 7.3 7.5 8.0 - -[intrinsic_alignment_parameters] -A = 0.0 - -[shear_calibration_parameters] -m1 = 0.0 - -[nofz_shifts] -bias_1 = -0.1 0.0 0.1 - - diff --git a/cosmo_inference/cosmosis_config/values_ia_test.ini b/cosmo_inference/cosmosis_config/values_ia_test.ini deleted file mode 100644 index 30b2c422..00000000 --- a/cosmo_inference/cosmosis_config/values_ia_test.ini +++ /dev/null @@ -1,27 +0,0 @@ -[cosmological_parameters] -omch2 = 0.051 0.11869577244577488 0.255 -h0 = 0.64 0.6766 0.82 -ombh2 = 0.019 0.0224178568132 0.026 -n_s = 0.84 0.9665 1.1 -#S_8_input = 0.1 0.81 1.3 -S_8_input = 0.1 0.8231408713507062 1.3 - -tau = 0.054 -w = -1.0 -mnu = 0.06 -omega_k = 0.0 -wa = 0.0 - -[halo_model_parameters] -logT_AGN = 7.3 7.8 8.0 - -[intrinsic_alignment_parameters] -A = -5.0 0.0 5.0 - -[shear_calibration_parameters] -m1 = -0.1 0.0 0.1 - -[nofz_shifts] -bias_1 = -0.1 0.0 0.1 - - diff --git a/cosmo_inference/cosmosis_config/values_psf.ini b/cosmo_inference/cosmosis_config/values_psf.ini deleted file mode 100644 index 7826eaf7..00000000 --- a/cosmo_inference/cosmosis_config/values_psf.ini +++ /dev/null @@ -1,30 +0,0 @@ -[cosmological_parameters] -#omega_m = 0.05 0.25 0.6 -omch2 = 0.051 0.12249999999999998 0.255 -h0 = 0.64 0.70 0.82 -ombh2 = 0.019 0.024499999999999997 0.026 -n_s = 0.84 0.96 1.1 -S_8_input = 0.1 0.79563645 1.3 - -tau = 0.0544 -w = -1.0 -mnu = 0.06 -omega_k = 0.0 -wa = 0.0 - -[halo_model_parameters] -logT_AGN = 7.3 7.4755796676459387 8.0 - -[intrinsic_alignment_parameters] -A = -5.0 0 5.0 - -[shear_calibration_parameters] -m1 = -0.1 0.0 0.1 - -[nofz_shifts] -bias_1 = -0.1 0.0 0.1 - -[psf_leakage_parameters] -alpha = -0.1 0.0 0.1 -beta = -2.0 0.0 2.0 - diff --git a/cosmo_inference/cosmosis_config/values_template.ini b/cosmo_inference/cosmosis_config/values_template.ini deleted file mode 100644 index 60ac60c6..00000000 --- a/cosmo_inference/cosmosis_config/values_template.ini +++ /dev/null @@ -1,23 +0,0 @@ -[cosmological_parameters] -omch2 = 0.01 0.12 0.3 -h0 = 0.55 0.7 0.91 -ombh2 = 0.01 0.023 0.07 -n_s = 0.87 0.96 1.07 -a_s = 0.5e-09 2.9e-09 5.0e-09 - -tau = 0.0544 -w = -1.0 -mnu = 0.06 -omega_k = 0.0 -wa = 0.0 - -[halo_model_parameters] -logt_agn = 6.5 7.81 8.5 - -[nofz_shifts] -bias_1 = -2.0 0.0 2.0 - -[intrinsic_alignment_parameters] -A = -3.0 0.0 3.0 - - diff --git a/cosmo_inference/get_chi2.ipynb b/cosmo_inference/get_chi2.ipynb deleted file mode 100644 index 69d026a2..00000000 --- a/cosmo_inference/get_chi2.ipynb +++ /dev/null @@ -1,1391 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import configparser\n", - "import os\n", - "import subprocess\n", - "\n", - "import healpy as hp\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import scipy.stats as stats\n", - "import seaborn as sns\n", - "from astropy.io import fits\n", - "from getdist import plots\n", - "from IPython.display import Markdown, display\n", - "from scipy.interpolate import interp1d\n", - "\n", - "%matplotlib inline\n", - "# import uncertainties\n", - "\n", - "# Use paper style and seaborn with husl palette\n", - "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", - "# Set default palette - will be updated per plot as needed\n", - "sns.set_palette(\"husl\")\n", - "\n", - "g = plots.get_subplot_plotter(width_inch=30)\n", - "g.settings.axes_fontsize = 30\n", - "g.settings.axes_labelsize = 30\n", - "g.settings.alpha_filled_add = 0.7\n", - "g.settings.legend_fontsize = 40\n", - "\n", - "\n", - "# SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE\n", - "root_dir = \"/n09data/guerrini/output_chains/\"\n", - "\n", - "catalog_version = \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1\"\n", - "\n", - "path_ini_files = \"/home/guerrini/sp_validation/cosmo_inference/cosmosis_config/\"\n", - "\n", - "roots = [\n", - " f\"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_{int(i)}.0_80.0_10.0_80.0\"\n", - " for i in [3, 5, 7, 10, 11]\n", - "]\n", - "\n", - "\"\"\" roots = [\n", - " \"SP_v1.4.5_A\",\n", - " #\"SP_v1.4.5_A_no_IA\",\n", - " #\"SP_v1.4.5_A_no_dz\",\n", - " #\"SP_v1.4.5_A_no_m_bias\",\n", - " \"SP_v1.4.5_A_sc_3_150\",\n", - " \"SP_v1.4.5_A_sc_3_60\",\n", - " \"SP_v1.4.5_A_sc_10_150\",\n", - " \"SP_v1.4.5_A_sc_10_60\",\n", - " \"SP_v1.4.5_A_sc_5_150\",\n", - " \"SP_v1.4.5_A_sc_7_150\",\n", - " #\"SP_v1.4.5_A_no_leakage\"\n", - "] \"\"\"\n", - "\n", - "\"\"\" roots = [\n", - " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0\",\n", - " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0_no_alpha_beta\"\n", - "] \"\"\"\n", - "\n", - "\n", - "properties = {}\n", - "\n", - "for root in roots:\n", - " config = configparser.ConfigParser()\n", - " config.optionxform = str # Preserve case sensitivity of option names\n", - " config.read(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\")\n", - "\n", - " add_xi_sys = config[\"2pt_like\"][\"add_xi_sys\"]\n", - " add_xi_sys = add_xi_sys == \"T\"\n", - " lower_bound_xi_plus, upper_bound_xi_plus = map(\n", - " float, config[\"2pt_like\"][\"angle_range_XI_PLUS_1_1\"].split()\n", - " )\n", - " lower_bound_xi_minus, upper_bound_xi_minus = map(\n", - " float, config[\"2pt_like\"][\"angle_range_XI_MINUS_1_1\"].split()\n", - " )\n", - "\n", - " properties[root] = {\n", - " \"add_xi_sys\": add_xi_sys,\n", - " \"lower_bound_xi_plus\": lower_bound_xi_plus,\n", - " \"upper_bound_xi_plus\": upper_bound_xi_plus,\n", - " \"lower_bound_xi_minus\": lower_bound_xi_minus,\n", - " \"upper_bound_xi_minus\": upper_bound_xi_minus,\n", - " }\n", - "\n", - "\n", - "print(roots)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Retrieve the chains" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# MAKE PARAMNAMES FILE\n", - "\n", - "for root in roots:\n", - " with open(root_dir + \"{}/samples_{}.txt\".format(\"/\" + root, root), \"r\") as file:\n", - " params = file.readline()[1:].split(\"\\t\")[:-4]\n", - " file.close()\n", - "\n", - " with open(\n", - " root_dir + \"{}/getdist_{}.paramnames\".format(\"/\" + root, root), \"w\"\n", - " ) as file:\n", - " for i in range(len(params)):\n", - " if len(params[i].split(\"--\")) > 1:\n", - " file.write(params[i].split(\"--\")[1] + \"\\n\")\n", - " else:\n", - " file.write(params[i].split(\"--\")[0] + \"\\n\")\n", - " file.close()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# READ CHAIN\n", - "\n", - "chains = []\n", - "\n", - "for root in roots:\n", - " samples = np.loadtxt(root_dir + \"{}/samples_{}.txt\".format(root, root))\n", - " print(len(samples))\n", - " if \"nautilus\" in root:\n", - " samples = np.column_stack(\n", - " (np.exp(samples[:, -3]), samples[:, -1] - samples[:, -2], samples[:, 0:-3])\n", - " )\n", - " else:\n", - " samples = np.column_stack((samples[:, -1], samples[:, -3], samples[:, 0:-4]))\n", - " np.savetxt(root_dir + \"{}/getdist_{}.txt\".format(root, root), samples)\n", - "\n", - " chain = g.samples_for_root(\n", - " root_dir + \"{}/getdist_{}\".format(root, root),\n", - " cache=False,\n", - " settings={\"ignore_rows\": 0, \"smooth_scale_2D\": 0.3, \"smooth_scale_1D\": 0.3},\n", - " )\n", - "\n", - " chains.append(chain)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "name_list = [\n", - " \"OMEGA_M\",\n", - " \"ombh2\",\n", - " \"h0\",\n", - " \"n_s\",\n", - " \"SIGMA_8\",\n", - " \"s_8_input\",\n", - " \"logt_agn\",\n", - " \"a\",\n", - " \"m1\",\n", - " \"bias_1\",\n", - " \"alpha\",\n", - " \"beta\",\n", - "]\n", - "label_list = [\n", - " r\"\\Omega_m\",\n", - " r\"\\omega_b h^2\",\n", - " \"h_0\",\n", - " \"n_s\",\n", - " r\"\\sigma_8\",\n", - " \"S_8\",\n", - " \"log T_{AGN}\",\n", - " \"A_{IA}\",\n", - " \"m_1\",\n", - " r\"\\Delta z_1\",\n", - " \"\\\\alpha_{PSF}\",\n", - " \"\\\\beta_{PSF}\",\n", - "]\n", - "\n", - "for chain in chains:\n", - " param_names = chain.getParamNames()\n", - " for name, label in zip(name_list, label_list):\n", - " param_names.parWithName(name).label = label" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Extract the best fit parameters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "best_fit = {}\n", - "\n", - "for root, chain in zip(roots, chains):\n", - " print(root)\n", - " likestats = chain.getLikeStats()\n", - " bestfit_idx = np.argmax(chain.loglikes)\n", - " maxlike = chain.loglikes[bestfit_idx]\n", - " print(f\"Maximum Likelihood: {maxlike:.5g}\")\n", - " best_fit[root] = {\"likelihood\": maxlike}\n", - " for i, par in enumerate(likestats.names):\n", - " best_fit[root].update(\n", - " {par.name: np.average(chain.samples[:, i], weights=chain.weights)}\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run `Cosmosis` in test mode to get the data vectors" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if not os.path.exists(path_ini_files + \"/values_empty.ini\"):\n", - " content = \"\"\"[cosmological_parameters]\n", - "\n", - "tau = 0.0544\n", - "w = -1.0\n", - "massive_nu = 1\n", - "massless_nu = 2.046\n", - "omega_k = 0.0\n", - "wa = 0.0\n", - "\n", - "[halo_model_parameters]\n", - "\n", - "[intrinsic_alignment_parameters]\n", - "\n", - "[shear_calibration_parameters]\n", - "\n", - "[nofz_shifts]\n", - "\n", - "[psf_leakage_parameters]\n", - "\"\"\"\n", - "\n", - " with open(path_ini_files + \"/values_empty.ini\", \"w\") as f:\n", - " f.write(content)\n", - " f.close()\n", - "\n", - " print(\"File created successfully\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "section_map = {\n", - " \"omch2\": \"cosmological_parameters\",\n", - " \"ombh2\": \"cosmological_parameters\",\n", - " \"h0\": \"cosmological_parameters\",\n", - " \"n_s\": \"cosmological_parameters\",\n", - " \"s_8_input\": \"cosmological_parameters\",\n", - " \"logt_agn\": \"halo_model_parameters\",\n", - " \"a\": \"intrinsic_alignment_parameters\",\n", - " \"m1\": \"shear_calibration_parameters\",\n", - " \"bias_1\": \"nofz_shifts\",\n", - " \"alpha\": \"psf_leakage_parameters\",\n", - " \"beta\": \"psf_leakage_parameters\",\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "env = os.environ.copy()\n", - "env[\"LD_LIBRARY_PATH\"] = (\n", - " \"/home/guerrini/.conda/envs/sp_validation/lib/python3.9/site-packages/cosmosis/datablock:\"\n", - " + env.get(\"LD_LIBRARY_PATH\", \"\")\n", - ")\n", - "\n", - "for root in roots:\n", - " print(root)\n", - " config = configparser.ConfigParser()\n", - " config.optionxform = str # Preserve case sensitivity of option names\n", - " config.read(path_ini_files + \"/values_empty.ini\")\n", - " for param, value in best_fit[root].items():\n", - " section = section_map.get(param)\n", - " if section is None:\n", - " continue\n", - " if section not in config:\n", - " config.add_section(section)\n", - " config[section][param] = str(value)\n", - "\n", - " with open(path_ini_files + \"/values_empty.ini\", \"w\") as configfile:\n", - " config.write(configfile)\n", - "\n", - " # Modify the ini file to run in test mode at the best fit\n", - " config = configparser.ConfigParser()\n", - " config.optionxform = str # Preserve case sensitivity of option names\n", - " config.read(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\")\n", - "\n", - " sampler = config[\"runtime\"][\"sampler\"]\n", - " config[\"runtime\"][\"sampler\"] = \"test\"\n", - " values = config[\"pipeline\"][\"values\"]\n", - " config[\"pipeline\"][\"values\"] = path_ini_files + \"/values_empty.ini\"\n", - "\n", - " with open(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\", \"w\") as configfile:\n", - " config.write(configfile)\n", - "\n", - " # Run cosmosis\n", - " result = subprocess.run(\n", - " [\"cosmosis\", \"cosmosis_config/cosmosis_pipeline_{}.ini\".format(root)],\n", - " env=env,\n", - " capture_output=True,\n", - " text=True,\n", - " )\n", - " print(f\"STDOUT:\\n{result.stdout}\")\n", - " print(f\"STDERR:\\n{result.stderr}\")\n", - "\n", - " # Modify the ini file to the previous one\n", - " config[\"pipeline\"][\"values\"] = values\n", - " config[\"runtime\"][\"sampler\"] = sampler\n", - "\n", - " with open(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\", \"w\") as configfile:\n", - " config.write(configfile)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Compute the $\\chi^2$" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "output_folder = \"/n09data/guerrini/output_chains/\"\n", - "\n", - "metrics = {}\n", - "\n", - "for root in roots:\n", - " print(root)\n", - "\n", - " add_xi_sys = properties[root][\"add_xi_sys\"]\n", - " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", - " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", - " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", - " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", - "\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " shear_xi_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " shear_xi_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_sys_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - "\n", - " # Read model tau_stats\n", - " theta_tau = np.loadtxt(\n", - " output_folder + \"best_fit/{}/tau_0_plus/theta.txt\".format(root)\n", - " )\n", - " theta_tau_arcmin = theta_tau * 180 * 60 / np.pi\n", - " tau_0_model = np.loadtxt(\n", - " output_folder + \"best_fit/{}/tau_0_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " tau_2_model = np.loadtxt(\n", - " output_folder + \"best_fit/{}/tau_2_plus/bin_1_1.txt\".format(root)\n", - " )\n", - "\n", - " # Read the data\n", - " data = fits.open(f\"data/{catalog_version}/cosmosis_{catalog_version}.fits\")\n", - "\n", - " theta_data = data[\"XI_PLUS\"].data[\"ANG\"]\n", - " xi_plus_data = data[\"XI_PLUS\"].data[\"VALUE\"]\n", - " xi_minus_data = data[\"XI_MINUS\"].data[\"VALUE\"]\n", - " tau_0_data = data[\"TAU_0_PLUS\"].data[\"VALUE\"]\n", - " tau_2_data = data[\"TAU_2_PLUS\"].data[\"VALUE\"]\n", - "\n", - " # Load the covariance\n", - " cov = data[\"COVMAT\"].data\n", - " cov_xi = cov[0 : 2 * len(xi_plus_data), 0 : 2 * len(xi_plus_data)]\n", - " cov_tau = cov[2 * len(xi_plus_data) :, 2 * len(xi_plus_data) :]\n", - "\n", - " # interpolate the model\n", - " interp_xi_plus = interp1d(\n", - " theta_arcmin, shear_xi_plus, kind=\"cubic\", fill_value=\"extrapolate\"\n", - " )\n", - " interp_xi_minus = interp1d(\n", - " theta_arcmin, shear_xi_minus, kind=\"cubic\", fill_value=\"extrapolate\"\n", - " )\n", - "\n", - " xi_plus_model = interp_xi_plus(theta_data)\n", - " if add_xi_sys:\n", - " xi_plus_model += xi_sys_plus\n", - " xi_minus_model = interp_xi_minus(theta_data)\n", - " if add_xi_sys:\n", - " xi_minus_model += xi_sys_minus\n", - "\n", - " # Concatenate the data vector\n", - " xi_data = np.concatenate((xi_plus_data, xi_minus_data))\n", - " xi_model = np.concatenate((xi_plus_model, xi_minus_model))\n", - "\n", - " tau_data = np.concatenate((tau_0_data, tau_2_data))\n", - " tau_model = np.concatenate((tau_0_model, tau_2_model))\n", - "\n", - " # Apply scale cuts\n", - " mask_xi_plus = (theta_data > lower_bound_xi_plus) & (\n", - " theta_data < upper_bound_xi_plus\n", - " )\n", - " mask_xi_minus = (theta_data > lower_bound_xi_minus) & (\n", - " theta_data < upper_bound_xi_minus\n", - " )\n", - " mask = np.concatenate((mask_xi_plus, mask_xi_minus))\n", - "\n", - " xi_data = xi_data[mask]\n", - " xi_model = xi_model[mask]\n", - " cov_xi = cov_xi[mask][:, mask]\n", - "\n", - " xi_plus_chi2 = np.dot(\n", - " (xi_model - xi_data), np.dot(np.linalg.inv(cov_xi), (xi_model - xi_data))\n", - " )\n", - " tau_chi2 = np.dot(\n", - " (tau_model - tau_data), np.dot(np.linalg.inv(cov_tau), (tau_model - tau_data))\n", - " )\n", - " n_dof_xi = np.sum(mask)\n", - " n_dof_tau = len(tau_0_data) + len(tau_2_data)\n", - " p_value_xi = 1 - stats.chi2.cdf(xi_plus_chi2, n_dof_xi)\n", - " p_value_tau = 1 - stats.chi2.cdf(tau_chi2, n_dof_tau)\n", - " chi2_tot = xi_plus_chi2 + tau_chi2\n", - " n_dof_tot = n_dof_xi + n_dof_tau\n", - " p_value_tot = 1 - stats.chi2.cdf(chi2_tot, n_dof_tot)\n", - "\n", - " metrics[root] = {\n", - " \"chi2_xi\": xi_plus_chi2,\n", - " \"n_dof_xi\": n_dof_xi,\n", - " \"p_value_xi\": p_value_xi,\n", - " \"chi2_tau\": tau_chi2,\n", - " \"n_dof_tau\": n_dof_tau,\n", - " \"p_value_tau\": p_value_tau,\n", - " \"chi2_tot\": chi2_tot,\n", - " \"n_dof_tot\": n_dof_tot,\n", - " \"p_value_tot\": p_value_tot,\n", - " }\n", - " print(\"Done!\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def get_latex_table(metrics):\n", - " latex_lines = [\n", - " r\"\\begin{tabular}{lccc|ccc|ccc}\",\n", - " r\"\\hline\",\n", - " r\"Root & $\\chi^2_{\\xi^+}$/dof & $p_{\\xi^+}$ & \"\n", - " r\"$\\chi^2_\\tau$/dof & $p_\\tau$ & $\\chi^2_{\\text{tot}}$/dof & $p_{\\text{tot}}$ \\\\\",\n", - " r\"\\hline\",\n", - " ]\n", - "\n", - " for root, vals in metrics.items():\n", - " escaped = root.replace(\"_\", r\"\\_\")\n", - " line = (\n", - " f\"{escaped} & \"\n", - " f\"{vals['chi2_xi']:.2f}/{vals['n_dof_xi']} & {vals['p_value_xi']:.5f} & \"\n", - " f\"{vals['chi2_tau']:.2f}/{vals['n_dof_tau']} & {vals['p_value_tau']:.5f} & \"\n", - " f\"{vals['chi2_tot']:.2f}/{vals['n_dof_tot']} & {vals['p_value_tot']:.5f} \\\\\\\\\"\n", - " )\n", - " latex_lines.append(line)\n", - "\n", - " latex_lines.append(r\"\\hline\")\n", - " latex_lines.append(r\"\\end{tabular}\")\n", - "\n", - " # Print LaTeX table\n", - " print(\"\\n\".join(latex_lines))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "get_latex_table(metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def display_markdown(metrics):\n", - " # Build Markdown table\n", - " header = (\n", - " \"| Root | $\\\\chi^2$ (ξ⁺) / dof | p-val (ξ⁺) | $\\\\chi^2$ (τ) / dof | p-val (τ) | $\\\\chi^2$ (tot) / dof | p-val (tot) |\\n\"\n", - " \"|------|----------------|------------|---------------|------------|------------------|--------------|\\n\"\n", - " )\n", - "\n", - " rows = []\n", - " for root, vals in metrics.items():\n", - " row = f\"| `{root}` \"\n", - " row += f\"| {vals['chi2_xi']:.2f} / {vals['n_dof_xi']} \"\n", - " row += f\"| {vals['p_value_xi']:.5f} \"\n", - " row += f\"| {vals['chi2_tau']:.2f} / {vals['n_dof_tau']} \"\n", - " row += f\"| {vals['p_value_tau']:.5f} \"\n", - " row += f\"| {vals['chi2_tot']:.2f} / {vals['n_dof_tot']} \"\n", - " row += f\"| {vals['p_value_tot']:.5f} |\"\n", - " rows.append(row)\n", - "\n", - " # Display in Jupyter\n", - " display(Markdown(header + \"\\n\".join(rows)))\n", - " return header + \"\\n\".join(rows)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "markdown_source = display_markdown(metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "markdown_source" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Plot the best-fit of each model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "data = fits.open(\n", - " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_version}/cosmosis_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.fits\"\n", - ")\n", - "xi_plus = data[\"XI_PLUS\"].data\n", - "xi_minus = data[\"XI_MINUS\"].data\n", - "cov_mat = data[\"COVMAT\"].data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.figure(figsize=(15, 15))\n", - "\n", - "plt.subplot(211)\n", - "\n", - "plt.errorbar(\n", - " xi_plus[\"ANG\"],\n", - " xi_plus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - ")\n", - "\n", - "plt.ylabel(r\"$\\xi_{+}$\", fontsize=26)\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "\n", - "plt.subplot(212)\n", - "\n", - "plt.errorbar(\n", - " xi_minus[\"ANG\"],\n", - " xi_minus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - ")\n", - "\n", - "plt.xlabel(r\"$\\theta$ [arcmin]\", fontsize=26)\n", - "plt.ylabel(r\"$\\xi_{-}$\", fontsize=26)\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.legend(fontsize=15)\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def plot_best_fit(\n", - " root_to_plot,\n", - " colours,\n", - " savefile,\n", - " theta_min=1.0,\n", - " theta_max=250.0,\n", - " multiply_theta=False,\n", - " plot_xi_sys=True,\n", - "):\n", - " data = fits.open(\n", - " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_version}/cosmosis_{catalog_version}.fits\"\n", - " )\n", - " xi_plus = data[\"XI_PLUS\"].data\n", - " xi_minus = data[\"XI_MINUS\"].data\n", - " cov_mat = data[\"COVMAT\"].data\n", - "\n", - " plt.figure(figsize=(15, 15))\n", - "\n", - " plt.subplot(211)\n", - "\n", - " y_plot_xi_plus = (\n", - " xi_plus[\"VALUE\"] if not multiply_theta else xi_plus[\"ANG\"] * xi_plus[\"VALUE\"]\n", - " )\n", - " y_errorbar = (\n", - " xi_plus[\"ANG\"] * np.sqrt(np.diag(cov_mat))[:20]\n", - " if multiply_theta\n", - " else np.sqrt(np.diag(cov_mat))[:20]\n", - " )\n", - " plt.errorbar(\n", - " xi_plus[\"ANG\"],\n", - " y_plot_xi_plus,\n", - " yerr=y_errorbar,\n", - " fmt=\"o\",\n", - " label=f\"{catalog_version} data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - " )\n", - "\n", - " for root, color in zip(root_to_plot, colours):\n", - " add_xi_sys = properties[root][\"add_xi_sys\"]\n", - " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", - " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", - " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", - " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", - "\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " shear_xi_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " shear_xi_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_sys_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - " theta_xi_sys = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", - " )\n", - " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", - "\n", - " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", - " xi_plus_model = shear_xi_plus[mask]\n", - " if add_xi_sys:\n", - " xi_plus_model += np.interp(\n", - " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_plus\n", - " )\n", - "\n", - " y_plot = theta_arcmin[mask] * xi_plus_model if multiply_theta else xi_plus_model\n", - " plt.plot(theta_arcmin[mask], y_plot, color=color, label=root, alpha=0.5)\n", - " if plot_xi_sys and add_xi_sys:\n", - " y_plot_xi_sys = (\n", - " theta_xi_sys_arcmin * xi_sys_plus if multiply_theta else xi_sys_plus\n", - " )\n", - " plt.plot(\n", - " theta_xi_sys_arcmin,\n", - " y_plot_xi_sys,\n", - " color=color,\n", - " linestyle=\"-.\",\n", - " alpha=0.5,\n", - " )\n", - " plt.axvline(x=lower_bound_xi_plus, color=color, linestyle=\"--\", alpha=0.3)\n", - " plt.axvline(x=upper_bound_xi_plus, color=color, linestyle=\"--\", alpha=0.3)\n", - "\n", - " y_label = r\"$\\xi_{+}$\" if not multiply_theta else r\"$\\theta \\xi_{+}$\"\n", - " plt.ylabel(y_label, fontsize=26)\n", - " plt.xscale(\"log\")\n", - " plt.yscale(\"log\")\n", - " plt.legend(loc=\"lower left\", fontsize=8)\n", - "\n", - " plt.subplot(212)\n", - "\n", - " y_plot_xi_minus = (\n", - " xi_minus[\"VALUE\"] if not multiply_theta else xi_minus[\"ANG\"] * xi_minus[\"VALUE\"]\n", - " )\n", - " y_errorbar = (\n", - " xi_minus[\"ANG\"] * np.sqrt(np.diag(cov_mat))[20:40]\n", - " if multiply_theta\n", - " else np.sqrt(np.diag(cov_mat))[20:40]\n", - " )\n", - " plt.errorbar(\n", - " xi_minus[\"ANG\"],\n", - " y_plot_xi_minus,\n", - " yerr=y_errorbar,\n", - " fmt=\"o\",\n", - " label=f\"{catalog_version} data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - " )\n", - "\n", - " for root, color in zip(root_to_plot, colours):\n", - " add_xi_sys = properties[root][\"add_xi_sys\"]\n", - " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", - " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", - " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", - " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", - "\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " shear_xi_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " shear_xi_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_sys_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - " theta_xi_sys = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", - " )\n", - " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", - "\n", - " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", - " xi_minus_model = shear_xi_minus[mask]\n", - " if add_xi_sys:\n", - " xi_minus_model += np.interp(\n", - " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_minus\n", - " )\n", - "\n", - " y_plot = (\n", - " theta_arcmin[mask] * xi_minus_model if multiply_theta else xi_minus_model\n", - " )\n", - " plt.plot(theta_arcmin[mask], y_plot, color=color, label=root, alpha=0.5)\n", - " if plot_xi_sys and add_xi_sys:\n", - " y_plot_xi_sys = (\n", - " theta_xi_sys_arcmin * xi_sys_minus if multiply_theta else xi_sys_minus\n", - " )\n", - " plt.plot(\n", - " theta_xi_sys_arcmin,\n", - " y_plot_xi_sys,\n", - " color=color,\n", - " linestyle=\"-.\",\n", - " alpha=0.5,\n", - " )\n", - " plt.axvline(x=lower_bound_xi_minus, color=color, linestyle=\"--\", alpha=0.3)\n", - " plt.axvline(x=upper_bound_xi_minus, color=color, linestyle=\"--\", alpha=0.3)\n", - "\n", - " plt.xlabel(r\"$\\theta$ [arcmin]\", fontsize=26)\n", - " y_label = r\"$\\xi_{-}$\" if not multiply_theta else r\"$\\theta \\xi_{-}$\"\n", - " plt.ylabel(y_label, fontsize=26)\n", - " plt.xscale(\"log\")\n", - " plt.yscale(\"log\")\n", - " plt.legend(loc=\"lower left\", fontsize=8)\n", - "\n", - " if savefile is not None:\n", - " plt.savefig(savefile, bbox_inches=\"tight\")\n", - "\n", - " plt.show()\n", - "\n", - "\n", - "def plot_best_fit_ratio(\n", - " root_to_plot, colours, savefile, theta_min=1.0, theta_max=250.0\n", - "):\n", - " data = fits.open(\n", - " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_version}/cosmosis_{catalog_version}.fits\"\n", - " )\n", - " xi_plus = data[\"XI_PLUS\"].data\n", - " xi_minus = data[\"XI_MINUS\"].data\n", - " cov_mat = data[\"COVMAT\"].data\n", - "\n", - " plt.figure(figsize=(15, 15))\n", - "\n", - " plt.subplot(211)\n", - "\n", - " root = roots[0]\n", - " add_xi_sys = properties[root][\"add_xi_sys\"]\n", - " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", - " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", - " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", - " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", - "\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " shear_xi_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " shear_xi_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_sys_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - " theta_xi_sys = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", - " )\n", - " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", - "\n", - " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", - " xi_plus_model_fiducial = shear_xi_plus[mask]\n", - " if add_xi_sys:\n", - " xi_plus_model_fiducial += np.interp(\n", - " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_plus\n", - " )\n", - "\n", - " plt.errorbar(\n", - " xi_plus[\"ANG\"],\n", - " xi_plus[\"VALUE\"]\n", - " / np.interp(xi_plus[\"ANG\"], theta_arcmin[mask], xi_plus_model_fiducial),\n", - " yerr=np.sqrt(np.diag(cov_mat))[:20]\n", - " / np.abs(np.interp(xi_plus[\"ANG\"], theta_arcmin[mask], xi_plus_model_fiducial)),\n", - " fmt=\"o\",\n", - " label=f\"{catalog_version} data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - " )\n", - "\n", - " for root, color in zip(root_to_plot, colours):\n", - " add_xi_sys = properties[root][\"add_xi_sys\"]\n", - " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", - " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", - " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", - " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", - "\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " shear_xi_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " shear_xi_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_sys_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - " theta_xi_sys = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", - " )\n", - " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", - "\n", - " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", - " xi_plus_model = shear_xi_plus[mask]\n", - " if add_xi_sys:\n", - " xi_plus_model += np.interp(\n", - " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_plus\n", - " )\n", - "\n", - " alpha = 1.0 if root == roots[0] else 0.5\n", - " plt.plot(\n", - " theta_arcmin[mask],\n", - " xi_plus_model / xi_plus_model_fiducial,\n", - " color=color,\n", - " label=root,\n", - " alpha=alpha,\n", - " )\n", - " plt.axvline(x=lower_bound_xi_plus, color=color, linestyle=\"--\", alpha=0.3)\n", - " plt.axvline(x=upper_bound_xi_plus, color=color, linestyle=\"--\", alpha=0.3)\n", - "\n", - " plt.ylabel(r\"$\\xi_{+}/\\xi_{+, \\text{fid}}$\", fontsize=26)\n", - " plt.xscale(\"log\")\n", - " # plt.yscale('log')\n", - " plt.legend(loc=\"lower left\", fontsize=8)\n", - "\n", - " plt.subplot(212)\n", - "\n", - " root = roots[0]\n", - " add_xi_sys = properties[root][\"add_xi_sys\"]\n", - " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", - " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", - " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", - " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", - "\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " shear_xi_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " shear_xi_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_sys_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - " theta_xi_sys = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", - " )\n", - " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", - "\n", - " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", - " xi_minus_model_fiducial = shear_xi_minus[mask]\n", - " if add_xi_sys:\n", - " xi_minus_model_fiducial += np.interp(\n", - " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_minus\n", - " )\n", - "\n", - " plt.errorbar(\n", - " xi_minus[\"ANG\"],\n", - " xi_minus[\"VALUE\"]\n", - " / np.interp(xi_minus[\"ANG\"], theta_arcmin[mask], xi_minus_model_fiducial),\n", - " yerr=np.sqrt(np.diag(cov_mat))[20:40]\n", - " / np.abs(\n", - " np.interp(xi_minus[\"ANG\"], theta_arcmin[mask], xi_minus_model_fiducial)\n", - " ),\n", - " fmt=\"o\",\n", - " label=f\"{catalog_version} data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - " )\n", - "\n", - " for root, color in zip(root_to_plot, colours):\n", - " add_xi_sys = properties[root][\"add_xi_sys\"]\n", - " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", - " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", - " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", - " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", - "\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " shear_xi_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " shear_xi_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_sys_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - " theta_xi_sys = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", - " )\n", - " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", - "\n", - " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", - " xi_minus_model = shear_xi_minus[mask]\n", - " if add_xi_sys:\n", - " xi_minus_model += np.interp(\n", - " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_minus\n", - " )\n", - "\n", - " alpha = 1.0 if root == roots[0] else 0.5\n", - " plt.plot(\n", - " theta_arcmin[mask],\n", - " xi_minus_model / xi_minus_model_fiducial,\n", - " color=color,\n", - " label=root,\n", - " alpha=alpha,\n", - " )\n", - " plt.axvline(x=lower_bound_xi_minus, color=color, linestyle=\"--\", alpha=0.3)\n", - " plt.axvline(x=upper_bound_xi_minus, color=color, linestyle=\"--\", alpha=0.3)\n", - "\n", - " plt.xlabel(r\"$\\theta$ [arcmin]\", fontsize=26)\n", - " plt.ylabel(r\"$\\xi_{-}/\\xi_{-, \\text{fid}}$\", fontsize=26)\n", - " plt.xscale(\"log\")\n", - " plt.ylim(0, 2)\n", - " # plt.yscale('log')\n", - " plt.legend(loc=\"lower left\", fontsize=8)\n", - "\n", - " if savefile is not None:\n", - " plt.savefig(savefile, bbox_inches=\"tight\")\n", - "\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "root_to_plot = [\n", - " \"SP_v1.4.5_A\",\n", - " # \"SP_v1.4.5_A_no_IA\",\n", - " # \"SP_v1.4.5_A_no_dz\",\n", - " # \"SP_v1.4.5_A_no_m_bias\",\n", - " \"SP_v1.4.5_A_sc_3_150\",\n", - " \"SP_v1.4.5_A_sc_3_60\",\n", - " \"SP_v1.4.5_A_sc_10_150\",\n", - " \"SP_v1.4.5_A_sc_10_60\",\n", - " \"SP_v1.4.5_A_sc_5_150\",\n", - " \"SP_v1.4.5_A_sc_7_150\",\n", - " # \"SP_v1.4.5_A_no_leakage\"\n", - "]\n", - "\n", - "root_to_plot = [\n", - " f\"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_{int(i)}.0_80.0_10.0_80.0\"\n", - " for i in [3, 5, 7, 10, 11]\n", - "]\n", - "\n", - "\"\"\" root_to_plot = [\n", - " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0\",\n", - " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0_no_alpha_beta\"\n", - "] \"\"\"\n", - "\n", - "\n", - "colours = [\n", - " \"cornflowerblue\",\n", - " \"salmon\",\n", - " \"darkorange\",\n", - " \"forestgreen\",\n", - " \"turquoise\",\n", - " \"darkviolet\",\n", - " \"crimson\",\n", - " \"gold\",\n", - " \"lightcoral\",\n", - " \"mediumseagreen\",\n", - " \"lightsteelblue\",\n", - " \"black\",\n", - " \"silver\",\n", - " \"peru\",\n", - " \"maroon\",\n", - " \"olive\",\n", - "]\n", - "\n", - "savefile = None\n", - "\n", - "plot_best_fit(root_to_plot, colours, savefile, multiply_theta=True, plot_xi_sys=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "root_to_plot = [\n", - " \"SP_v1.4.5_A\",\n", - " # \"SP_v1.4.5_A_no_IA\",\n", - " # \"SP_v1.4.5_A_no_dz\",\n", - " # \"SP_v1.4.5_A_no_m_bias\",\n", - " \"SP_v1.4.5_A_sc_3_150\",\n", - " \"SP_v1.4.5_A_sc_3_60\",\n", - " \"SP_v1.4.5_A_sc_10_150\",\n", - " \"SP_v1.4.5_A_sc_10_60\",\n", - " \"SP_v1.4.5_A_sc_5_150\",\n", - " \"SP_v1.4.5_A_sc_7_150\",\n", - " # \"SP_v1.4.5_A_no_leakage\"\n", - "]\n", - "\n", - "\"\"\" root_to_plot = [\n", - " f\"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_{int(i)}.0_80.0_10.0_80.0\" for i in [3, 5, 7, 10, 11]\n", - "] \"\"\"\n", - "\n", - "root_to_plot = [\n", - " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0\",\n", - " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0_no_alpha_beta\",\n", - "]\n", - "\n", - "\n", - "colours = [\n", - " \"red\",\n", - " \"salmon\",\n", - " \"darkorange\",\n", - " \"forestgreen\",\n", - " \"turquoise\",\n", - " \"darkviolet\",\n", - " \"crimson\",\n", - " \"gold\",\n", - " \"lightcoral\",\n", - " \"mediumseagreen\",\n", - " \"lightsteelblue\",\n", - " \"black\",\n", - " \"silver\",\n", - " \"peru\",\n", - " \"maroon\",\n", - " \"olive\",\n", - "]\n", - "\n", - "savefile = \"best_fit_ratio_w_wo_leakage.png\"\n", - "\n", - "plot_best_fit_ratio(root_to_plot, colours, savefile)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def plot_best_fit_tau(root_to_plot, colours, savefile, theta_min=1.0, theta_max=250.0):\n", - " data = fits.open(\n", - " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_version}/cosmosis_{catalog_version}.fits\"\n", - " )\n", - " tau_0 = data[\"TAU_0_PLUS\"].data\n", - " tau_2 = data[\"TAU_2_PLUS\"].data\n", - " cov_mat = data[\"COVMAT\"].data\n", - "\n", - " plt.figure(figsize=(15, 15))\n", - "\n", - " plt.subplot(211)\n", - "\n", - " plt.errorbar(\n", - " tau_0[\"ANG\"],\n", - " tau_0[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[40:60],\n", - " fmt=\"o\",\n", - " label=f\"{catalog_version} data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - " )\n", - "\n", - " for root, color in zip(root_to_plot, colours):\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/tau_0_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " tau_0_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/tau_0_plus/bin_1_1.txt\".format(root)\n", - " )\n", - "\n", - " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", - "\n", - " plt.plot(\n", - " theta_arcmin[mask], tau_0_plus[mask], color=color, label=root, alpha=0.5\n", - " )\n", - "\n", - " plt.ylabel(r\"$\\tau_0$\", fontsize=26)\n", - " plt.xscale(\"log\")\n", - " # plt.yscale('log')\n", - " plt.legend(loc=\"upper right\", fontsize=8)\n", - "\n", - " plt.subplot(212)\n", - "\n", - " y_plot_tau_2 = tau_2[\"ANG\"] * tau_2[\"VALUE\"]\n", - " y_errorbar = tau_2[\"ANG\"] * np.sqrt(np.diag(cov_mat))[60:80]\n", - " plt.errorbar(\n", - " tau_2[\"ANG\"],\n", - " y_plot_tau_2,\n", - " yerr=y_errorbar,\n", - " fmt=\"o\",\n", - " label=f\"{catalog_version} data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - " )\n", - "\n", - " for root, color in zip(root_to_plot, colours):\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/tau_2_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " tau_2_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/tau_2_plus/bin_1_1.txt\".format(root)\n", - " )\n", - "\n", - " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", - "\n", - " plt.plot(\n", - " theta_arcmin[mask],\n", - " theta_arcmin[mask] * tau_2_plus[mask],\n", - " color=color,\n", - " label=root,\n", - " alpha=0.5,\n", - " )\n", - "\n", - " plt.xlabel(r\"$\\theta$ [arcmin]\", fontsize=26)\n", - " plt.ylabel(r\"$\\theta \\tau_2$\", fontsize=26)\n", - " plt.xscale(\"log\")\n", - " # plt.yscale('log')\n", - " plt.legend(loc=\"upper left\", fontsize=8)\n", - "\n", - " if savefile is not None:\n", - " plt.savefig(savefile, bbox_inches=\"tight\")\n", - "\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "root_to_plot = [\n", - " \"SP_v1.4.5_A\",\n", - " # \"SP_v1.4.5_A_no_IA\",\n", - " # \"SP_v1.4.5_A_no_dz\",\n", - " # \"SP_v1.4.5_A_no_m_bias\",\n", - " \"SP_v1.4.5_A_sc_3_150\",\n", - " \"SP_v1.4.5_A_sc_3_60\",\n", - " \"SP_v1.4.5_A_sc_10_150\",\n", - " \"SP_v1.4.5_A_sc_10_60\",\n", - " \"SP_v1.4.5_A_sc_5_150\",\n", - " \"SP_v1.4.5_A_sc_7_150\",\n", - " # \"SP_v1.4.5_A_no_leakage\"\n", - "]\n", - "\n", - "root_to_plot = [\n", - " f\"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_{int(i)}.0_80.0_10.0_80.0\"\n", - " for i in [3, 5, 7, 10, 11]\n", - "]\n", - "\n", - "colours = [\n", - " \"red\",\n", - " \"salmon\",\n", - " \"darkorange\",\n", - " \"forestgreen\",\n", - " \"turquoise\",\n", - " \"darkviolet\",\n", - " \"crimson\",\n", - " \"gold\",\n", - " \"lightcoral\",\n", - " \"mediumseagreen\",\n", - " \"lightsteelblue\",\n", - " \"black\",\n", - " \"silver\",\n", - " \"peru\",\n", - " \"maroon\",\n", - " \"olive\",\n", - "]\n", - "\n", - "savefile = \"best_fit_tau_new_binning.png\"\n", - "\n", - "plot_best_fit_tau(root_to_plot, colours, savefile)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pseudo_cell = fits.open(\n", - " \"/home/guerrini/sp_validation/cosmo_val/output/pseudo_cl_SP_v1.4.5.fits\"\n", - ")[1].data\n", - "cov_pseudo_cell = fits.open(\n", - " \"/home/guerrini/sp_validation/cosmo_val/output/pseudo_cl_cov_SP_v1.4.5.fits\"\n", - ")\n", - "\n", - "theory_ell = np.loadtxt(\n", - " \"/n09data/guerrini/output_chains/best_fit/SP_v1.4.5_A/shear_cl/ell.txt\"\n", - ")\n", - "theory_cell = np.loadtxt(\n", - " \"/n09data/guerrini/output_chains/best_fit/SP_v1.4.5_A/shear_cl/bin_1_1.txt\"\n", - ")\n", - "\n", - "pw = hp.pixwin(1024, lmax=2048)\n", - "\n", - "plt.errorbar(\n", - " pseudo_cell[\"ELL\"],\n", - " pseudo_cell[\"ELL\"] * pseudo_cell[\"EE\"],\n", - " yerr=pseudo_cell[\"ELL\"] * np.sqrt(np.diag(cov_pseudo_cell[\"COVAR_EE_EE\"].data)),\n", - " capsize=2,\n", - " c=\"k\",\n", - " fmt=\"o\",\n", - " markersize=2,\n", - ")\n", - "\n", - "mask = (theory_ell > 0.1) & (theory_ell < 2048)\n", - "plt.plot(\n", - " theory_ell[mask],\n", - " theory_ell[mask]\n", - " * theory_cell[mask]\n", - " * np.interp(theory_ell[mask], np.arange(0, 2049), pw) ** 2,\n", - " c=\"r\",\n", - " label=\"best-fit $\\\\theta \\\\in [3-200]$\",\n", - ")\n", - "\n", - "plt.xlabel(r\"$\\ell$\", fontsize=26)\n", - "plt.ylabel(r\"$\\ell C_\\ell^{EE}$\", fontsize=26)\n", - "plt.legend()\n", - "plt.savefig(\"SP_v1.4.5_A_cell.png\")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cov_pseudo_cell.info()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "sp_validation_3.11", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cosmo_inference/get_chi2_cell.ipynb b/cosmo_inference/get_chi2_cell.ipynb deleted file mode 100644 index c97748d1..00000000 --- a/cosmo_inference/get_chi2_cell.ipynb +++ /dev/null @@ -1,1527 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "# Trick to plot with tex\n", - "os.environ[\"LD_LIBRARY_PATH\"] = \"\"\n", - "os.environ[\"CONDA_PREFIX\"] = \"/home/guerrini/.conda/envs/sp_validation\"\n", - "\n", - "import configparser\n", - "import subprocess\n", - "\n", - "import healpy as hp\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib.scale as mscale\n", - "import matplotlib.ticker as ticker\n", - "import matplotlib.transforms as mtransforms\n", - "import numpy as np\n", - "import scipy.stats as stats\n", - "import seaborn as sns\n", - "from astropy.io import fits\n", - "from getdist import plots\n", - "from IPython.display import Markdown, display\n", - "from scipy.interpolate import interp1d\n", - "\n", - "plt.style.use(\"../papers/harmonic/matplotlib_config/paper.mplstyle\")\n", - "\n", - "plt.rcParams[\"text.usetex\"] = True\n", - "\n", - "sns.set_palette(\"husl\")\n", - "\n", - "\n", - "class SquareRootScale(mscale.ScaleBase):\n", - " \"\"\"\n", - " ScaleBase class for generating square root scale.\n", - "\n", - " Usage example: axis.set_yscale('squareroot')\n", - "\n", - " \"\"\"\n", - "\n", - " name = \"squareroot\"\n", - "\n", - " def __init__(self, axis, **kwargs):\n", - " mscale.ScaleBase.__init__(self, axis, **kwargs)\n", - "\n", - " def set_default_locators_and_formatters(self, axis):\n", - " axis.set_major_locator(ticker.AutoLocator())\n", - " axis.set_major_formatter(ticker.ScalarFormatter())\n", - " axis.set_minor_locator(ticker.NullLocator())\n", - " axis.set_minor_formatter(ticker.NullFormatter())\n", - "\n", - " def limit_range_for_scale(self, vmin, vmax, minpos):\n", - " return max(0.0, vmin), vmax\n", - "\n", - " class SquareRootTransform(mtransforms.Transform):\n", - " input_dims = 1\n", - " output_dims = 1\n", - " is_separable = True\n", - "\n", - " def transform_non_affine(self, a):\n", - " return np.array(a) ** 0.5\n", - "\n", - " def inverted(self):\n", - " return SquareRootScale.InvertedSquareRootTransform()\n", - "\n", - " class InvertedSquareRootTransform(mtransforms.Transform):\n", - " input_dims = 1\n", - " output_dims = 1\n", - " is_separable = True\n", - "\n", - " def transform(self, a):\n", - " return np.array(a) ** 2\n", - "\n", - " def inverted(self):\n", - " return SquareRootScale.SquareRootTransform()\n", - "\n", - " def get_transform(self):\n", - " return self.SquareRootTransform()\n", - "\n", - "\n", - "mscale.register_scale(SquareRootScale)\n", - "%matplotlib inline\n", - "# import uncertainties\n", - "\n", - "plt.rc(\"mathtext\", fontset=\"stix\")\n", - "plt.rc(\"font\", family=\"sans-serif\")\n", - "\n", - "g = plots.get_subplot_plotter(width_inch=30)\n", - "g.settings.axes_fontsize = 30\n", - "g.settings.axes_labelsize = 30\n", - "g.settings.alpha_filled_add = 0.7\n", - "g.settings.legend_fontsize = 40\n", - "\n", - "\n", - "# SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE\n", - "root_dir = \"/n09data/guerrini/output_chains/\"\n", - "\n", - "catalog_version = \"SP_v1.4.6_leak_corr_cell\"\n", - "catalog_version_real_space = \"SP_v1.4.6_leak_corr_A_10_80\"\n", - "\n", - "path_ini_files = \"/home/guerrini/sp_validation/cosmo_inference/cosmosis_config/\"\n", - "\n", - "roots = [\n", - " \"SP_v1.4.6_leak_corr_A_lmin=300_lmax=1600_cell\",\n", - " \"SP_v1.4.6_leak_corr_B_lmin=300_lmax=1600_cell\",\n", - " \"SP_v1.4.6_leak_corr_C_lmin=300_lmax=1600_cell\",\n", - " \"SP_v1.4.6_leak_corr_A_10_80\",\n", - " # f\"SP_v1.4.6_leak_corr_A_kmax=5Mpc_cell\",\n", - " \"SP_v1.4.6_leak_corr_A_kmax=3Mpc_cell\",\n", - " \"SP_v1.4.6_leak_corr_A_kmax=1Mpc_cell\",\n", - " \"SP_v1.4.6_leak_corr_A_include_large_scales_cell\",\n", - " \"SP_v1.4.6_leak_corr_A_small_scales_cell\",\n", - " \"SP_v1.4.6_leak_corr_A_large_scales_cell\",\n", - " \"SP_v1.4.6_leak_corr_A_halofit_cell\",\n", - " \"SP_v1.4.6_leak_corr_A_HMCode_nobar_cell\",\n", - " \"SP_v1.4.6_leak_corr_A_OneCov_cell\",\n", - " \"SP_v1.4.6_A_fid_cell\",\n", - "]\n", - "\n", - "labels = [\n", - " r\"UNIONS $C_\\ell$, Blind A\",\n", - " r\"UNIONS $C_\\ell$, Blind B\",\n", - " r\"UNIONS $C_\\ell$, Blind C\",\n", - " r\"UNIONS $\\xi_\\pm(\\vartheta)$, (Goh et al., 2026)\",\n", - " # rf\"$k_\\mathrm{{max}}=5 h$ Mpc$^{{-1}}$, $\\ell_\\mathrm{{max}}=2048$\",\n", - " r\"$k_\\mathrm{max}=3 h$ Mpc$^{-1}$, $\\ell_\\mathrm{max}=1800$\",\n", - " r\"$k_\\mathrm{max}=1 h$ Mpc$^{-1}$, $\\ell_\\mathrm{max}=500$\",\n", - " r\"Include Large Scales, $\\ell_\\mathrm{max}=1600$\",\n", - " \"Small Scales only\",\n", - " \"Large Scales only\",\n", - " r\"Halofit\",\n", - " r\"HMCode no baryons\",\n", - " \"OneCovariance only\",\n", - " \"No leakage correction\",\n", - "]\n", - "\n", - "bases = [\n", - " \"harmonic\",\n", - " \"harmonic\",\n", - " \"harmonic\",\n", - " \"configuration\",\n", - " \"harmonic\",\n", - " \"harmonic\",\n", - " \"harmonic\",\n", - " \"harmonic\",\n", - " \"harmonic\",\n", - " \"harmonic\",\n", - " \"harmonic\",\n", - " \"harmonic\",\n", - " \"harmonic\",\n", - " \"harmonic\",\n", - "]\n", - "\n", - "\n", - "properties = {}\n", - "\n", - "for i, root in enumerate(roots):\n", - " config = configparser.ConfigParser()\n", - " config.optionxform = str # Preserve case sensitivity of option names\n", - " config.read(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\")\n", - "\n", - " try:\n", - " lower_bound_cell_ee, upper_bound_cell_ee = map(\n", - " float, config[\"2pt_like\"][\"angle_range_CELL_EE_1_1\"].split()\n", - " )\n", - "\n", - " properties[root] = {\n", - " \"lower_bound_cell_ee\": lower_bound_cell_ee,\n", - " \"upper_bound_cell_ee\": upper_bound_cell_ee,\n", - " }\n", - " except KeyError:\n", - " properties[root] = {\"lower_bound_cell_ee\": 0.0, \"upper_bound_cell_ee\": 2048.0}\n", - "\n", - " if bases[i] == \"configuration\":\n", - " # Also save the scale cuts in theta for xi\n", - " add_xi_sys = config[\"2pt_like\"][\"add_xi_sys\"]\n", - " add_xi_sys = add_xi_sys == \"T\"\n", - " lower_bound_xi_plus, upper_bound_xi_plus = map(\n", - " float, config[\"2pt_like\"][\"angle_range_XI_PLUS_1_1\"].split()\n", - " )\n", - " lower_bound_xi_minus, upper_bound_xi_minus = map(\n", - " float, config[\"2pt_like\"][\"angle_range_XI_MINUS_1_1\"].split()\n", - " )\n", - "\n", - " properties[root].update(\n", - " {\n", - " \"add_xi_sys\": add_xi_sys,\n", - " \"lower_bound_xi_plus\": lower_bound_xi_plus,\n", - " \"upper_bound_xi_plus\": upper_bound_xi_plus,\n", - " \"lower_bound_xi_minus\": lower_bound_xi_minus,\n", - " \"upper_bound_xi_minus\": upper_bound_xi_minus,\n", - " }\n", - " )\n", - "\n", - "\n", - "print(roots)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Retrieve the chains" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# MAKE PARAMNAMES FILE\n", - "\n", - "for root in roots:\n", - " with open(root_dir + \"{}/samples_{}.txt\".format(\"/\" + root, root), \"r\") as file:\n", - " params = file.readline()[1:].split(\"\\t\")[:-4]\n", - " file.close()\n", - "\n", - " with open(\n", - " root_dir + \"{}/getdist_{}.paramnames\".format(\"/\" + root, root), \"w\"\n", - " ) as file:\n", - " for i in range(len(params)):\n", - " if len(params[i].split(\"--\")) > 1:\n", - " file.write(params[i].split(\"--\")[1] + \"\\n\")\n", - " else:\n", - " file.write(params[i].split(\"--\")[0] + \"\\n\")\n", - " file.close()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# READ CHAIN\n", - "\n", - "chains = []\n", - "\n", - "for root in roots:\n", - " samples = np.loadtxt(root_dir + \"{}/samples_{}.txt\".format(root, root))\n", - " print(len(samples))\n", - " if \"nautilus\" in root:\n", - " samples = np.column_stack(\n", - " (np.exp(samples[:, -3]), samples[:, -1] - samples[:, -2], samples[:, 0:-3])\n", - " )\n", - " else:\n", - " samples = np.column_stack((samples[:, -1], samples[:, -3], samples[:, 0:-4]))\n", - " np.savetxt(root_dir + \"{}/getdist_{}.txt\".format(root, root), samples)\n", - "\n", - " chain = g.samples_for_root(\n", - " root_dir + \"{}/getdist_{}\".format(root, root),\n", - " cache=False,\n", - " settings={\"ignore_rows\": 0, \"smooth_scale_2D\": 0.3, \"smooth_scale_1D\": 0.3},\n", - " )\n", - "\n", - " chains.append(chain)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "name_list = [\n", - " \"OMEGA_M\",\n", - " \"ombh2\",\n", - " \"h0\",\n", - " \"n_s\",\n", - " \"SIGMA_8\",\n", - " \"s_8_input\",\n", - " \"logt_agn\",\n", - " \"a\",\n", - " \"m1\",\n", - " \"bias_1\",\n", - "]\n", - "label_list = [\n", - " r\"\\Omega_m\",\n", - " r\"\\omega_b h^2\",\n", - " \"h_0\",\n", - " \"n_s\",\n", - " r\"\\sigma_8\",\n", - " \"S_8\",\n", - " \"log T_{AGN}\",\n", - " \"A_{IA}\",\n", - " \"m_1\",\n", - " r\"\\Delta z_1\",\n", - "]\n", - "\n", - "for chain in chains:\n", - " param_names = chain.getParamNames()\n", - " for name, label in zip(name_list, label_list):\n", - " param_names.parWithName(name).label = label" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Extract the best fit parameters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "best_fit = {}\n", - "\n", - "for root, chain in zip(roots, chains):\n", - " print(root)\n", - " likestats = chain.getLikeStats()\n", - " bestfit_idx = np.argmax(chain.loglikes)\n", - " maxlike = chain.loglikes[bestfit_idx]\n", - " print(f\"Maximum Likelihood: {maxlike:.5g}\")\n", - " best_fit[root] = {\"likelihood\": maxlike}\n", - " margestats = chain.getMargeStats()\n", - " s8_stats = margestats.parWithName(\"S_8\")\n", - " sigma8_stats = margestats.parWithName(\"SIGMA_8\")\n", - " omegam_stats = margestats.parWithName(\"OMEGA_M\")\n", - " a_ia_stats = margestats.parWithName(\"a\")\n", - "\n", - " best_fit[root].update(\n", - " {\n", - " \"S_8_mean\": s8_stats.mean,\n", - " \"S_8_lower\": s8_stats.mean - s8_stats.limits[0].lower,\n", - " \"S_8_upper\": s8_stats.limits[0].upper - s8_stats.mean,\n", - " \"sigma_8_mean\": sigma8_stats.mean,\n", - " \"sigma_8_lower\": sigma8_stats.mean - sigma8_stats.limits[0].lower,\n", - " \"sigma_8_upper\": sigma8_stats.limits[0].upper - sigma8_stats.mean,\n", - " \"omega_m_mean\": omegam_stats.mean,\n", - " \"omega_m_lower\": omegam_stats.mean - omegam_stats.limits[0].lower,\n", - " \"omega_m_upper\": omegam_stats.limits[0].upper - omegam_stats.mean,\n", - " \"A_IA_mean\": a_ia_stats.mean,\n", - " \"A_IA_lower\": a_ia_stats.mean - a_ia_stats.limits[0].lower,\n", - " \"A_IA_upper\": a_ia_stats.limits[0].upper - a_ia_stats.mean,\n", - " }\n", - " )\n", - " try:\n", - " t_agn_stats = margestats.parWithName(\"logt_agn\")\n", - " best_fit[root].update(\n", - " {\n", - " \"logt_agn_mean\": t_agn_stats.mean,\n", - " \"logt_agn_lower\": t_agn_stats.mean - t_agn_stats.limits[0].lower,\n", - " \"logt_agn_upper\": t_agn_stats.limits[0].upper - t_agn_stats.mean,\n", - " }\n", - " )\n", - " except Exception:\n", - " pass\n", - " for i, par in enumerate(likestats.names):\n", - " best_fit[root].update(\n", - " {par.name: np.average(chain.samples[:, i], weights=chain.weights)}\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run `Cosmosis` in test mode to get the data vectors" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if not os.path.exists(path_ini_files + \"/values_empty.ini\"):\n", - " content = \"\"\"[cosmological_parameters]\n", - "\n", - "tau = 0.0544\n", - "w = -1.0\n", - "mnu = 0.06\n", - "omega_k = 0.0\n", - "wa = 0.0\n", - "\n", - "[halo_model_parameters]\n", - "\n", - "[intrinsic_alignment_parameters]\n", - "\n", - "[shear_calibration_parameters]\n", - "\n", - "[nofz_shifts]\n", - "\n", - "[psf_leakage_parameters]\n", - "\"\"\"\n", - "\n", - " with open(path_ini_files + \"/values_empty.ini\", \"w\") as f:\n", - " f.write(content)\n", - " f.close()\n", - "\n", - " print(\"File created successfully\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "section_map = {\n", - " \"omch2\": \"cosmological_parameters\",\n", - " \"ombh2\": \"cosmological_parameters\",\n", - " \"h0\": \"cosmological_parameters\",\n", - " \"n_s\": \"cosmological_parameters\",\n", - " \"s_8_input\": \"cosmological_parameters\",\n", - " \"logt_agn\": \"halo_model_parameters\",\n", - " \"a\": \"intrinsic_alignment_parameters\",\n", - " \"m1\": \"shear_calibration_parameters\",\n", - " \"bias_1\": \"nofz_shifts\",\n", - " \"alpha\": \"psf_leakage_parameters\",\n", - " \"beta\": \"psf_leakage_parameters\",\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "env = os.environ.copy()\n", - "env[\"LD_LIBRARY_PATH\"] = (\n", - " \"/home/guerrini/.conda/envs/sp_validation/lib/python3.9/site-packages/cosmosis/datablock:\"\n", - " + env.get(\"LD_LIBRARY_PATH\", \"\")\n", - ")\n", - "\n", - "for root in roots:\n", - " print(root)\n", - " config = configparser.ConfigParser()\n", - " config.optionxform = str # Preserve case sensitivity of option names\n", - " config.read(path_ini_files + \"/values_empty.ini\")\n", - " for param, value in best_fit[root].items():\n", - " section = section_map.get(param)\n", - " if section is None:\n", - " continue\n", - " if section not in config:\n", - " config.add_section(section)\n", - " config[section][param] = str(value)\n", - "\n", - " with open(path_ini_files + \"/values_empty.ini\", \"w\") as configfile:\n", - " config.write(configfile)\n", - "\n", - " # Modify the ini file to run in test mode at the best fit\n", - " config = configparser.ConfigParser()\n", - " config.optionxform = str # Preserve case sensitivity of option names\n", - " config.read(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\")\n", - "\n", - " sampler = config[\"runtime\"][\"sampler\"]\n", - " config[\"runtime\"][\"sampler\"] = \"test\"\n", - " values = config[\"pipeline\"][\"values\"]\n", - " config[\"pipeline\"][\"values\"] = path_ini_files + \"/values_empty.ini\"\n", - "\n", - " with open(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\", \"w\") as configfile:\n", - " config.write(configfile)\n", - "\n", - " # Run cosmosis\n", - " result = subprocess.run(\n", - " [\"cosmosis\", \"cosmosis_config/cosmosis_pipeline_{}.ini\".format(root)],\n", - " env=env,\n", - " capture_output=True,\n", - " text=True,\n", - " )\n", - " print(f\"STDOUT:\\n{result.stdout}\")\n", - " print(f\"STDERR:\\n{result.stderr}\")\n", - "\n", - " # Modify the ini file to the previous one\n", - " config[\"pipeline\"][\"values\"] = values\n", - " config[\"runtime\"][\"sampler\"] = sampler\n", - "\n", - " with open(path_ini_files + f\"/cosmosis_pipeline_{root}.ini\", \"w\") as configfile:\n", - " config.write(configfile)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Compute the $\\chi^2$" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "output_folder = \"/n09data/guerrini/output_chains/\"\n", - "\n", - "metrics = {}\n", - "\n", - "for i, root in enumerate(roots):\n", - " print(root)\n", - "\n", - " base = bases[i]\n", - "\n", - " if base == \"harmonic\":\n", - " # Remove cell from the end of root\n", - " root_cell_removed = root.replace(\"_cell\", \"\")\n", - "\n", - " lower_bound_cell_ee = properties[root][\"lower_bound_cell_ee\"]\n", - " upper_bound_cell_ee = properties[root][\"upper_bound_cell_ee\"]\n", - " print(upper_bound_cell_ee)\n", - "\n", - " # Read the results\n", - " ell = np.loadtxt(output_folder + \"best_fit/{}/shear_cl/ell.txt\".format(root))\n", - " shear_cl = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_cl/bin_1_1.txt\".format(root)\n", - " )\n", - "\n", - " # Read the data\n", - " data = fits.open(f\"data/{root_cell_removed}/cosmosis_{root}.fits\")\n", - "\n", - " ell_data = data[\"CELL_EE\"].data[\"ANG\"]\n", - " cell_data = data[\"CELL_EE\"].data[\"VALUE\"]\n", - "\n", - " # Load the covariance\n", - " cov = data[\"COVMAT\"].data\n", - " cov_cell = cov\n", - "\n", - " # interpolate the model\n", - " interp_cell_ee = interp1d(ell, shear_cl, kind=\"cubic\", fill_value=\"extrapolate\")\n", - "\n", - " cell_model = interp_cell_ee(ell_data)\n", - "\n", - " # Apply scale cuts\n", - " mask_cell = (ell_data > lower_bound_cell_ee) & (ell_data < upper_bound_cell_ee)\n", - " cell_data = cell_data[mask_cell]\n", - " cell_model = cell_model[mask_cell]\n", - " cov_cell = cov_cell[mask_cell][:, mask_cell]\n", - "\n", - " cell_chi2 = np.dot(\n", - " (cell_model - cell_data),\n", - " np.dot(np.linalg.inv(cov_cell), (cell_model - cell_data)),\n", - " )\n", - " n_dof_cell = np.sum(mask_cell)\n", - " print(n_dof_cell)\n", - " n_dof_cell -= 9\n", - " p_value_cell = 1 - stats.chi2.cdf(cell_chi2, n_dof_cell)\n", - "\n", - " metrics[root] = {\n", - " \"chi2\": cell_chi2,\n", - " \"n_dof\": n_dof_cell,\n", - " \"p_value\": p_value_cell,\n", - " }\n", - "\n", - " elif base == \"configuration\":\n", - " add_xi_sys = properties[root][\"add_xi_sys\"]\n", - " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", - " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", - " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", - " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", - "\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " shear_xi_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " shear_xi_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_sys_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - "\n", - " # Read model tau_stats\n", - " theta_tau = np.loadtxt(\n", - " output_folder + \"best_fit/{}/tau_0_plus/theta.txt\".format(root)\n", - " )\n", - " theta_tau_arcmin = theta_tau * 180 * 60 / np.pi\n", - " tau_0_model = np.loadtxt(\n", - " output_folder + \"best_fit/{}/tau_0_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " tau_2_model = np.loadtxt(\n", - " output_folder + \"best_fit/{}/tau_2_plus/bin_1_1.txt\".format(root)\n", - " )\n", - "\n", - " # Read the data\n", - " data = fits.open(\n", - " f\"data/{catalog_version_real_space}/cosmosis_{catalog_version_real_space}.fits\"\n", - " )\n", - "\n", - " theta_data = data[\"XI_PLUS\"].data[\"ANG\"]\n", - " xi_plus_data = data[\"XI_PLUS\"].data[\"VALUE\"]\n", - " xi_minus_data = data[\"XI_MINUS\"].data[\"VALUE\"]\n", - " tau_0_data = data[\"TAU_0_PLUS\"].data[\"VALUE\"]\n", - " tau_2_data = data[\"TAU_2_PLUS\"].data[\"VALUE\"]\n", - "\n", - " # Load the covariance\n", - " cov = data[\"COVMAT\"].data\n", - " cov_xi = cov[0 : 2 * len(xi_plus_data), 0 : 2 * len(xi_plus_data)]\n", - " cov_tau = cov[2 * len(xi_plus_data) :, 2 * len(xi_plus_data) :]\n", - "\n", - " # interpolate the model\n", - " interp_xi_plus = interp1d(\n", - " theta_arcmin, shear_xi_plus, kind=\"cubic\", fill_value=\"extrapolate\"\n", - " )\n", - " interp_xi_minus = interp1d(\n", - " theta_arcmin, shear_xi_minus, kind=\"cubic\", fill_value=\"extrapolate\"\n", - " )\n", - "\n", - " xi_plus_model = interp_xi_plus(theta_data)\n", - " if add_xi_sys:\n", - " xi_plus_model += xi_sys_plus\n", - " xi_minus_model = interp_xi_minus(theta_data)\n", - " if add_xi_sys:\n", - " xi_minus_model += xi_sys_minus\n", - "\n", - " # Concatenate the data vector\n", - " xi_data = np.concatenate((xi_plus_data, xi_minus_data))\n", - " xi_model = np.concatenate((xi_plus_model, xi_minus_model))\n", - "\n", - " tau_data = np.concatenate((tau_0_data, tau_2_data))\n", - " tau_model = np.concatenate((tau_0_model, tau_2_model))\n", - "\n", - " # Apply scale cuts\n", - " mask_xi_plus = (theta_data > lower_bound_xi_plus) & (\n", - " theta_data < upper_bound_xi_plus\n", - " )\n", - " mask_xi_minus = (theta_data > lower_bound_xi_minus) & (\n", - " theta_data < upper_bound_xi_minus\n", - " )\n", - " mask = np.concatenate((mask_xi_plus, mask_xi_minus))\n", - "\n", - " xi_data = xi_data[mask]\n", - " xi_model = xi_model[mask]\n", - " cov_xi = cov_xi[mask][:, mask]\n", - "\n", - " xi_plus_chi2 = np.dot(\n", - " (xi_model - xi_data), np.dot(np.linalg.inv(cov_xi), (xi_model - xi_data))\n", - " )\n", - " tau_chi2 = np.dot(\n", - " (tau_model - tau_data),\n", - " np.dot(np.linalg.inv(cov_tau), (tau_model - tau_data)),\n", - " )\n", - " n_dof_xi = np.sum(mask)\n", - " n_dof_xi -= 11\n", - " n_dof_tau = len(tau_0_data) + len(tau_2_data)\n", - " p_value_xi = 1 - stats.chi2.cdf(xi_plus_chi2, n_dof_xi)\n", - " p_value_tau = 1 - stats.chi2.cdf(tau_chi2, n_dof_tau)\n", - " chi2_tot = xi_plus_chi2 + tau_chi2\n", - " n_dof_tot = n_dof_xi + n_dof_tau\n", - " p_value_tot = 1 - stats.chi2.cdf(chi2_tot, n_dof_tot)\n", - "\n", - " metrics[root] = {\"chi2\": xi_plus_chi2, \"n_dof\": n_dof_xi, \"p_value\": p_value_xi}\n", - "\n", - " print(\"Done!\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def get_latex_table(metrics):\n", - " latex_lines = [\n", - " r\"\\begin{tabular}{|l|c|c|c|c|c|c|c|}\",\n", - " r\"\\hline\",\n", - " r\"Experiment name & $S_8$ & $\\Omega_m$ & $\\sigma_8$ & $A_\\mathrm{IA}$ & $\\log T_\\mathrm{AGN}$ & $\\chi^2$/dof & PTE \\\\ \",\n", - " r\"\\hline\",\n", - " ]\n", - "\n", - " for i, (root, vals) in enumerate(metrics.items()):\n", - " label = labels[i]\n", - " best_fit_vals = best_fit[root]\n", - " log_t_agn_mean = best_fit_vals.get(\"logt_agn_mean\", None)\n", - "\n", - " if log_t_agn_mean is None:\n", - " logt_agn_mean_str = \"N/A\"\n", - " else:\n", - " logt_agn_mean_str = f\"${best_fit_vals['logt_agn_mean']:.3f}^{{+{best_fit_vals['logt_agn_upper']:.3f}}}_{{-{best_fit_vals['logt_agn_lower']:.3f}}}$\"\n", - " line = (\n", - " f\"{label} & \"\n", - " rf\" ${best_fit_vals['S_8_mean']:.3f}^{{+{best_fit_vals['S_8_upper']:.3f}}}_{{-{best_fit_vals['S_8_lower']:.3f}}}$ & \"\n", - " rf\" ${best_fit_vals['omega_m_mean']:.3f}^{{+{best_fit_vals['omega_m_upper']:.3f}}}_{{-{best_fit_vals['omega_m_lower']:.3f}}}$ & \"\n", - " rf\" ${best_fit_vals['sigma_8_mean']:.3f}^{{+{best_fit_vals['sigma_8_upper']:.3f}}}_{{-{best_fit_vals['sigma_8_lower']:.3f}}}$ & \"\n", - " rf\" ${best_fit_vals['A_IA_mean']:.3f}^{{+{best_fit_vals['A_IA_upper']:.3f}}}_{{-{best_fit_vals['A_IA_lower']:.3f}}}$ & \"\n", - " rf\" {logt_agn_mean_str} & \"\n", - " f\"{vals['chi2']:.2f}/{vals['n_dof']} & {vals['p_value']:.5f} \\\\\\\\\"\n", - " )\n", - " latex_lines.append(line)\n", - "\n", - " latex_lines.append(r\"\\hline\")\n", - " latex_lines.append(r\"\\end{tabular}\")\n", - "\n", - " # Print LaTeX table\n", - " print(\"\\n\".join(latex_lines))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "get_latex_table(metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def display_markdown(metrics):\n", - " # Build Markdown table\n", - " header = (\n", - " \"| Root | $\\\\chi^2$ ($C_\\\\ell$) / dof | p-val ($C_\\\\ell$) |\\n\"\n", - " \"|------|----------------|------------|\\n\"\n", - " )\n", - "\n", - " rows = []\n", - " for root, vals in metrics.items():\n", - " row = f\"| `{root}` \"\n", - " row += f\"| {vals['chi2']:.2f} / {vals['n_dof']} \"\n", - " row += f\"| {vals['p_value']:.5f} \"\n", - " rows.append(row)\n", - "\n", - " # Display in Jupyter\n", - " display(Markdown(header + \"\\n\".join(rows)))\n", - " return header + \"\\n\".join(rows)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "markdown_source = display_markdown(metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "markdown_source" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Plot the best-fit of each model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "catalog_version = \"SP_v1.4.6_leak_corr_A_lmin=300_lmax=1600\"\n", - "data = fits.open(\n", - " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_version}/cosmosis_{catalog_version}_cell.fits\"\n", - ")\n", - "cell_ee = data[\"CELL_EE\"].data\n", - "cov_mat = data[\"COVMAT\"].data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "fig, ax = plt.subplots(1, 1, figsize=(15, 8))\n", - "\n", - "ell = cell_ee[\"ANG\"]\n", - "cell = cell_ee[\"VALUE\"]\n", - "\n", - "ax.errorbar(\n", - " ell,\n", - " ell * cell,\n", - " yerr=ell * np.sqrt(np.diag(cov)),\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - ")\n", - "ax.set_xlabel(r\"$\\ell$\")\n", - "ax.set_ylabel(r\"$\\ell C_\\ell$\")\n", - "ax.set_xlim(ell.min() - 10, ell.max() + 100)\n", - "ax.set_xscale(\"squareroot\")\n", - "ax.set_xticks(np.array([100, 400, 900, 1600]))\n", - "ax.minorticks_on()\n", - "ax.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", - "minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)]\n", - "ax.xaxis.set_ticks(minor_ticks, minor=True)\n", - "\n", - "plt.legend(fontsize=15)\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def plot_best_fit(\n", - " data_points,\n", - " root_to_plot,\n", - " line_args,\n", - " savefile,\n", - " ell_min=10.0,\n", - " ell_max=2048.0,\n", - " multiply_ell=True,\n", - " loc_legend=\"best\",\n", - " bbox_to_anchor=None,\n", - " label_data=\"Fiducial data\",\n", - " labels=None,\n", - "):\n", - " data = fits.open(\n", - " f\"/home/guerrini/sp_validation/cosmo_inference/data/{data_points}/cosmosis_{data_points}_cell.fits\"\n", - " )\n", - " cell_ee = data[\"CELL_EE\"].data\n", - " cov_mat = data[\"COVMAT\"].data\n", - "\n", - " if labels is None:\n", - " labels = root_to_plot\n", - "\n", - " fig, ax = plt.subplots(1, 1, figsize=(8, 5))\n", - "\n", - " ell, cell = cell_ee[\"ANG\"], cell_ee[\"VALUE\"]\n", - " ax.errorbar(\n", - " ell,\n", - " ell * cell,\n", - " yerr=ell * np.sqrt(np.diag(cov_mat)),\n", - " fmt=\"o\",\n", - " label=label_data,\n", - " color=\"black\",\n", - " capsize=2,\n", - " )\n", - "\n", - " for idx, (label, root) in enumerate(zip(labels, root_to_plot)):\n", - " # Read the results\n", - " ell = np.loadtxt(output_folder + \"best_fit/{}/shear_cl/ell.txt\".format(root))\n", - " shear_cl = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_cl/bin_1_1.txt\".format(root)\n", - " )\n", - "\n", - " mask = (ell > ell_min) & (ell < ell_max)\n", - "\n", - " ax.plot(\n", - " ell[mask],\n", - " ell[mask] * shear_cl[mask] if multiply_ell else shear_cl[mask],\n", - " label=label,\n", - " **line_args[idx],\n", - " )\n", - "\n", - " # Plot the scale cuts for different k_max\n", - " ax.axvline(x=1800, color=\"black\", linestyle=\"--\", alpha=0.5)\n", - " ax.axvline(x=2048, color=\"black\", linestyle=\"--\", alpha=1.0)\n", - " ax.axvline(x=500, color=\"black\", linestyle=\"--\", alpha=0.3)\n", - "\n", - " # Add labels directly under the tick\n", - " ax.text(\n", - " 1740,\n", - " 0.90,\n", - " r\"$k_\\mathrm{max} = 3 h$ Mpc$^{-1}$\",\n", - " transform=ax.get_xaxis_transform(),\n", - " ha=\"center\",\n", - " va=\"top\",\n", - " fontsize=14,\n", - " rotation=90,\n", - " )\n", - "\n", - " ax.text(\n", - " 1978,\n", - " 0.90,\n", - " r\"$k_\\mathrm{max} = 5 h$ Mpc$^{-1}$\",\n", - " transform=ax.get_xaxis_transform(),\n", - " ha=\"center\",\n", - " va=\"top\",\n", - " fontsize=14,\n", - " rotation=90,\n", - " )\n", - "\n", - " ax.text(\n", - " 470,\n", - " 0.90,\n", - " r\"$k_\\mathrm{max} = 1 h$ Mpc$^{-1}$\",\n", - " transform=ax.get_xaxis_transform(),\n", - " ha=\"center\",\n", - " va=\"top\",\n", - " fontsize=14,\n", - " rotation=90,\n", - " )\n", - "\n", - " ell, cell = cell_ee[\"ANG\"], cell_ee[\"VALUE\"]\n", - " ax.set_ylabel(r\"$\\ell C_\\ell \\times 10^{-7}$\", fontsize=20)\n", - " ax.set_xlabel(r\"Multipole $\\ell$\", fontsize=20)\n", - " ax.set_xlim(ell.min() - 10, ell.max() + 100)\n", - " ax.set_xscale(\"squareroot\")\n", - " ax.set_xticks(np.array([100, 400, 900, 1600]))\n", - " ax.minorticks_on()\n", - " ax.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", - " minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)]\n", - " ax.xaxis.set_ticks(minor_ticks, minor=True)\n", - " ax.tick_params(axis=\"both\", which=\"major\", labelsize=14)\n", - " ax.tick_params(axis=\"both\", which=\"minor\", labelsize=10)\n", - " ax.yaxis.get_offset_text().set_visible(False)\n", - "\n", - " plt.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor, fontsize=11)\n", - "\n", - " if savefile is not None:\n", - " plt.savefig(savefile, bbox_inches=\"tight\")\n", - "\n", - " plt.show()\n", - "\n", - "\n", - "def plot_best_fit_ratio(\n", - " root_to_plot, colours, savefile, theta_min=1.0, theta_max=250.0\n", - "):\n", - " data = fits.open(\n", - " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_version}/cosmosis_{catalog_version}.fits\"\n", - " )\n", - " xi_plus = data[\"XI_PLUS\"].data\n", - " xi_minus = data[\"XI_MINUS\"].data\n", - " cov_mat = data[\"COVMAT\"].data\n", - "\n", - " plt.figure(figsize=(15, 15))\n", - "\n", - " plt.subplot(211)\n", - "\n", - " root = roots[0]\n", - " add_xi_sys = properties[root][\"add_xi_sys\"]\n", - " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", - " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", - " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", - " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", - "\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " shear_xi_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " shear_xi_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_sys_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - " theta_xi_sys = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", - " )\n", - " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", - "\n", - " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", - " xi_plus_model_fiducial = shear_xi_plus[mask]\n", - " if add_xi_sys:\n", - " xi_plus_model_fiducial += np.interp(\n", - " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_plus\n", - " )\n", - "\n", - " plt.errorbar(\n", - " xi_plus[\"ANG\"],\n", - " xi_plus[\"VALUE\"]\n", - " / np.interp(xi_plus[\"ANG\"], theta_arcmin[mask], xi_plus_model_fiducial),\n", - " yerr=np.sqrt(np.diag(cov_mat))[:20]\n", - " / np.abs(np.interp(xi_plus[\"ANG\"], theta_arcmin[mask], xi_plus_model_fiducial)),\n", - " fmt=\"o\",\n", - " label=f\"{catalog_version} data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - " )\n", - "\n", - " for root, color in zip(root_to_plot, colours):\n", - " add_xi_sys = properties[root][\"add_xi_sys\"]\n", - " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", - " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", - " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", - " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", - "\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " shear_xi_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " shear_xi_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_sys_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - " theta_xi_sys = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", - " )\n", - " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", - "\n", - " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", - " xi_plus_model = shear_xi_plus[mask]\n", - " if add_xi_sys:\n", - " xi_plus_model += np.interp(\n", - " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_plus\n", - " )\n", - "\n", - " alpha = 1.0 if root == roots[0] else 0.5\n", - " plt.plot(\n", - " theta_arcmin[mask],\n", - " xi_plus_model / xi_plus_model_fiducial,\n", - " color=color,\n", - " label=root,\n", - " alpha=alpha,\n", - " )\n", - " plt.axvline(x=lower_bound_xi_plus, color=color, linestyle=\"--\", alpha=0.3)\n", - " plt.axvline(x=upper_bound_xi_plus, color=color, linestyle=\"--\", alpha=0.3)\n", - "\n", - " plt.ylabel(r\"$\\xi_{+}/\\xi_{+, \\text{fid}}$\", fontsize=26)\n", - " plt.xscale(\"log\")\n", - " # plt.yscale('log')\n", - " plt.legend(loc=\"lower left\", fontsize=8)\n", - "\n", - " plt.subplot(212)\n", - "\n", - " root = roots[0]\n", - " add_xi_sys = properties[root][\"add_xi_sys\"]\n", - " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", - " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", - " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", - " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", - "\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " shear_xi_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " shear_xi_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_sys_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - " theta_xi_sys = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", - " )\n", - " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", - "\n", - " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", - " xi_minus_model_fiducial = shear_xi_minus[mask]\n", - " if add_xi_sys:\n", - " xi_minus_model_fiducial += np.interp(\n", - " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_minus\n", - " )\n", - "\n", - " plt.errorbar(\n", - " xi_minus[\"ANG\"],\n", - " xi_minus[\"VALUE\"]\n", - " / np.interp(xi_minus[\"ANG\"], theta_arcmin[mask], xi_minus_model_fiducial),\n", - " yerr=np.sqrt(np.diag(cov_mat))[20:40]\n", - " / np.abs(\n", - " np.interp(xi_minus[\"ANG\"], theta_arcmin[mask], xi_minus_model_fiducial)\n", - " ),\n", - " fmt=\"o\",\n", - " label=f\"{catalog_version} data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - " )\n", - "\n", - " for root, color in zip(root_to_plot, colours):\n", - " add_xi_sys = properties[root][\"add_xi_sys\"]\n", - " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", - " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", - " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", - " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", - "\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " shear_xi_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " shear_xi_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_sys_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - " theta_xi_sys = np.loadtxt(\n", - " output_folder + \"best_fit/{}/xi_sys/theta.txt\".format(root)\n", - " )\n", - " theta_xi_sys_arcmin = theta_xi_sys * 180 * 60 / np.pi\n", - "\n", - " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", - " xi_minus_model = shear_xi_minus[mask]\n", - " if add_xi_sys:\n", - " xi_minus_model += np.interp(\n", - " theta_arcmin[mask], theta_xi_sys_arcmin, xi_sys_minus\n", - " )\n", - "\n", - " alpha = 1.0 if root == roots[0] else 0.5\n", - " plt.plot(\n", - " theta_arcmin[mask],\n", - " xi_minus_model / xi_minus_model_fiducial,\n", - " color=color,\n", - " label=root,\n", - " alpha=alpha,\n", - " )\n", - " plt.axvline(x=lower_bound_xi_minus, color=color, linestyle=\"--\", alpha=0.3)\n", - " plt.axvline(x=upper_bound_xi_minus, color=color, linestyle=\"--\", alpha=0.3)\n", - "\n", - " plt.xlabel(r\"$\\theta$ [arcmin]\", fontsize=26)\n", - " plt.ylabel(r\"$\\xi_{-}/\\xi_{-, \\text{fid}}$\", fontsize=26)\n", - " plt.xscale(\"log\")\n", - " plt.ylim(0, 2)\n", - " # plt.yscale('log')\n", - " plt.legend(loc=\"lower left\", fontsize=8)\n", - "\n", - " if savefile is not None:\n", - " plt.savefig(savefile, bbox_inches=\"tight\")\n", - "\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.rcParams.update({\"font.family\": \"serif\"})\n", - "\n", - "root_to_plot = [\n", - " \"SP_v1.4.6_leak_corr_A_lmin=300_lmax=1600_cell\",\n", - " \"SP_v1.4.6_leak_corr_A_halofit_cell\",\n", - " \"SP_v1.4.6_leak_corr_A_10_80\",\n", - "]\n", - "\n", - "labels = [\n", - " r\"UNIONS $C_\\ell$, Blind A\",\n", - " r\"UNIONS $C_\\ell$, Halofit\",\n", - " r\"UNIONS $\\xi_\\pm(\\vartheta)$ (Goh et al., 2026)\",\n", - "]\n", - "\n", - "line_args = [\n", - " {\"color\": \"royalblue\", \"linestyle\": \"-\"},\n", - " {\"color\": \"royalblue\", \"linestyle\": \"--\"},\n", - " {\"color\": \"orange\", \"linestyle\": \"-\"},\n", - "]\n", - "\n", - "log_legend = \"lower center\"\n", - "bbox_to_anchor = (0.685, 0.70)\n", - "\n", - "savefile = \"../papers/harmonic/plots/paperplot_Cell_EE_and_best_fit.png\"\n", - "\n", - "plot_best_fit(\n", - " catalog_version,\n", - " root_to_plot,\n", - " line_args,\n", - " savefile,\n", - " labels=labels,\n", - " loc_legend=log_legend,\n", - " bbox_to_anchor=bbox_to_anchor,\n", - ")\n", - "\n", - "savefile = \"../papers/harmonic/plots/paperplot_Cell_EE_and_best_fit.pdf\"\n", - "\n", - "plot_best_fit(\n", - " catalog_version,\n", - " root_to_plot,\n", - " line_args,\n", - " savefile,\n", - " labels=labels,\n", - " loc_legend=log_legend,\n", - " bbox_to_anchor=bbox_to_anchor,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "root_to_plot = [\n", - " \"SP_v1.4.6_leak_corr_A_small_scales_cell\",\n", - " \"SP_v1.4.6_leak_corr_A_large_scales_cell\",\n", - "]\n", - "\n", - "labels = [r\"Small scales only\", r\"Large scales only\"]\n", - "\n", - "line_args = [\n", - " {\"color\": \"royalblue\", \"linestyle\": \"-\"},\n", - " {\"color\": \"royalblue\", \"linestyle\": \"--\"},\n", - "]\n", - "\n", - "savefile = \"../papers/harmonic/plots/small_vs_large_scale.png\"\n", - "\n", - "plot_best_fit(catalog_version, root_to_plot, line_args, savefile, labels=labels)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# DEPRECATED code\n", - "\n", - "root_to_plot = [\n", - " \"SP_v1.4.5_A\",\n", - " # \"SP_v1.4.5_A_no_IA\",\n", - " # \"SP_v1.4.5_A_no_dz\",\n", - " # \"SP_v1.4.5_A_no_m_bias\",\n", - " \"SP_v1.4.5_A_sc_3_150\",\n", - " \"SP_v1.4.5_A_sc_3_60\",\n", - " \"SP_v1.4.5_A_sc_10_150\",\n", - " \"SP_v1.4.5_A_sc_10_60\",\n", - " \"SP_v1.4.5_A_sc_5_150\",\n", - " \"SP_v1.4.5_A_sc_7_150\",\n", - " # \"SP_v1.4.5_A_no_leakage\"\n", - "]\n", - "\n", - "\"\"\" root_to_plot = [\n", - " f\"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_{int(i)}.0_80.0_10.0_80.0\" for i in [3, 5, 7, 10, 11]\n", - "] \"\"\"\n", - "\n", - "root_to_plot = [\n", - " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0\",\n", - " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0_no_alpha_beta\",\n", - "]\n", - "\n", - "\n", - "colours = [\n", - " \"red\",\n", - " \"salmon\",\n", - " \"darkorange\",\n", - " \"forestgreen\",\n", - " \"turquoise\",\n", - " \"darkviolet\",\n", - " \"crimson\",\n", - " \"gold\",\n", - " \"lightcoral\",\n", - " \"mediumseagreen\",\n", - " \"lightsteelblue\",\n", - " \"black\",\n", - " \"silver\",\n", - " \"peru\",\n", - " \"maroon\",\n", - " \"olive\",\n", - "]\n", - "\n", - "savefile = \"best_fit_ratio_w_wo_leakage.png\"\n", - "\n", - "plot_best_fit_ratio(root_to_plot, colours, savefile)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def plot_best_fit_tau(root_to_plot, colours, savefile, theta_min=1.0, theta_max=250.0):\n", - " data = fits.open(\n", - " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_version}/cosmosis_{catalog_version}.fits\"\n", - " )\n", - " tau_0 = data[\"TAU_0_PLUS\"].data\n", - " tau_2 = data[\"TAU_2_PLUS\"].data\n", - " cov_mat = data[\"COVMAT\"].data\n", - "\n", - " plt.figure(figsize=(15, 15))\n", - "\n", - " plt.subplot(211)\n", - "\n", - " plt.errorbar(\n", - " tau_0[\"ANG\"],\n", - " tau_0[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[40:60],\n", - " fmt=\"o\",\n", - " label=f\"{catalog_version} data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - " )\n", - "\n", - " for root, color in zip(root_to_plot, colours):\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/tau_0_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " tau_0_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/tau_0_plus/bin_1_1.txt\".format(root)\n", - " )\n", - "\n", - " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", - "\n", - " plt.plot(\n", - " theta_arcmin[mask], tau_0_plus[mask], color=color, label=root, alpha=0.5\n", - " )\n", - "\n", - " plt.ylabel(r\"$\\tau_0$\", fontsize=26)\n", - " plt.xscale(\"log\")\n", - " # plt.yscale('log')\n", - " plt.legend(loc=\"upper right\", fontsize=8)\n", - "\n", - " plt.subplot(212)\n", - "\n", - " y_plot_tau_2 = tau_2[\"ANG\"] * tau_2[\"VALUE\"]\n", - " y_errorbar = tau_2[\"ANG\"] * np.sqrt(np.diag(cov_mat))[60:80]\n", - " plt.errorbar(\n", - " tau_2[\"ANG\"],\n", - " y_plot_tau_2,\n", - " yerr=y_errorbar,\n", - " fmt=\"o\",\n", - " label=f\"{catalog_version} data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - " )\n", - "\n", - " for root, color in zip(root_to_plot, colours):\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"best_fit/{}/tau_2_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " tau_2_plus = np.loadtxt(\n", - " output_folder + \"best_fit/{}/tau_2_plus/bin_1_1.txt\".format(root)\n", - " )\n", - "\n", - " mask = (theta_arcmin > theta_min) & (theta_arcmin < theta_max)\n", - "\n", - " plt.plot(\n", - " theta_arcmin[mask],\n", - " theta_arcmin[mask] * tau_2_plus[mask],\n", - " color=color,\n", - " label=root,\n", - " alpha=0.5,\n", - " )\n", - "\n", - " plt.xlabel(r\"$\\theta$ [arcmin]\", fontsize=26)\n", - " plt.ylabel(r\"$\\theta \\tau_2$\", fontsize=26)\n", - " plt.xscale(\"log\")\n", - " # plt.yscale('log')\n", - " plt.legend(loc=\"upper left\", fontsize=8)\n", - "\n", - " if savefile is not None:\n", - " plt.savefig(savefile, bbox_inches=\"tight\")\n", - "\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "root_to_plot = [\n", - " \"SP_v1.4.5_A\",\n", - " # \"SP_v1.4.5_A_no_IA\",\n", - " # \"SP_v1.4.5_A_no_dz\",\n", - " # \"SP_v1.4.5_A_no_m_bias\",\n", - " \"SP_v1.4.5_A_sc_3_150\",\n", - " \"SP_v1.4.5_A_sc_3_60\",\n", - " \"SP_v1.4.5_A_sc_10_150\",\n", - " \"SP_v1.4.5_A_sc_10_60\",\n", - " \"SP_v1.4.5_A_sc_5_150\",\n", - " \"SP_v1.4.5_A_sc_7_150\",\n", - " # \"SP_v1.4.5_A_no_leakage\"\n", - "]\n", - "\n", - "root_to_plot = [\n", - " f\"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_{int(i)}.0_80.0_10.0_80.0\"\n", - " for i in [3, 5, 7, 10, 11]\n", - "]\n", - "\n", - "colours = [\n", - " \"red\",\n", - " \"salmon\",\n", - " \"darkorange\",\n", - " \"forestgreen\",\n", - " \"turquoise\",\n", - " \"darkviolet\",\n", - " \"crimson\",\n", - " \"gold\",\n", - " \"lightcoral\",\n", - " \"mediumseagreen\",\n", - " \"lightsteelblue\",\n", - " \"black\",\n", - " \"silver\",\n", - " \"peru\",\n", - " \"maroon\",\n", - " \"olive\",\n", - "]\n", - "\n", - "savefile = \"best_fit_tau_new_binning.png\"\n", - "\n", - "plot_best_fit_tau(root_to_plot, colours, savefile)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pseudo_cell = fits.open(\n", - " \"/home/guerrini/sp_validation/cosmo_val/output/pseudo_cl_SP_v1.4.5.fits\"\n", - ")[1].data\n", - "cov_pseudo_cell = fits.open(\n", - " \"/home/guerrini/sp_validation/cosmo_val/output/pseudo_cl_cov_SP_v1.4.5.fits\"\n", - ")\n", - "\n", - "theory_ell = np.loadtxt(\n", - " \"/n09data/guerrini/output_chains/best_fit/SP_v1.4.5_A/shear_cl/ell.txt\"\n", - ")\n", - "theory_cell = np.loadtxt(\n", - " \"/n09data/guerrini/output_chains/best_fit/SP_v1.4.5_A/shear_cl/bin_1_1.txt\"\n", - ")\n", - "\n", - "pw = hp.pixwin(1024, lmax=2048)\n", - "\n", - "plt.errorbar(\n", - " pseudo_cell[\"ELL\"],\n", - " pseudo_cell[\"ELL\"] * pseudo_cell[\"EE\"],\n", - " yerr=pseudo_cell[\"ELL\"] * np.sqrt(np.diag(cov_pseudo_cell[\"COVAR_EE_EE\"].data)),\n", - " capsize=2,\n", - " c=\"k\",\n", - " fmt=\"o\",\n", - " markersize=2,\n", - ")\n", - "\n", - "mask = (theory_ell > 0.1) & (theory_ell < 2048)\n", - "plt.plot(\n", - " theory_ell[mask],\n", - " theory_ell[mask]\n", - " * theory_cell[mask]\n", - " * np.interp(theory_ell[mask], np.arange(0, 2049), pw) ** 2,\n", - " c=\"r\",\n", - " label=\"best-fit $\\\\theta \\\\in [3-200]$\",\n", - ")\n", - "\n", - "plt.xlabel(r\"$\\ell$\", fontsize=26)\n", - "plt.ylabel(r\"$\\ell C_\\ell^{EE}$\", fontsize=26)\n", - "plt.legend()\n", - "plt.savefig(\"SP_v1.4.5_A_cell.png\")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cov_pseudo_cell.info()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "sp_validation", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/S8_om_sigma8_whisker.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/S8_om_sigma8_whisker.ipynb deleted file mode 100644 index 7a4fd5d2..00000000 --- a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/S8_om_sigma8_whisker.ipynb +++ /dev/null @@ -1,645 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "0", - "metadata": {}, - "source": [ - "# Whisker plot\n", - "\n", - "This notebook plots the whisker plot of $S_8$, $\\Omega_m$ and $\\sigma_8$" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import sys\n", - "\n", - "# Trick to plot with tex\n", - "os.environ[\"LD_LIBRARY_PATH\"] = \"\"\n", - "os.environ[\"CONDA_PREFIX\"] = \"/home/guerrini/.conda/envs/sp_validation_3.11\"\n", - "\n", - "sys.path.append(\"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/notebooks/\")\n", - "\n", - "import sys\n", - "import warnings\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import seaborn as sns\n", - "from getdist import plots\n", - "\n", - "sys.path.append(\"/home/guerrini/sp_validation/cosmo_inference/scripts\")\n", - "\n", - "import chain_postprocessing as cp\n", - "\n", - "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", - "\n", - "plt.rc(\"text\", usetex=True)\n", - "\n", - "sns.set_palette(\"husl\")\n", - "\n", - "g = plots.get_subplot_plotter(width_inch=30)\n", - "g.settings.axes_fontsize = 60\n", - "g.settings.axes_labelsize = 60\n", - "g.settings.alpha_filled_add = 0.7\n", - "g.settings.legend_fontsize = 60\n", - "\n", - "%matplotlib inline\n", - "\n", - "# SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE\n", - "root_dir = \"/n09data/guerrini/output_chains/\"\n", - "root_external = f\"{root_dir}/ext_data/\"\n", - "blind = \"B\"\n", - "\n", - "roots = [\n", - " f\"SP_v1.4.6.3_{blind}_fiducial_config\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}\",\n", - " \"Planck18\",\n", - " \"DES Y6\",\n", - " \"KiDS-Legacy_bandpowers\",\n", - " \"KiDS-Legacy_cosebis\",\n", - " \"KiDS-Legacy_xipm\",\n", - " \"HSC_Y3\",\n", - " \"HSC_Y3_cell\",\n", - " f\"SP_v1.4.6.3_{blind}_small_scales_config\",\n", - " f\"SP_v1.4.6.3_{blind}_flat_alpha_beta_config\",\n", - " f\"SP_v1.4.6.3_{blind}_no_xi_sys_config\",\n", - " f\"SP_v1.4.6.3_{blind}_no_leak_corr_config\",\n", - " f\"SP_v1.4.6.3_{blind}_flat_delta_z_config\",\n", - " f\"SP_v1.4.6.3_{blind}_no_delta_z_config\",\n", - " f\"SP_v1.4.6.3_{blind}_flat_ia_config\",\n", - " f\"SP_v1.4.6.3_{blind}_no_ia_config\",\n", - " f\"SP_v1.4.6.3_{blind}_no_m_bias_config\",\n", - " f\"SP_v1.4.6.3_{blind}_unmasked_covmat_config\",\n", - " f\"SP_v1.4.6.3_{blind}_halofit_config\",\n", - " f\"SP_v1.4.6.3_{blind}_no_baryons_config\",\n", - " f\"SP_v1.4.6.3_{blind}_nautilus_config\",\n", - " f\"SP_v1.4.6.3_{blind}_planck_config\",\n", - " f\"SP_v1.4.6.3_{blind}_planck_desi_config\",\n", - "]\n", - "\n", - "legend_labels = [\n", - " r\"UNIONS-3500 $\\xi_{\\pm}(\\theta)$ (This work)\",\n", - " r\"UNIONS-3500 $C_\\ell$ (Guerrini et al. 2026)\",\n", - " r\"$\\textit{Planck}$ 2018\",\n", - " r\"DES Y6 $\\xi_{\\pm}$, NLA\",\n", - " r\"KiDS-Legacy Bandpowers ($C_{\\rm E}$)\",\n", - " r\"KiDS-Legacy COSEBIs ($E_n$)\",\n", - " r\"KiDS-Legacy $\\xi_{\\pm}(\\theta)$\",\n", - " r\"HSC-Y3 $\\xi_{\\pm}(\\theta)$\",\n", - " r\"HSC-Y3 $C_\\ell$\",\n", - " r\"$\\xi_+$ small scales, $\\theta$=[5,83] arcmin\",\n", - " r\"Flat $\\alpha_{\\rm{PSF}}$ and $\\beta_{\\rm{PSF}}$ priors\",\n", - " r\"No $\\xi^{\\rm sys}_{\\pm}$\",\n", - " r\"No leakage correction\",\n", - " r\"Flat $\\Delta z$ priors\",\n", - " r\"No $\\Delta z$\",\n", - " r\"Flat $A_{\\rm IA}$ prior\",\n", - " r\"No $A_{\\rm IA}$\",\n", - " r\"No $m$ bias\",\n", - " r\"Unmasked covmat\",\n", - " r\"$\\texttt{Halofit}$\",\n", - " r\"$\\texttt{HMCode}$ no baryons\",\n", - " r\"Nautilus sampler\",\n", - " r\"UNIONS-3500 + $\\textit{Planck}$\",\n", - " r\"UNIONS-3500 + $\\textit{Planck}$ + DESI BAO\",\n", - "]\n", - "\n", - "categories = [\n", - " \"configuration\",\n", - " \"harmonic\",\n", - " \"external\",\n", - " \"external\",\n", - " \"external\",\n", - " \"external\",\n", - " \"external\",\n", - " \"external\",\n", - " \"external\",\n", - " \"configuration\",\n", - " \"configuration\",\n", - " \"configuration\",\n", - " \"configuration\",\n", - " \"configuration\",\n", - " \"configuration\",\n", - " \"configuration\",\n", - " \"configuration\",\n", - " \"configuration\",\n", - " \"configuration\",\n", - " \"configuration\",\n", - " \"configuration\",\n", - " \"configuration\",\n", - " \"configuration\",\n", - " \"configuration\",\n", - "]\n", - "colours = [\n", - " \"darkorange\",\n", - " \"royalblue\",\n", - " \"violet\",\n", - " \"black\",\n", - " \"black\",\n", - " \"black\",\n", - " \"black\",\n", - " \"black\",\n", - " \"black\",\n", - " \"forestgreen\",\n", - " \"forestgreen\",\n", - " \"forestgreen\",\n", - " \"forestgreen\",\n", - " \"forestgreen\",\n", - " \"forestgreen\",\n", - " \"forestgreen\",\n", - " \"forestgreen\",\n", - " \"forestgreen\",\n", - " \"forestgreen\",\n", - " \"forestgreen\",\n", - " \"forestgreen\",\n", - " \"forestgreen\",\n", - " \"forestgreen\",\n", - " \"forestgreen\",\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ - "chains = []\n", - "for i, root in enumerate(roots):\n", - " category = categories[i]\n", - " if root == \"DES Y6\":\n", - " continue\n", - " if category != \"external\":\n", - " if category == \"configuration\":\n", - " path_samples = os.path.join(root_dir, f\"{root}/samples_{root}.txt\")\n", - " path_getdist = os.path.join(root_dir, f\"{root}/getdist_{root}\")\n", - " elif category == \"harmonic\":\n", - " path_samples = os.path.join(\n", - " root_dir, f\"{root}/{root}/samples_{root}_cell.txt\"\n", - " )\n", - " path_getdist = os.path.join(root_dir, f\"{root}/{root}/getdist_{root}\")\n", - " elif category == \"external_compute_sample\":\n", - " path_samples = os.path.join(root_dir, f\"ext_data/{root}/samples_{root}.txt\")\n", - " path_getdist = os.path.join(root_dir, f\"ext_data/{root}/getdist_{root}\")\n", - " else:\n", - " raise ValueError(f\"The category, {category}, of {root} is not correct\")\n", - " if \"nautilus\" not in root:\n", - " cp.load_samples_and_write_paramnames(\n", - " path_samples, path_getdist + \".paramnames\"\n", - " )\n", - " cp.write_samples_getdist_format(path_samples, path_getdist + \".txt\")\n", - " else:\n", - " cp.load_samples_and_write_paramnames(\n", - " path_samples, path_getdist + \".paramnames\", chain_type=\"nautilus\"\n", - " )\n", - " cp.write_samples_getdist_format(\n", - " path_samples, path_getdist + \".txt\", chain_type=\"nautilus\"\n", - " )\n", - " chains.append(cp.load_chain(path_getdist, smoothing_scale=0.5))\n", - " else:\n", - " path_getdist = os.path.join(root_dir, f\"ext_data/{root}/getdist_{root}\")\n", - " chains.append(cp.load_chain(path_getdist))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "name_list = [\n", - " \"OMEGA_M\",\n", - " \"ombh2\",\n", - " \"h0\",\n", - " \"n_s\",\n", - " \"SIGMA_8\",\n", - " \"S_8\",\n", - " \"s_8_input\",\n", - " \"logt_agn\",\n", - " \"a\",\n", - " \"m1\",\n", - " \"bias_1\",\n", - "]\n", - "label_list = [\n", - " r\"\\Omega_{\\rm m}\",\n", - " r\"\\omega_b h^2\",\n", - " r\"h_0\",\n", - " r\"n_s\",\n", - " r\"\\sigma_8\",\n", - " r\"S_8\",\n", - " r\"S_8\",\n", - " r\"\\log T_{\\rm AGN}\",\n", - " r\"A_{\\rm IA}\",\n", - " r\"m_1\",\n", - " r\"\\Delta z_1\",\n", - "]\n", - "\n", - "for i, chain in enumerate(chains):\n", - " print(legend_labels[i])\n", - " param_names = chain.getParamNames()\n", - " for name, label in zip(name_list, label_list):\n", - " try:\n", - " param_names.parWithName(name).label = label\n", - " except Exception:\n", - " warnings.warn(f\"Parameter {name} not found in chain {roots[i]}.\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "# Micro management of external chains\n", - "\n", - "# Account for the missing parameter conventions\n", - "\n", - "idx = roots.index(\"KiDS-Legacy_xipm\")\n", - "cp.derive_parameter_S8(chains[idx])\n", - "\n", - "idx = roots.index(\"KiDS-Legacy_bandpowers\")\n", - "cp.derive_parameter_S8(chains[idx])\n", - "\n", - "idx = roots.index(\"KiDS-Legacy_cosebis\")\n", - "cp.derive_parameter_S8(chains[idx])\n", - "\n", - "# OMEGA_M not in HSC_Y3_cell\n", - "idx = roots.index(\"HSC_Y3_cell\")\n", - "cp.adjust_paramname_chain(chains[idx], \"omega_m\", \"OMEGA_M\", r\"\\Omega_{\\rm m}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5", - "metadata": {}, - "outputs": [], - "source": [ - "param_values = np.array(\n", - " [\n", - " \"# Expt\",\n", - " \"Colour\",\n", - " \"S8_Mean\",\n", - " \"S8_low\",\n", - " \"S8_high\",\n", - " \"sigma_8_Mean\",\n", - " \"sigma_8_low\",\n", - " \"sigma_8_high\",\n", - " \"Omega_m_Mean\",\n", - " \"Omega_m_low\",\n", - " \"Omega_m_high\",\n", - " ]\n", - ")\n", - "escaped = np.char.replace(legend_labels, \"\\\\\", \"\\\\\\\\\")\n", - "\n", - "for i, root in enumerate(roots):\n", - " chain = chains[i]\n", - " if root == \"DES Y6\":\n", - " param_values = np.vstack(\n", - " (\n", - " param_values,\n", - " [\n", - " escaped[i],\n", - " colours[i],\n", - " 0.798,\n", - " 0.015,\n", - " 0.014,\n", - " 0.763,\n", - " 0.057,\n", - " 0.050,\n", - " 0.332,\n", - " 0.040,\n", - " 0.035,\n", - " ],\n", - " )\n", - " )\n", - " else:\n", - " best_fit_params = cp.extract_best_fit_params(chain, best_fit_method=\"2Dkde\")\n", - " margestats = chain.getMargeStats()\n", - "\n", - " s8_stats = margestats.parWithName(\"S_8\")\n", - " sigma8_stats = margestats.parWithName(\"SIGMA_8\")\n", - " omegam_stats = margestats.parWithName(\"OMEGA_M\")\n", - "\n", - " param_values = np.vstack(\n", - " (\n", - " param_values,\n", - " [\n", - " escaped[i],\n", - " colours[i],\n", - " best_fit_params[\"S_8\"],\n", - " best_fit_params[\"S_8\"] - s8_stats.limits[0].lower,\n", - " s8_stats.limits[0].upper - best_fit_params[\"S_8\"],\n", - " best_fit_params[\"SIGMA_8\"],\n", - " best_fit_params[\"SIGMA_8\"] - sigma8_stats.limits[0].lower,\n", - " sigma8_stats.limits[0].upper - best_fit_params[\"SIGMA_8\"],\n", - " best_fit_params[\"OMEGA_M\"],\n", - " best_fit_params[\"OMEGA_M\"] - omegam_stats.limits[0].lower,\n", - " omegam_stats.limits[0].upper - best_fit_params[\"OMEGA_M\"],\n", - " ],\n", - " )\n", - " )\n", - "print(param_values)\n", - "np.savetxt(\n", - " f\"{root_dir}/param_values.txt\",\n", - " param_values,\n", - " fmt=[\"%s\" for i in range(11)],\n", - " delimiter=\";\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "# Load the value of the parameters\n", - "cosmo = np.loadtxt(\n", - " f\"{root_dir}/param_values.txt\",\n", - " dtype={\n", - " \"names\": (\n", - " \"Expt\",\n", - " \"colour\",\n", - " \"s8_mean\",\n", - " \"s8_low\",\n", - " \"s8_high\",\n", - " \"sigma8_mean\",\n", - " \"sigma8_low\",\n", - " \"sigma8_high\",\n", - " \"omegam_mean\",\n", - " \"omegam_low\",\n", - " \"omegam_high\",\n", - " ),\n", - " \"formats\": (\n", - " \"U250\",\n", - " \"U20\",\n", - " \"U20\",\n", - " \"U20\",\n", - " \"U20\",\n", - " \"U20\",\n", - " \"U20\",\n", - " \"U20\",\n", - " \"U20\",\n", - " \"U20\",\n", - " \"U20\",\n", - " ),\n", - " },\n", - " skiprows=1,\n", - " delimiter=\";\",\n", - ")\n", - "expt = np.char.replace(cosmo[\"Expt\"], \"\\\\\\\\\", \"\\\\\")\n", - "colours = cosmo[\"colour\"]\n", - "s8_mean = cosmo[\"s8_mean\"].astype(np.float64)\n", - "s8_low = cosmo[\"s8_low\"].astype(np.float64)\n", - "s8_high = cosmo[\"s8_high\"].astype(np.float64)\n", - "sigma8_mean = cosmo[\"sigma8_mean\"].astype(np.float64)\n", - "sigma8_low = cosmo[\"sigma8_low\"].astype(np.float64)\n", - "sigma8_high = cosmo[\"sigma8_high\"].astype(np.float64)\n", - "omegam_mean = cosmo[\"omegam_mean\"].astype(np.float64)\n", - "omegam_low = cosmo[\"omegam_low\"].astype(np.float64)\n", - "omegam_high = cosmo[\"omegam_high\"].astype(np.float64)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [ - "from matplotlib.gridspec import GridSpec\n", - "\n", - "fig = plt.figure(figsize=(13, 8))\n", - "gs = GridSpec(1, 3, width_ratios=[1, 0.5, 0.5])\n", - "ax1 = fig.add_subplot(gs[0])\n", - "ax2 = fig.add_subplot(gs[1], sharey=ax1)\n", - "ax3 = fig.add_subplot(gs[2], sharey=ax1)\n", - "\n", - "axs = [ax1, ax2, ax3]\n", - "\n", - "params = [\n", - " (s8_mean, s8_low, s8_high, r\"$S_8$\"),\n", - " (sigma8_mean, sigma8_low, sigma8_high, r\"$\\sigma_8$\"),\n", - " (omegam_mean, omegam_low, omegam_high, r\"$\\Omega_{\\rm m}$\"),\n", - "]\n", - "reference = r\"UNIONS-3500 $\\xi_{\\pm}(\\theta)$ (This work)\"\n", - "\n", - "separation_after = [\n", - " r\"UNIONS-3500 $C_\\ell$ (Guerrini et al. 2026)\",\n", - " r\"HSC-Y3 $C_\\ell$\",\n", - " r\"$\\xi_+$ small scales, $\\theta$=[5,83] arcmin\",\n", - " r\"Unmasked covmat\",\n", - " r\"$\\texttt{HMCode}$ no baryons\",\n", - " r\"Nautilus sampler\",\n", - "]\n", - "list_section_index = [r\"(ii)\", r\"(iii)\", r\"(iv)\", r\"(v)\", r\"(vi)\", r\"(vii)\"]\n", - "\n", - "preliminary_watermark = False\n", - "blind_axes = False\n", - "row_spacing = 0.2\n", - "\n", - "index_ref = np.where(expt == reference)[0][0]\n", - "\n", - "y = np.arange(len(expt))\n", - "for ax, param in zip(axs, params):\n", - " means, lows, highs, label = param\n", - " for i, mean, low, high, color in zip(y, means, lows, highs, colours):\n", - " ax.errorbar(\n", - " mean,\n", - " 0.05 + i * row_spacing,\n", - " xerr=np.array([low, high])[:, None],\n", - " fmt=\"o\",\n", - " color=color,\n", - " ecolor=color,\n", - " elinewidth=2,\n", - " capsize=3,\n", - " )\n", - " ax.set_xlabel(label, fontsize=14)\n", - "\n", - " ax.grid(False)\n", - " ax.tick_params(axis=\"y\", left=False, labelleft=False)\n", - " if label == r\"$S_8$\":\n", - " ax.axvspan(\n", - " s8_mean[index_ref] - s8_low[index_ref],\n", - " s8_mean[index_ref] + s8_high[index_ref],\n", - " color=colours[index_ref],\n", - " alpha=0.2,\n", - " )\n", - " ax.set_xlim(0.6, 1.35)\n", - " if blind_axes:\n", - " ref_tick = np.mean(s8_mean[:4])\n", - " ax.set_xticks([ref_tick + i * 0.1 for i in range(-5, 5)], labels=[])\n", - " elif label == r\"$\\sigma_8$\":\n", - " ax.axvspan(\n", - " sigma8_mean[index_ref] - sigma8_low[index_ref],\n", - " sigma8_mean[index_ref] + sigma8_high[index_ref],\n", - " color=colours[index_ref],\n", - " alpha=0.2,\n", - " )\n", - " ax.set_xlim(0.5, 1.35)\n", - " if blind_axes:\n", - " ref_tick = np.mean(sigma8_mean[:4])\n", - " ax.set_xticks([ref_tick + i * 0.2 for i in range(-2, 2)], labels=[])\n", - " elif label == r\"$\\Omega_{\\rm m}$\":\n", - " ax.axvspan(\n", - " omegam_mean[index_ref] - omegam_low[index_ref],\n", - " omegam_mean[index_ref] + omegam_high[index_ref],\n", - " color=colours[index_ref],\n", - " alpha=0.2,\n", - " )\n", - " ax.set_xlim(0.1, 0.5)\n", - " if blind_axes:\n", - " ref_tick = np.mean(omegam_mean[:4])\n", - " ax.set_xticks([ref_tick + i * 0.1 for i in range(-2, 3)], labels=[])\n", - "\n", - "\n", - "ax1.set_yticks(0.01 + y * row_spacing)\n", - "ax1.set_yticklabels([])\n", - "for label, color in zip(expt, colours):\n", - " if \"This work\" in label:\n", - " label_bold = (\n", - " r\"$\\bf{UNIONS}$-$\\bf{3500}$ $\\xi_{\\pm}(\\theta)$ $\\bf{(This\\ work)}$\"\n", - " )\n", - " ax1.text(\n", - " -0.6,\n", - " 0.05 + row_spacing * np.where(expt == label)[0][0],\n", - " label_bold,\n", - " fontsize=12,\n", - " ha=\"left\",\n", - " va=\"center\",\n", - " color=color,\n", - " )\n", - " else:\n", - " ax1.text(\n", - " -0.6,\n", - " 0.05 + row_spacing * np.where(expt == label)[0][0],\n", - " label,\n", - " fontsize=12,\n", - " ha=\"left\",\n", - " va=\"center\",\n", - " color=color,\n", - " )\n", - " if label != reference:\n", - " index = np.where(expt == label)[0][0]\n", - " s8_tension = cp.get_sigma_tension(\n", - " s8_mean[index],\n", - " s8_low[index],\n", - " s8_high[index],\n", - " s8_mean[index_ref],\n", - " s8_low[index_ref],\n", - " s8_high[index_ref],\n", - " )\n", - " sign_str = \"+\" if s8_tension > 0 else \"-\"\n", - " ax1.text(\n", - " 1.32,\n", - " 0.05 + row_spacing * index,\n", - " rf\"${sign_str}{np.abs(s8_tension):.2f}\" + r\"\\, \\sigma$\",\n", - " fontsize=10,\n", - " ha=\"right\",\n", - " va=\"center\",\n", - " color=color,\n", - " )\n", - "# Add separation lines\n", - "for i, sep in enumerate(separation_after):\n", - " print(sep)\n", - " index_sep = np.where(expt == sep)[0][0]\n", - " ax2.axhline(\n", - " row_spacing * (index_sep + 1) - 0.07,\n", - " color=\"black\",\n", - " linestyle=\"dotted\",\n", - " linewidth=1,\n", - " )\n", - " ax3.axhline(\n", - " row_spacing * (index_sep + 1) - 0.07,\n", - " color=\"black\",\n", - " linestyle=\"dotted\",\n", - " linewidth=1,\n", - " )\n", - " ax1.axhline(\n", - " row_spacing * (index_sep + 1) - 0.07,\n", - " xmin=-1.8,\n", - " color=\"black\",\n", - " linestyle=\"dotted\",\n", - " linewidth=1,\n", - " clip_on=False,\n", - " )\n", - " ax1.text(\n", - " -0.61,\n", - " row_spacing * (index_sep + 1) + 0.05,\n", - " list_section_index[i],\n", - " fontsize=12,\n", - " fontweight=\"bold\",\n", - " va=\"center\",\n", - " ha=\"right\",\n", - " )\n", - "\n", - "\n", - "# --- Add section label (i)) ---\n", - "ax1.text(-0.61, 0.05, r\"(i)\", fontsize=12, fontweight=\"bold\", va=\"center\", ha=\"right\")\n", - "\n", - "if preliminary_watermark:\n", - " plt.figtext(\n", - " 0.5,\n", - " 0.5,\n", - " \"PRELIMINARY\",\n", - " fontsize=50,\n", - " color=\"gray\",\n", - " ha=\"center\",\n", - " va=\"center\",\n", - " alpha=0.3,\n", - " rotation=330,\n", - " )\n", - "\n", - "plt.gca().invert_yaxis()\n", - "\n", - "plt.tight_layout()\n", - "\n", - "# plt.savefig(\"./plots/whisker_plot.png\", dpi=300)\n", - "# #Save pdf\n", - "plt.savefig(\"../Plots/S8_whisker_plot.pdf\", bbox_inches=\"tight\")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "my_env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/best_fit_xipm.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/best_fit_xipm.ipynb deleted file mode 100644 index d6ed4c01..00000000 --- a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/best_fit_xipm.ipynb +++ /dev/null @@ -1,607 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "0", - "metadata": {}, - "source": [ - "# Best-fit $\\xi_\\pm$\n", - "\n", - "This notebook plots the best-fit 2PCFs for the fiducial and other cases" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import sys\n", - "\n", - "sys.path.append(\"/home/guerrini/sp_validation/cosmo_inference/scripts\")\n", - "\n", - "import chain_postprocessing as cp\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib.scale as mscale\n", - "import numpy as np\n", - "import seaborn as sns\n", - "from astropy.io import fits\n", - "from getdist import plots\n", - "\n", - "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", - "\n", - "from sp_validation.rho_tau import SquareRootScale\n", - "\n", - "mscale.register_scale(SquareRootScale)\n", - "\n", - "plt.rcParams[\"text.usetex\"] = True\n", - "\n", - "sns.set_palette(\"husl\")\n", - "\n", - "g = plots.get_subplot_plotter(width_inch=30)\n", - "g.settings.axes_fontsize = 40\n", - "g.settings.axes_labelsize = 40\n", - "g.settings.alpha_filled_add = 0.7\n", - "g.settings.legend_fontsize = 50\n", - "\n", - "# Directory where the chains are located\n", - "root_dir = \"/n09data/guerrini/output_chains\"\n", - "\n", - "# THE BLIND TO USE FOR THE PLOTS\n", - "blind = \"B\"\n", - "catalog_version = \"SP_v1.4.6.3\"\n", - "fiducial_root_cell = f\"SP_v1.4.6.3_leak_corr_{blind}\"\n", - "label_fiducial_cell = r\"UNIONS $C_{\\ell}$\"\n", - "fiducial_root_xi_data = f\"SP_v1.4.6.3_leak_corr_{blind}_masked\"\n", - "fiducial_root_xi_chains = f\"SP_v1.4.6.3_{blind}_fiducial_config\"\n", - "label_fiducial_xi = r\"UNIONS $\\xi_{\\pm}$\"\n", - "\n", - "# Path to the ini files used\n", - "path_ini_files = \"/home/guerrini/sp_validation/cosmo_inference/cosmosis_config\"\n", - "path_datavectors = \"/home/guerrini/sp_validation/cosmo_inference/data/\"\n", - "path_output_chains = \"/n09data/guerrini/output_chains/\"\n", - "\n", - "\n", - "data_cell = fits.open(\n", - " os.path.join(\n", - " path_datavectors, f\"{fiducial_root_cell}/cosmosis_{fiducial_root_cell}.fits\"\n", - " )\n", - ")\n", - "\n", - "data_xi = fits.open(\n", - " os.path.join(\n", - " path_datavectors,\n", - " f\"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}/cosmosis_{fiducial_root_xi_data}.fits\",\n", - " )\n", - ")\n", - "\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ - "# Perform the computation for the fiducial of Cell\n", - "path_samples_fiducial_cell = os.path.join(\n", - " path_output_chains,\n", - " fiducial_root_cell,\n", - " fiducial_root_cell,\n", - " f\"samples_{fiducial_root_cell}_cell.txt\",\n", - ")\n", - "path_gd_fiducial_cell = os.path.join(\n", - " path_output_chains,\n", - " fiducial_root_cell,\n", - " fiducial_root_cell,\n", - " f\"getdist_{fiducial_root_cell}_cell\",\n", - ")\n", - "cp.load_samples_and_write_paramnames(\n", - " path_samples_fiducial_cell, path_gd_fiducial_cell + \".paramnames\"\n", - ")\n", - "cp.write_samples_getdist_format(\n", - " path_samples_fiducial_cell, path_gd_fiducial_cell + \".txt\", chain_type=\"polychord\"\n", - ")\n", - "\n", - "chain_fiducial_cell = cp.load_chain(path_gd_fiducial_cell, smoothing_scale=0.5)\n", - "\n", - "best_fit_params_fiducial_cell = cp.extract_best_fit_params(\n", - " chain_fiducial_cell, best_fit_method=\"2Dkde\"\n", - ")\n", - "\n", - "cp.compute_best_fit(\n", - " path_ini_files,\n", - " best_fit_params_fiducial_cell,\n", - " fiducial_root_cell,\n", - " is_harmonic=True,\n", - " blind=blind,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "# Perform the computation for the fiducial of xi\n", - "path_samples_fiducial_xi = os.path.join(\n", - " path_output_chains,\n", - " fiducial_root_xi_chains,\n", - " f\"samples_{fiducial_root_xi_chains}.txt\",\n", - ")\n", - "\n", - "path_gd_fiducial_xi = os.path.join(\n", - " path_output_chains, fiducial_root_xi_chains, f\"getdist_{fiducial_root_xi_chains}\"\n", - ")\n", - "cp.load_samples_and_write_paramnames(\n", - " path_samples_fiducial_xi, path_gd_fiducial_xi + \".paramnames\"\n", - ")\n", - "cp.write_samples_getdist_format(\n", - " path_samples_fiducial_xi, path_gd_fiducial_xi + \".txt\", chain_type=\"polychord\"\n", - ")\n", - "\n", - "chain_fiducial_xi = cp.load_chain(path_gd_fiducial_xi, smoothing_scale=0.5)\n", - "\n", - "best_fit_params_fiducial_xi = cp.extract_best_fit_params(\n", - " chain_fiducial_xi, best_fit_method=\"2Dkde\"\n", - ")\n", - "\n", - "ini_file_root = os.path.join(\n", - " path_ini_files,\n", - " f\"config_space_v1.4.6.3_fiducial/pipeline/blind_{blind}/fiducial.ini\",\n", - ")\n", - "cp.compute_best_fit(\n", - " path_ini_files,\n", - " best_fit_params_fiducial_xi,\n", - " fiducial_root_xi_chains,\n", - " is_harmonic=False,\n", - " blind=blind,\n", - " ini_file_root=ini_file_root,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "# Make the plot for the best-fit datavector for Cell EE\n", - "root_to_plot = [\n", - " fiducial_root_xi_chains,\n", - " fiducial_root_cell,\n", - "]\n", - "\n", - "labels = [\n", - " r\"UNIONS $\\xi_\\pm(\\theta)$\",\n", - " r\"UNIONS $C_\\ell$\",\n", - "]\n", - "\n", - "line_args = [\n", - " {\"color\": \"royalblue\", \"linestyle\": \"-\"},\n", - " {\"color\": \"orange\", \"linestyle\": \"-\"},\n", - "]\n", - "\n", - "properties = {}\n", - "\n", - "properties = cp.update_properties_w_roots(\n", - " properties, fiducial_root_cell, path_ini_files, with_configuration=False\n", - ")\n", - "properties = cp.update_properties_w_roots(\n", - " properties,\n", - " fiducial_root_xi_chains,\n", - " path_ini_files,\n", - " with_configuration=True,\n", - " path_to_this_ini=ini_file_root,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5", - "metadata": {}, - "outputs": [], - "source": [ - "root_to_plot = [fiducial_root_cell, fiducial_root_xi_chains]\n", - "labels = [r\"Best fit $C_\\ell$\", r\"Best fit $\\xi_\\pm(\\theta)$\"]\n", - "path_best_fit_xi_theta = os.path.join(\n", - " path_output_chains, fiducial_root_xi_chains, \"best_fit/shear_xi_plus/theta.txt\"\n", - ")\n", - "\n", - "theta_rad = np.loadtxt(path_best_fit_xi_theta)\n", - "theta_min = 1\n", - "theta_max = 250\n", - "\n", - "cp.compute_best_fit_xi_from_cell(\n", - " path_output_chains, fiducial_root_cell, best_fit_params_fiducial_cell, theta_rad\n", - ")\n", - "\n", - "data = fits.open(\n", - " os.path.join(\n", - " path_datavectors,\n", - " f\"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}/cosmosis_{fiducial_root_xi_data}.fits\",\n", - " )\n", - ")\n", - "bbox_to_anchor_xip = (0.685, 0.09)\n", - "bbox_to_anchor_xim = (0.3, 0.65)\n", - "xi_p_data = data[\"XI_PLUS\"].data\n", - "xi_m_data = data[\"XI_MINUS\"].data\n", - "cov_mat = data[\"COVMAT\"].data\n", - "\n", - "# Plot hyperparameter\n", - "loc_legend = \"lower center\"\n", - "\n", - "fig, [ax, ax2] = plt.subplots(1, 2, figsize=(20, 8))\n", - "\n", - "theta, xi_p, xi_m = xi_p_data[\"ANG\"], xi_p_data[\"VALUE\"], xi_m_data[\"VALUE\"]\n", - "ax.errorbar(\n", - " theta,\n", - " theta * xi_p,\n", - " yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])),\n", - " fmt=\"o\",\n", - " label=r\"UNIONS $\\xi_+$ data\",\n", - " color=\"black\",\n", - " capsize=2,\n", - ")\n", - "ax2.errorbar(\n", - " theta,\n", - " theta * xi_m,\n", - " yerr=theta\n", - " * np.sqrt(\n", - " np.diag(cov_mat[len(theta) : 2 * len(theta), len(theta) : 2 * len(theta)])\n", - " ),\n", - " fmt=\"o\",\n", - " label=r\"UNIONS $\\xi_-$ data\",\n", - " color=\"black\",\n", - " capsize=2,\n", - ")\n", - "\n", - "for idx, (label, root) in enumerate(zip(labels, root_to_plot)):\n", - " # Read the results\n", - " theta = (\n", - " (\n", - " np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " )\n", - " * 180\n", - " / np.pi\n", - " * 60\n", - " )\n", - " xi_plus = np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_minus = np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " if r\"$C_\\ell$\" not in label:\n", - " xi_sys_plus = np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - " theta_xi_sys = (\n", - " np.loadtxt(path_output_chains + \"{}/best_fit/xi_sys/theta.txt\".format(root))\n", - " * 180\n", - " / np.pi\n", - " * 60\n", - " )\n", - "\n", - " xi_sys_plus = np.interp(theta, theta_xi_sys, xi_sys_plus)\n", - " xi_sys_minus = np.interp(theta, theta_xi_sys, xi_sys_minus)\n", - " xi_plus += xi_sys_plus\n", - " xi_minus += xi_sys_minus\n", - "\n", - " mask = (theta > theta_min) & (theta < theta_max)\n", - " theta = theta[mask]\n", - " ax.plot(\n", - " theta,\n", - " theta * xi_plus[mask],\n", - " label=r\"Best fit $\\xi_+(\\theta)$\",\n", - " **line_args[idx],\n", - " lw=2.5,\n", - " )\n", - " ax.plot(\n", - " theta,\n", - " theta * xi_sys_plus[mask],\n", - " label=r\"Best fit $\\xi^{\\rm sys}_{+}(\\theta)$\",\n", - " c=\"r\",\n", - " )\n", - " ax2.plot(\n", - " theta,\n", - " theta * xi_minus[mask],\n", - " label=r\"Best fit $\\xi_-(\\theta)$\",\n", - " **line_args[idx],\n", - " lw=2.5,\n", - " )\n", - " ax2.plot(\n", - " theta,\n", - " theta * xi_sys_minus[mask],\n", - " label=r\"Best fit $\\xi^{\\rm sys}_{-}(\\theta)$\",\n", - " c=\"r\",\n", - " )\n", - "\n", - " else:\n", - " mask = (theta > theta_min) & (theta < theta_max)\n", - " theta = theta[mask]\n", - " ax.plot(theta, theta * xi_plus[mask], label=label, **line_args[idx], lw=2.5)\n", - " ax2.plot(theta, theta * xi_minus[mask], label=label, **line_args[idx], lw=2.5)\n", - "# XI PLUS PLOT SETTINGS\n", - "\n", - "# Plot the scale cuts for different k_max\n", - "ax.axvline(x=5, color=\"gray\", linestyle=\"--\", alpha=0.7)\n", - "ax.axhline(y=0, color=\"black\", linestyle=\"--\", alpha=0.7)\n", - "\n", - "ymin = ax.get_ylim()[0]\n", - "ymax = ax.get_ylim()[1]\n", - "# Shadowing cut scaled\n", - "ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color=\"gray\", alpha=0.2)\n", - "ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color=\"gray\", alpha=0.2)\n", - "\n", - "ax.set_ylim(ymin, ymax)\n", - "\n", - "# Add labels directly under the tick\n", - "ax.text(\n", - " 4.5,\n", - " 0.47e-4,\n", - " r\"$k_\\mathrm{max} = 1 h$ Mpc$^{-1}$\",\n", - " ha=\"center\",\n", - " va=\"top\",\n", - " fontsize=20,\n", - " rotation=90,\n", - ")\n", - "\n", - "ax.set_ylabel(r\"$\\theta \\xi_\\pm$\", fontsize=26)\n", - "ax.set_xlabel(r\"$\\theta$ (arcmin)\", fontsize=26)\n", - "ax.set_xlim([theta.min() - 0.1, theta.max() + 20])\n", - "ax.set_title(r\"$\\xi_+(\\theta)$\", fontsize=26)\n", - "ax.set_xscale(\"log\")\n", - "ax.set_xticks(np.array([1, 10, 100]))\n", - "ax.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", - "ax.tick_params(axis=\"both\", which=\"major\", labelsize=24)\n", - "ax.tick_params(axis=\"both\", which=\"minor\", labelsize=20)\n", - "ax.yaxis.get_offset_text().set_fontsize(24)\n", - "ax.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", - "ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=20)\n", - "\n", - "# XI_MINUS PLOT SETTINGS\n", - "\n", - "# Plot the scale cuts for different k_max\n", - "ax2.axvline(x=50, color=\"gray\", linestyle=\"--\", alpha=0.7)\n", - "ax2.axhline(y=0, color=\"black\", linestyle=\"--\", alpha=0.7)\n", - "\n", - "ymin = ax2.get_ylim()[0]\n", - "ymax = ax2.get_ylim()[1]\n", - "# Shadowing cut scaled\n", - "ax2.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color=\"gray\", alpha=0.2)\n", - "ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color=\"gray\", alpha=0.2)\n", - "\n", - "ax2.set_ylim(ymin, ymax)\n", - "\n", - "# Add labels directly under the tick\n", - "ax2.text(\n", - " 45,\n", - " 1.15e-4,\n", - " r\"$k_\\mathrm{max} = 1 h$ Mpc$^{-1}$\",\n", - " ha=\"center\",\n", - " va=\"top\",\n", - " fontsize=20,\n", - " rotation=90,\n", - ")\n", - "\n", - "# ax2.set_ylabel(r'$\\theta \\xi_-$', fontsize=16)\n", - "ax2.set_xlabel(r\"$\\theta$ (arcmin)\", fontsize=26)\n", - "ax2.set_xlim([theta.min() - 0.1, theta.max() + 20])\n", - "ax2.set_xscale(\"log\")\n", - "ax2.set_title(r\"$\\xi_-(\\theta)$\", fontsize=26)\n", - "ax2.set_xticks(np.array([1, 10, 100]))\n", - "ax2.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", - "ax2.tick_params(axis=\"both\", which=\"major\", labelsize=24)\n", - "ax2.tick_params(axis=\"both\", which=\"minor\", labelsize=20)\n", - "ax2.yaxis.get_offset_text().set_fontsize(24)\n", - "ax2.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", - "ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=20)\n", - "\n", - "plt.savefig(\n", - " \"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/best_fit_xipm_SP_v1.4.6.3_B.pdf\",\n", - " bbox_inches=\"tight\",\n", - ")\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "root_to_plot = [fiducial_root_xi_chains]\n", - "labels = [r\"Best fit $\\tau_{0,2}(\\theta)$\"]\n", - "\n", - "bbox_to_anchor_xip = (0.285, 0.7)\n", - "bbox_to_anchor_xim = (0.3, 0.65)\n", - "tau0_data = data[\"TAU_0_PLUS\"].data\n", - "tau2_data = data[\"TAU_2_PLUS\"].data\n", - "cov_mat = data[\"COVMAT\"].data\n", - "\n", - "# Plot hyperparameter\n", - "\n", - "fig, [ax, ax2] = plt.subplots(1, 2, figsize=(20, 8))\n", - "\n", - "theta, tau0, tau2 = tau0_data[\"ANG\"], tau0_data[\"VALUE\"], tau2_data[\"VALUE\"]\n", - "ax.errorbar(\n", - " theta,\n", - " theta * tau0,\n", - " yerr=theta\n", - " * np.sqrt(\n", - " np.diag(\n", - " cov_mat[2 * len(theta) : 3 * len(theta), 2 * len(theta) : 3 * len(theta)]\n", - " )\n", - " ),\n", - " fmt=\"o\",\n", - " label=r\"UNIONS $\\tau_{0,+}$\",\n", - " color=\"black\",\n", - " capsize=2,\n", - ")\n", - "ax2.errorbar(\n", - " theta,\n", - " theta * tau2,\n", - " yerr=theta\n", - " * np.sqrt(\n", - " np.diag(\n", - " cov_mat[3 * len(theta) : 4 * len(theta), 3 * len(theta) : 4 * len(theta)]\n", - " )\n", - " ),\n", - " fmt=\"o\",\n", - " label=r\"UNIONS $\\tau_{2,+}$\",\n", - " color=\"black\",\n", - " capsize=2,\n", - ")\n", - "\n", - "for idx, (label, root) in enumerate(zip(labels, root_to_plot)):\n", - " # Read the results\n", - " theta = (\n", - " (\n", - " np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/tau_0_plus/theta.txt\".format(root)\n", - " )\n", - " )\n", - " * 180\n", - " / np.pi\n", - " * 60\n", - " )\n", - " tau0_plus = np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/tau_0_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " tau2_plus = np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/tau_2_plus/bin_1_1.txt\".format(root)\n", - " )\n", - "\n", - " mask = (theta > theta_min) & (theta < theta_max)\n", - " theta = theta[mask]\n", - " ax.plot(\n", - " theta,\n", - " theta * tau0_plus[mask],\n", - " label=r\"Best fit $\\tau_{0,+}(\\theta)$\",\n", - " c=\"orange\",\n", - " lw=2.5,\n", - " )\n", - " ax2.plot(\n", - " theta,\n", - " theta * tau2_plus[mask],\n", - " label=r\"Best fit $\\tau_{2,+}(\\theta)$\",\n", - " c=\"orange\",\n", - " lw=2.5,\n", - " )\n", - "\n", - "# XI PLUS PLOT SETTINGS\n", - "\n", - "# Plot the scale cuts for different k_max\n", - "ax.axhline(y=0, color=\"black\", linestyle=\"--\", alpha=0.7)\n", - "\n", - "ymin = ax.get_ylim()[0]\n", - "ymax = ax.get_ylim()[1]\n", - "\n", - "ax.set_ylim(ymin, ymax)\n", - "\n", - "ax.set_ylabel(r\"$\\theta\\tau_{0,2}$\", fontsize=26)\n", - "ax.set_xlabel(r\"$\\theta$ (arcmin)\", fontsize=26)\n", - "ax.set_xlim([theta.min() - 0.1, theta.max() + 20])\n", - "ax.set_title(r\"$\\tau_{0,+}(\\theta)$\", fontsize=26)\n", - "ax.set_xscale(\"log\")\n", - "ax.set_xticks(np.array([1, 10, 100]))\n", - "ax.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", - "ax.tick_params(axis=\"both\", which=\"major\", labelsize=24)\n", - "ax.tick_params(axis=\"both\", which=\"minor\", labelsize=20)\n", - "ax.yaxis.get_offset_text().set_fontsize(24)\n", - "ax.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", - "ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=20)\n", - "\n", - "# XI_MINUS PLOT SETTINGS\n", - "\n", - "# Plot the scale cuts for different k_max\n", - "ax2.axhline(y=0, color=\"black\", linestyle=\"--\", alpha=0.7)\n", - "\n", - "ymin = ax2.get_ylim()[0]\n", - "ymax = ax2.get_ylim()[1]\n", - "# Shadowing cut scaled\n", - "ax2.fill_betweenx(\n", - " y=[ymin, ymax],\n", - " x1=0,\n", - " x2=12,\n", - " color=\"gray\",\n", - " alpha=0.2,\n", - " label=r\"$B$-mode informed scale cut\",\n", - ")\n", - "ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color=\"gray\", alpha=0.2)\n", - "\n", - "ax2.set_ylim(ymin, ymax)\n", - "\n", - "# ax2.set_ylabel(r'$\\theta \\xi_-$', fontsize=16)\n", - "ax2.set_xlabel(r\"$\\theta$ (arcmin)\", fontsize=26)\n", - "ax2.set_xlim([theta.min() - 0.1, theta.max() + 20])\n", - "ax2.set_xscale(\"log\")\n", - "ax2.set_title(r\"$\\tau_{2,+}(\\theta)$\", fontsize=26)\n", - "ax2.set_xticks(np.array([1, 10, 100]))\n", - "ax2.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", - "ax2.tick_params(axis=\"both\", which=\"major\", labelsize=24)\n", - "ax2.tick_params(axis=\"both\", which=\"minor\", labelsize=20)\n", - "ax2.yaxis.get_offset_text().set_fontsize(24)\n", - "ax2.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", - "ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=20)\n", - "\n", - "plt.savefig(\n", - " \"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/notebooks/Plots/best_fit_tau_02_SP_v1.4.6.3_B.pdf\",\n", - " bbox_inches=\"tight\",\n", - ")\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "jupytext": { - "cell_metadata_filter": "-all", - "main_language": "python", - "notebook_metadata_filter": "-all" - }, - "kernelspec": { - "display_name": "my_env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/contours.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/contours.ipynb deleted file mode 100644 index e95be1b5..00000000 --- a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/contours.ipynb +++ /dev/null @@ -1,950 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "0", - "metadata": {}, - "source": [ - "# 2D contour plots\n", - "\n", - "This notebook produces the plots for all the 2D contours in the results section." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1", - "metadata": {}, - "outputs": [], - "source": [ - "import os.path\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import seaborn as sns\n", - "from astropy.io import fits\n", - "from getdist import plots\n", - "\n", - "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", - "\n", - "plt.rcParams[\"text.usetex\"] = True\n", - "\n", - "sns.set_palette(\"husl\")\n", - "g = plots.get_subplot_plotter(width_inch=30)\n", - "g.settings.axes_fontsize = 70\n", - "g.settings.axes_labelsize = 80\n", - "g.settings.alpha_filled_add = 0.7\n", - "g.settings.legend_fontsize = 70\n", - "\n", - "\n", - "# SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE\n", - "\n", - "root_dir = \"/n09data/guerrini/output_chains/\"\n", - "path_datavectors = \"/home/guerrini/sp_validation/cosmo_inference/data/\"\n", - "path_output_chains = \"/n09data/guerrini/output_chains/\"\n", - "\n", - "data = fits.open(\n", - " os.path.join(\n", - " path_datavectors,\n", - " \"SP_v1.4.6.3_config/SP_v1.4.6.3_B/cosmosis_SP_v1.4.6.3_leak_corr_B_masked.fits\",\n", - " )\n", - ")\n", - "\n", - "roots_fid = {\n", - " \"SP_v1.4.6.3_leak_corr_B\": r\"UNIONS-3500 $C_\\ell$\",\n", - " \"SP_v1.4.6.3_B_fiducial_config\": r\"UNIONS-3500 $\\xi_\\pm$ (This work) \",\n", - " \"KiDS-Legacy_xipm\": r\"KiDS-Legacy $\\xi_\\pm$\",\n", - " \"HSC_Y3\": r\"HSC-Y3 $\\xi_\\pm$\",\n", - " \"Planck18\": r\"$\\textit{Planck}$ 2018\",\n", - "}\n", - "\n", - "roots_full = {\n", - " \"SP_v1.4.6.3_B_fiducial_config\": r\"UNIONS-3500 $\\xi_\\pm$ (This work) \",\n", - "}\n", - "\n", - "roots_ia = {\n", - " \"SP_v1.4.6.3_B_fiducial_config\": r\"Gaussian $A_{\\rm{IA}}$ prior\",\n", - " \"SP_v1.4.6.3_B_flat_ia_config\": r\"Flat $A_{\\rm{IA}}$ prior\",\n", - " \"SP_v1.4.6.3_B_no_ia_config\": r\"No IA\",\n", - "}\n", - "\n", - "roots_ext = {\n", - " \"SP_v1.4.6.3_B_fiducial_config\": r\"UNIONS-3500 $\\xi_\\pm$\",\n", - " \"SP_v1.4.6.3_B_planck_config\": r\"UNIONS-3500 $\\xi_\\pm$ + CMB\",\n", - " \"SP_v1.4.6.3_B_planck_desi_config\": r\"UNIONS-3500 $\\xi_\\pm$ + CMB + BAO\",\n", - " \"Planck18\": r\"$\\textit{Planck}$ 2018\",\n", - "}\n", - "\n", - "roots_dz = {\n", - " \"SP_v1.4.6.3_B_fiducial_config\": r\"Gaussian $\\Delta z$ prior\",\n", - " \"SP_v1.4.6.3_B_flat_delta_z_config\": r\"Flat $\\Delta z$ prior\",\n", - " \"SP_v1.4.6.3_B_no_delta_z_config\": r\"No $\\Delta z$ modelling\",\n", - "}\n", - "\n", - "roots_psf = {\n", - " \"SP_v1.4.6.3_B_flat_alpha_beta_config\": r\"Flat $\\alpha$ and $\\beta$ priors\",\n", - " \"SP_v1.4.6.3_B_fiducial_config\": r\"Gaussian $\\alpha$ and $\\beta$ priors\",\n", - " \"SP_v1.4.6.3_B_no_xi_sys_config\": r\"No $\\xi^{\\rm sys}$ included\",\n", - " \"SP_v1.4.6.3_B_no_leak_corr_config\": r\"No object-wise leakage correction\",\n", - "}\n", - "\n", - "roots_scale = {\n", - " \"SP_v1.4.6.3_B_fiducial_config\": r\"$\\xi_+$: $\\theta=[12,83]$\",\n", - " \"SP_v1.4.6.3_B_small_scales_config\": r\"$\\xi_+$: $\\theta=[5,83]$\",\n", - "}\n", - "\n", - "roots_nonlin = {\n", - " \"SP_v1.4.6.3_B_fiducial_config\": r\"Fiducial (\\texttt{HMCode2020}, $\\log(T_{\\rm AGN})$)\",\n", - " \"SP_v1.4.6.3_B_no_baryons_config\": r\"\\texttt{HMCode2020} no baryons\",\n", - " \"SP_v1.4.6.3_B_halofit_config\": r\"\\texttt{Halofit}\",\n", - "}\n", - "roots = roots_ext" - ] - }, - { - "cell_type": "markdown", - "id": "2", - "metadata": {}, - "source": [ - "## Retrieve the chains" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "# READ CHAIN\n", - "\n", - "chains = []\n", - "\n", - "for i, root in enumerate(list(roots.keys())):\n", - " burnin = 0\n", - " if \"SP\" not in root:\n", - " chain = g.samples_for_root(\n", - " root_dir + \"ext_data/{}/getdist_{}\".format(root, root),\n", - " cache=False,\n", - " settings={\n", - " \"ignore_rows\": burnin,\n", - " # 'smooth_scale_2D':0.2,\n", - " # 'smooth_scale_1D':0.2\n", - " },\n", - " )\n", - " p = chain.getParams()\n", - " if hasattr(p, \"S_8\") == False:\n", - " omega_m = chain.getParams().OMEGA_M\n", - " sigma_8 = chain.getParams().SIGMA_8\n", - "\n", - " s_8 = sigma_8 * (omega_m / 0.3) ** 0.5\n", - "\n", - " chain.addDerived(s_8, name=\"S_8\", label=r\"S_8\")\n", - "\n", - " p = chain.paramNames.parWithName(\"S_8\")\n", - "\n", - " elif \"config\" in root:\n", - " if os.path.isfile(root_dir + \"{}/getdist_{}.txt\".format(root, root)) == False:\n", - " samples = np.loadtxt(root_dir + \"{}/samples_{}.txt\".format(root, root))\n", - "\n", - " if \"nautilus\" in root:\n", - " weights = np.exp(samples[:, -3])\n", - " neglogL = samples[:, -2] - samples[:, -1]\n", - "\n", - " samples = np.column_stack((weights, neglogL, samples[:, 0:-3]))\n", - " elif \"mh\" in root:\n", - " samples = np.column_stack(\n", - " (\n", - " np.ones_like(samples[:, -1]),\n", - " np.log(samples[:, -1]) - np.log(samples[:, -2]),\n", - " samples[:, 0:-2],\n", - " )\n", - " )\n", - " burnin = 0.3\n", - " else:\n", - " samples = np.column_stack(\n", - " (samples[:, -1], samples[:, -3], samples[:, 0:-4])\n", - " )\n", - "\n", - " np.savetxt(root_dir + \"{}/getdist_{}.txt\".format(root, root), samples)\n", - "\n", - " chain = g.samples_for_root(\n", - " root_dir + \"{}/getdist_{}\".format(root, root),\n", - " cache=False,\n", - " settings={\n", - " \"ignore_rows\": burnin,\n", - " # 'smooth_scale_2D':0.2,\n", - " # 'smooth_scale_1D':0.2\n", - " },\n", - " )\n", - " else:\n", - " if (\n", - " os.path.isfile(\n", - " root_dir + \"{}/{}/getdist_{}_cell.txt\".format(root, root, root)\n", - " )\n", - " == False\n", - " ):\n", - " samples = np.loadtxt(\n", - " root_dir + \"{}/{}/samples_{}_cell.txt\".format(root, root, root)\n", - " )\n", - "\n", - " if \"nautilus\" in root:\n", - " weights = np.exp(samples[:, -3])\n", - " neglogL = samples[:, -2] - samples[:, -1]\n", - "\n", - " samples = np.column_stack((weights, neglogL, samples[:, 0:-3]))\n", - " elif \"mh\" in root:\n", - " samples = np.column_stack(\n", - " (\n", - " np.ones_like(samples[:, -1]),\n", - " np.log(samples[:, -1]) - np.log(samples[:, -2]),\n", - " samples[:, 0:-2],\n", - " )\n", - " )\n", - " burnin = 0.3\n", - " else:\n", - " samples = np.column_stack(\n", - " (samples[:, -1], samples[:, -3], samples[:, 0:-4])\n", - " )\n", - "\n", - " np.savetxt(\n", - " root_dir + \"{}/{}/getdist_{}_cell.txt\".format(root, root, root), samples\n", - " )\n", - "\n", - " chain = g.samples_for_root(\n", - " root_dir + \"{}/{}/getdist_{}_cell\".format(root, root, root),\n", - " cache=False,\n", - " settings={\n", - " \"ignore_rows\": burnin,\n", - " # 'smooth_scale_2D':0.2,\n", - " # 'smooth_scale_1D':0.2\n", - " },\n", - " )\n", - " p = chain.getParams()\n", - "\n", - " chains.append(chain)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "name_list = [\n", - " \"OMEGA_M\",\n", - " \"ombh2\",\n", - " \"h0\",\n", - " \"n_s\",\n", - " \"SIGMA_8\",\n", - " \"S_8\",\n", - " \"logt_agn\",\n", - " \"a\",\n", - " \"m1\",\n", - " \"bias_1\",\n", - " \"alpha\",\n", - " \"beta\",\n", - " \"omch2\",\n", - "]\n", - "label_list = [\n", - " r\"\\Omega_{\\rm m}\",\n", - " r\"\\omega_{\\rm b}\",\n", - " r\"h\",\n", - " r\"n_{\\rm s}\",\n", - " r\"\\sigma_8\",\n", - " r\"S_8\",\n", - " r\"\\log T_{\\rm AGN}\",\n", - " r\"A_{\\rm IA}\",\n", - " r\"m_1\",\n", - " r\"\\Delta z\",\n", - " r\"\\alpha_{\\rm PSF}\",\n", - " r\"\\beta_{\\rm PSF}\",\n", - " r\"\\omega_{\\rm c}\",\n", - "]\n", - "\n", - "for chain in chains:\n", - " param_names = chain.getParamNames()\n", - " p = chain.getParams()\n", - " for name, label in zip(name_list, label_list):\n", - " if hasattr(p, name):\n", - " param_names.parWithName(name).label = label\n", - "\n", - "legend_labels = list(roots.values())" - ] - }, - { - "cell_type": "markdown", - "id": "5", - "metadata": {}, - "source": [ - "## Plot the chains" - ] - }, - { - "cell_type": "markdown", - "id": "6", - "metadata": {}, - "source": [ - "### FIDUCIAL PLOT" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [ - "%matplotlib inline\n", - "\n", - "colours = [\n", - " \"royalblue\",\n", - " \"orange\",\n", - " \"crimson\",\n", - " \"forestgreen\",\n", - " \"indigo\",\n", - "]\n", - "\n", - "linestyle = [\"solid\", \"solid\", \"solid\", \"solid\", \"solid\"]\n", - "\n", - "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", - "\n", - "# FIDUCIAL PLOT\n", - "g.triangle_plot(\n", - " chains,\n", - " [\"SIGMA_8\", \"S_8\", \"OMEGA_M\"], #\n", - " legend_labels=legend_labels,\n", - " line_args=line_args,\n", - " contour_colors=colours,\n", - " label_order=[1, 0, 2, 3, 4],\n", - " filled=[True, True, False, False, True],\n", - ")\n", - "\n", - "g.export(\"../Plots/SP_v1.4.6.3_B_fiducial_config_contour_plot.pdf\")" - ] - }, - { - "cell_type": "markdown", - "id": "8", - "metadata": {}, - "source": [ - "### FULL PLOT" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9", - "metadata": {}, - "outputs": [], - "source": [ - "%matplotlib inline\n", - "\n", - "g.settings.axes_fontsize = 40\n", - "g.settings.axes_labelsize = 50\n", - "\n", - "colours = [\n", - " \"orange\",\n", - "]\n", - "\n", - "linestyle = [\n", - " \"solid\",\n", - "]\n", - "\n", - "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", - "\n", - "# FIDUCIAL PLOT\n", - "g.triangle_plot(\n", - " chains,\n", - " [\n", - " \"OMEGA_M\",\n", - " \"ombh2\",\n", - " \"h0\",\n", - " \"n_s\",\n", - " \"SIGMA_8\",\n", - " \"S_8\",\n", - " \"logt_agn\",\n", - " \"a\",\n", - " \"m1\",\n", - " \"bias_1\",\n", - " ],\n", - " legend_labels=legend_labels,\n", - " line_args=line_args,\n", - " contour_colors=colours,\n", - " filled=True,\n", - ")\n", - "\n", - "g.export(\"../Plots/SP_v1.4.6.3_B_fiducial_config_contour_plot_full.pdf\")" - ] - }, - { - "cell_type": "markdown", - "id": "10", - "metadata": {}, - "source": [ - "### IA PLOT" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11", - "metadata": {}, - "outputs": [], - "source": [ - "colours = [\n", - " \"orange\",\n", - " \"royalblue\",\n", - " \"forestgreen\",\n", - "]\n", - "\n", - "linestyle = [\n", - " \"solid\",\n", - " \"solid\",\n", - " \"solid\",\n", - "]\n", - "\n", - "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", - "\n", - "g.triangle_plot(\n", - " chains,\n", - " [\"S_8\", \"OMEGA_M\", \"a\"], #\n", - " legend_labels=legend_labels,\n", - " line_args=line_args,\n", - " contour_args={\"alpha\": 0.6},\n", - " contour_colors=colours,\n", - " filled=[True, False, True],\n", - ")\n", - "\n", - "g.export(\"../Plots/SP_v1.4.6.3_B_fiducial_config_contour_plot_ia.pdf\")" - ] - }, - { - "cell_type": "markdown", - "id": "12", - "metadata": {}, - "source": [ - "### PSF PLOT" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "13", - "metadata": {}, - "outputs": [], - "source": [ - "colours = [\n", - " \"royalblue\",\n", - " \"orange\",\n", - " \"hotpink\",\n", - " \"slategray\",\n", - "]\n", - "\n", - "linestyle = [\n", - " \"solid\",\n", - " \"solid\",\n", - " \"solid\",\n", - " \"solid\",\n", - "]\n", - "\n", - "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", - "\n", - "g.triangle_plot(\n", - " chains,\n", - " [\"S_8\", \"OMEGA_M\", \"alpha\", \"beta\"], #\n", - " legend_labels=legend_labels,\n", - " line_args=line_args,\n", - " contour_args=[{\"alpha\": 1}, {\"alpha\": 0.6}, {\"alpha\": 0.8}, {\"alpha\": 0.8}],\n", - " contour_colors=colours,\n", - " legend_loc=\"upper right\",\n", - " label_order=[1, 0, 2, 3],\n", - " filled=[False, True, True, True],\n", - ")\n", - "\n", - "g.subplots[3, 2].scatter(\n", - " 0.005, 0.81, color=\"k\", marker=\"X\", s=400, label=\"Fiducial config best-fit\"\n", - ")\n", - "g.subplots[3, 2].scatter(\n", - " 0.022, 0.798, color=\"k\", marker=\"P\", s=400, label=\"Fiducial config best-fit\"\n", - ")\n", - "\n", - "g.export(\"../Plots/SP_v1.4.6.3_B_fiducial_config_contour_plot_psf.pdf\")" - ] - }, - { - "cell_type": "markdown", - "id": "14", - "metadata": {}, - "source": [ - "### DELTA Z PLOT" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15", - "metadata": {}, - "outputs": [], - "source": [ - "colours = [\n", - " \"orange\",\n", - " \"royalblue\",\n", - " \"indigo\",\n", - "]\n", - "\n", - "linestyle = [\n", - " \"solid\",\n", - " \"solid\",\n", - " \"solid\",\n", - "]\n", - "\n", - "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", - "g.triangle_plot(\n", - " chains,\n", - " [\"S_8\", \"OMEGA_M\", \"bias_1\"], #\n", - " legend_labels=legend_labels,\n", - " line_args=line_args,\n", - " contour_args=[{\"alpha\": 1.0}, {\"alpha\": 0.9}, {\"alpha\": 0.5}],\n", - " contour_colors=colours,\n", - " filled=[True, False, True],\n", - ")\n", - "\n", - "g.export(\"../Plots/SP_v1.4.6.3_B_fiducial_config_contour_plot_dz.pdf\")" - ] - }, - { - "cell_type": "markdown", - "id": "16", - "metadata": {}, - "source": [ - "### EXTERNAL DATA" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "17", - "metadata": {}, - "outputs": [], - "source": [ - "colours = [\n", - " \"orange\",\n", - " \"royalblue\",\n", - " \"crimson\",\n", - " \"forestgreen\",\n", - "]\n", - "\n", - "linestyle = [\n", - " \"solid\",\n", - " \"solid\",\n", - " \"solid\",\n", - " \"solid\",\n", - " \"solid\",\n", - "]\n", - "\n", - "line_args = [dict(color=col, ls=ls) for col, ls in zip(colours, linestyle)]\n", - "\n", - "g = plots.get_subplot_plotter(width_inch=10)\n", - "g.settings.axes_fontsize = 25\n", - "g.settings.axes_labelsize = 25\n", - "g.settings.legend_fontsize = 22\n", - "\n", - "g.plot_2d(\n", - " chains,\n", - " [\"S_8\", \"OMEGA_M\", \"SIGMA_8\"], #\n", - " line_args=line_args,\n", - " contour_colors=colours,\n", - " legend_labels=legend_labels,\n", - " alphas=[0.7, 1.0, 1.0, 1.0],\n", - " filled=[True, True, True, False],\n", - ")\n", - "\n", - "g.add_y_bands(0.2975, 0.0086, alpha2=0, color=\"k\", label=\"BAO\")\n", - "g.add_legend(legend_labels, legend_loc=\"upper right\")\n", - "\n", - "g.export(\"../Plots/SP_v1.4.6.3_B_fiducial_config_contour_plot_ext.pdf\")" - ] - }, - { - "cell_type": "markdown", - "id": "18", - "metadata": {}, - "source": [ - "### Small scales" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "19", - "metadata": {}, - "outputs": [], - "source": [ - "%matplotlib inline\n", - "\n", - "colours = [\n", - " \"orange\",\n", - " \"dodgerblue\",\n", - "]\n", - "\n", - "linestyle = [\n", - " \"solid\",\n", - " \"solid\",\n", - "]\n", - "\n", - "line_args = [dict(color=col, ls=ls) for col, ls in zip(colours, linestyle)]\n", - "\n", - "g = plots.get_subplot_plotter(width_inch=9)\n", - "g.settings.axes_fontsize = 25\n", - "g.settings.axes_labelsize = 25\n", - "g.settings.alpha_filled_add = 0.7\n", - "g.settings.legend_fontsize = 30\n", - "\n", - "g.plot_2d(\n", - " chains,\n", - " [\"S_8\", \"OMEGA_M\"], #\n", - " line_args=line_args,\n", - " contour_args=[{\"alpha\": 0.7}, {\"alpha\": 1.0}],\n", - " contour_colors=colours,\n", - " filled=[True, True],\n", - ")\n", - "g.add_legend(legend_labels, legend_loc=\"upper right\")\n", - "\n", - "g.export(\"../Plots/SP_v1.4.6.3_B_fiducial_config_contour_plot_scales.pdf\")" - ] - }, - { - "cell_type": "markdown", - "id": "20", - "metadata": {}, - "source": [ - "### BBN Prior" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "21", - "metadata": {}, - "outputs": [], - "source": [ - "%matplotlib inline\n", - "\n", - "from getdist.gaussian_mixtures import Gaussian1D\n", - "\n", - "colours = [\n", - " \"orange\",\n", - " \"royalblue\",\n", - "]\n", - "\n", - "linestyle = [\n", - " \"solid\",\n", - " \"solid\",\n", - "]\n", - "\n", - "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", - "\n", - "# BBN PRIOR\n", - "bbn_prior = Gaussian1D(\n", - " mean=0.02218,\n", - " sigma=0.00055,\n", - " name=\"ombh2\",\n", - " labels=[r\"\\omega_{\\rm b}\"],\n", - " label=\"BBN prior\",\n", - ")\n", - "bbn_chain = bbn_prior.MCSamples(3000, label=\"BBN prior\")\n", - "\n", - "g.triangle_plot(\n", - " chains + [bbn_chain],\n", - " name_list,\n", - " legend_labels=legend_labels,\n", - " line_args=line_args,\n", - " contour_colors=colours,\n", - " filled=[True, False],\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "22", - "metadata": {}, - "source": [ - "## Plot the best-fit $\\xi_\\pm$" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "23", - "metadata": {}, - "outputs": [], - "source": [ - "xi_p_data = data[\"XI_PLUS\"].data\n", - "xi_m_data = data[\"XI_MINUS\"].data\n", - "cov_mat = data[\"COVMAT\"].data\n", - "\n", - "labels = roots_scale.values()\n", - "\n", - "bbox_to_anchor_xip = (0.685, 0.09)\n", - "bbox_to_anchor_xim = (0.3, 0.65)\n", - "theta_min = 1.0\n", - "theta_max = 250.0\n", - "loc_legend = \"lower center\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24", - "metadata": {}, - "outputs": [], - "source": [ - "colours = [\n", - " \"orange\",\n", - " \"dodgerblue\",\n", - "]\n", - "\n", - "linestyle = [\n", - " \"solid\",\n", - " \"solid\",\n", - "]\n", - "\n", - "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", - "\n", - "labels = roots_scale.values()\n", - "\n", - "fig, ax = plt.subplots(1, 1, figsize=(11, 7))\n", - "\n", - "theta, xi_p, xi_m = xi_p_data[\"ANG\"], xi_p_data[\"VALUE\"], xi_m_data[\"VALUE\"]\n", - "ax.errorbar(\n", - " theta,\n", - " theta * xi_p,\n", - " yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])),\n", - " fmt=\"o\",\n", - " color=\"black\",\n", - " capsize=2,\n", - ")\n", - "\n", - "for idx, (label, root) in enumerate(zip(labels, roots_scale)):\n", - " # Read the results\n", - " theta = (\n", - " (\n", - " np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " )\n", - " * 180\n", - " / np.pi\n", - " * 60\n", - " )\n", - " xi_plus = np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_minus = np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_sys_plus = np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - " theta_xi_sys = (\n", - " np.loadtxt(path_output_chains + \"{}/best_fit/xi_sys/theta.txt\".format(root))\n", - " * 180\n", - " / np.pi\n", - " * 60\n", - " )\n", - "\n", - " xi_sys_plus = np.interp(theta, theta_xi_sys, xi_sys_plus)\n", - " xi_sys_minus = np.interp(theta, theta_xi_sys, xi_sys_minus)\n", - " xi_plus += xi_sys_plus\n", - " xi_minus += xi_sys_minus\n", - "\n", - " mask = (theta > theta_min) & (theta < theta_max)\n", - " theta = theta[mask]\n", - " ax.plot(theta, theta * xi_plus[mask], label=label, **line_args[idx])\n", - "\n", - "ymin = ax.get_ylim()[0]\n", - "ymax = ax.get_ylim()[1]\n", - "\n", - "ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color=\"gray\", alpha=0.2)\n", - "ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=5, color=\"gray\", alpha=0.7)\n", - "ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=300, color=\"gray\", alpha=0.2)\n", - "\n", - "ax.set_ylim(ymin, ymax)\n", - "\n", - "ax.set_ylabel(r\"$\\theta \\xi_\\pm$\", fontsize=26)\n", - "ax.set_xlabel(r\"$\\theta$ (arcmin)\", fontsize=26)\n", - "ax.set_xlim([theta.min() - 0.1, theta.max() + 20])\n", - "ax.set_title(r\"$\\xi_+(\\theta)$\", fontsize=26)\n", - "ax.set_xscale(\"log\")\n", - "ax.set_xticks(np.array([1, 10, 100]))\n", - "ax.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", - "ax.tick_params(axis=\"both\", which=\"major\", labelsize=24)\n", - "ax.tick_params(axis=\"both\", which=\"minor\", labelsize=20)\n", - "ax.yaxis.get_offset_text().set_fontsize(24)\n", - "ax.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", - "ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=20)\n", - "\n", - "\n", - "plt.savefig(\"./../Plots/scale_cut_xipm_SP_v1.4.6.3_B.pdf\", bbox_inches=\"tight\")\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "25", - "metadata": {}, - "outputs": [], - "source": [ - "labels = roots_nonlin.values()\n", - "\n", - "colours = [\"orange\", \"hotpink\", \"teal\"]\n", - "\n", - "linestyle = [\"solid\", \"solid\", \"dashed\"]\n", - "\n", - "line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)]\n", - "\n", - "fig, [ax, ax2] = plt.subplots(2, 1, figsize=(11, 14))\n", - "\n", - "theta, xi_p, xi_m = xi_p_data[\"ANG\"], xi_p_data[\"VALUE\"], xi_m_data[\"VALUE\"]\n", - "ax.errorbar(\n", - " theta,\n", - " theta * xi_p,\n", - " yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])),\n", - " fmt=\"o\",\n", - " color=\"black\",\n", - " capsize=2,\n", - ")\n", - "ax2.errorbar(\n", - " theta,\n", - " theta * xi_m,\n", - " yerr=theta\n", - " * np.sqrt(\n", - " np.diag(cov_mat[len(theta) : 2 * len(theta), len(theta) : 2 * len(theta)])\n", - " ),\n", - " fmt=\"o\",\n", - " color=\"black\",\n", - " capsize=2,\n", - ")\n", - "\n", - "for idx, (label, root) in enumerate(zip(labels, roots_nonlin)):\n", - " # Read the results\n", - " theta = (\n", - " (\n", - " np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " )\n", - " * 180\n", - " / np.pi\n", - " * 60\n", - " )\n", - " xi_plus = np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_minus = np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - " xi_sys_plus = np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " path_output_chains + \"{}/best_fit/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - " theta_xi_sys = (\n", - " np.loadtxt(path_output_chains + \"{}/best_fit/xi_sys/theta.txt\".format(root))\n", - " * 180\n", - " / np.pi\n", - " * 60\n", - " )\n", - "\n", - " xi_sys_plus = np.interp(theta, theta_xi_sys, xi_sys_plus)\n", - " xi_sys_minus = np.interp(theta, theta_xi_sys, xi_sys_minus)\n", - " xi_plus += xi_sys_plus\n", - " xi_minus += xi_sys_minus\n", - "\n", - " mask = (theta > theta_min) & (theta < theta_max)\n", - " theta = theta[mask]\n", - " ax.plot(theta, theta * xi_plus[mask], label=label, **line_args[idx])\n", - " ax2.plot(theta, theta * xi_minus[mask], label=label, **line_args[idx])\n", - "\n", - "ymin = ax.get_ylim()[0]\n", - "ymax = ax.get_ylim()[1]\n", - "ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color=\"gray\", alpha=0.2)\n", - "ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=300, color=\"gray\", alpha=0.2)\n", - "\n", - "ax.set_ylim(ymin, ymax)\n", - "\n", - "ax.set_ylabel(r\"$\\theta \\xi_\\pm$\", fontsize=26)\n", - "ax.set_xlabel(r\"$\\theta$ (arcmin)\", fontsize=26)\n", - "ax.set_xlim([theta.min() - 0.1, theta.max() + 20])\n", - "ax.set_title(r\"$\\xi_+(\\theta)$\", fontsize=26)\n", - "ax.set_xscale(\"log\")\n", - "ax.set_xticks(np.array([1, 10, 100]))\n", - "ax.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", - "ax.tick_params(axis=\"both\", which=\"major\", labelsize=24)\n", - "ax.tick_params(axis=\"both\", which=\"minor\", labelsize=20)\n", - "ax.yaxis.get_offset_text().set_fontsize(24)\n", - "ax.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", - "\n", - "\n", - "ymin = ax2.get_ylim()[0]\n", - "ymax = ax2.get_ylim()[1]\n", - "ax2.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color=\"gray\", alpha=0.2)\n", - "ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=3000, color=\"gray\", alpha=0.2)\n", - "\n", - "ax2.set_ylim(ymin, ymax)\n", - "ax2.set_xlabel(r\"$\\theta$ (arcmin)\", fontsize=26)\n", - "ax2.set_xlim([theta.min() - 0.1, theta.max()])\n", - "ax2.set_xscale(\"log\")\n", - "ax2.set_title(r\"$\\xi_-(\\vartheta)$\", fontsize=26)\n", - "ax2.set_xticks(np.array([1, 10, 100]))\n", - "ax2.tick_params(axis=\"x\", which=\"minor\", length=2, width=0.8)\n", - "ax2.tick_params(axis=\"both\", which=\"major\", labelsize=24)\n", - "ax2.tick_params(axis=\"both\", which=\"minor\", labelsize=20)\n", - "ax2.yaxis.get_offset_text().set_fontsize(24)\n", - "ax2.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", - "ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=20)\n", - "\n", - "plt.savefig(\"./../Plots/nonlin_xipm_SP_v1.4.6.3_B.pdf\", bbox_inches=\"tight\")\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "26", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "my_env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2.ipynb deleted file mode 100644 index f124e4cd..00000000 --- a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2.ipynb +++ /dev/null @@ -1,690 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import configparser\n", - "import os\n", - "import re\n", - "import subprocess\n", - "import sys\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import scipy.stats as stats\n", - "from astropy.io import fits\n", - "from getdist import plots\n", - "from IPython.display import Markdown, display\n", - "from scipy.interpolate import interp1d\n", - "\n", - "sys.path.append(\"/home/guerrini/sp_validation/cosmo_inference/scripts\")\n", - "\n", - "import chain_postprocessing\n", - "\n", - "%matplotlib inline\n", - "\n", - "plt.rc(\"mathtext\", fontset=\"stix\")\n", - "plt.rc(\"font\", family=\"sans-serif\")\n", - "\n", - "g = plots.get_subplot_plotter(width_inch=30)\n", - "g.settings.axes_fontsize = 30\n", - "g.settings.axes_labelsize = 30\n", - "g.settings.alpha_filled_add = 0.7\n", - "g.settings.legend_fontsize = 40\n", - "\n", - "# #SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE\n", - "root_dir = \"/n09data/guerrini/output_chains/\"\n", - "blind = \"B\"\n", - "\n", - "roots = [\n", - " f\"SP_v1.4.6.3_{blind}_fiducial_config\",\n", - " f\"SP_v1.4.6.3_{blind}_small_scales_config\",\n", - " f\"SP_v1.4.6.3_{blind}_flat_alpha_beta_config\",\n", - " f\"SP_v1.4.6.3_{blind}_no_xi_sys_config\",\n", - " f\"SP_v1.4.6.3_{blind}_no_leak_corr_config\",\n", - " f\"SP_v1.4.6.3_{blind}_flat_delta_z_config\",\n", - " f\"SP_v1.4.6.3_{blind}_no_delta_z_config\",\n", - " f\"SP_v1.4.6.3_{blind}_flat_ia_config\",\n", - " f\"SP_v1.4.6.3_{blind}_no_ia_config\",\n", - " f\"SP_v1.4.6.3_{blind}_no_m_bias_config\",\n", - " f\"SP_v1.4.6.3_{blind}_unmasked_covmat_config\",\n", - " f\"SP_v1.4.6.3_{blind}_halofit_config\",\n", - " f\"SP_v1.4.6.3_{blind}_no_baryons_config\",\n", - " f\"SP_v1.4.6.3_{blind}_nautilus_config\",\n", - " f\"SP_v1.4.6.3_{blind}_planck_config\",\n", - " f\"SP_v1.4.6.3_{blind}_planck_desi_config\",\n", - "]\n", - "\n", - "catalog_versions = [\n", - " f\"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}\",\n", - "]\n", - "\n", - "catalog_sub_versions = [\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", - " f\"SP_v1.4.6.3_{blind}_masked\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", - " f\"SP_v1.4.6.3_leak_corr_{blind}_masked\",\n", - "]\n", - "output_folder = \"/n09data/guerrini/output_chains/\"\n", - "\n", - "path_ini_files = \"/home/guerrini/sp_validation/cosmo_inference/cosmosis_config/\"\n", - "\n", - "\n", - "ini_roots = [\n", - " f\"blind_{blind}/fiducial\",\n", - " f\"blind_{blind}/small_scales\",\n", - " f\"blind_{blind}/flat_alpha_beta\",\n", - " f\"blind_{blind}/no_xi_sys\",\n", - " f\"blind_{blind}/no_leak_corr\",\n", - " f\"blind_{blind}/flat_delta_z\",\n", - " f\"blind_{blind}/no_delta_z\",\n", - " f\"blind_{blind}/flat_ia\",\n", - " f\"blind_{blind}/no_ia\",\n", - " f\"blind_{blind}/no_m_bias\",\n", - " f\"blind_{blind}/unmasked_covmat\",\n", - " f\"blind_{blind}/halofit\",\n", - " f\"blind_{blind}/no_baryons\",\n", - " f\"blind_{blind}/nautilus\",\n", - " f\"blind_{blind}/planck\",\n", - " f\"blind_{blind}/planck_desi\",\n", - "]\n", - "\n", - "properties = {}\n", - "\n", - "for i, root in enumerate(roots):\n", - " print(root)\n", - " config = configparser.ConfigParser()\n", - " config.optionxform = str # Preserve case sensitivity of option names\n", - " config.read(\n", - " path_ini_files\n", - " + \"config_space_v1.4.6.3_fiducial/pipeline/\"\n", - " + ini_roots[i]\n", - " + \".ini\"\n", - " )\n", - " add_xi_sys = config[\"2pt_like\"][\"add_xi_sys\"]\n", - " lower_bound_xi_plus, upper_bound_xi_plus = map(\n", - " float, config[\"2pt_like\"][\"angle_range_XI_PLUS_1_1\"].split()\n", - " )\n", - " lower_bound_xi_minus, upper_bound_xi_minus = map(\n", - " float, config[\"2pt_like\"][\"angle_range_XI_MINUS_1_1\"].split()\n", - " )\n", - "\n", - " properties[root] = {\n", - " \"add_xi_sys\": add_xi_sys,\n", - " \"lower_bound_xi_plus\": lower_bound_xi_plus,\n", - " \"upper_bound_xi_plus\": upper_bound_xi_plus,\n", - " \"lower_bound_xi_minus\": lower_bound_xi_minus,\n", - " \"upper_bound_xi_minus\": upper_bound_xi_minus,\n", - " }" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Retrieve the chains" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# READ CHAIN\n", - "\n", - "chains = []\n", - "\n", - "for i, root in enumerate(roots):\n", - " burnin = 0\n", - "\n", - " if os.path.isfile(root_dir + \"{}/getdist_{}.txt\".format(root, root)) == False:\n", - " samples = np.loadtxt(root_dir + \"{}/samples_{}.txt\".format(root, root))\n", - "\n", - " if \"nautilus\" in root:\n", - " samples = np.column_stack(\n", - " (\n", - " np.exp(samples[:, -3]),\n", - " samples[:, -1] - samples[:, -2],\n", - " samples[:, 0:-3],\n", - " )\n", - " )\n", - " elif \"mh\" in root:\n", - " samples = np.column_stack(\n", - " (\n", - " np.ones_like(samples[:, -1]),\n", - " np.log(samples[:, -1]) - np.log(samples[:, -2]),\n", - " samples[:, 0:-2],\n", - " )\n", - " )\n", - " burnin = 0.3\n", - " else:\n", - " samples = np.column_stack(\n", - " (samples[:, -1], samples[:, -3], samples[:, 0:-4])\n", - " )\n", - "\n", - " np.savetxt(root_dir + \"{}/getdist_{}.txt\".format(root, root), samples)\n", - "\n", - " chain = g.samples_for_root(\n", - " root_dir + \"{}/getdist_{}\".format(root, root),\n", - " cache=False,\n", - " settings={\n", - " \"ignore_rows\": burnin,\n", - " \"smooth_scale_2D\": 0.5,\n", - " \"smooth_scale_1D\": 0.5,\n", - " },\n", - " )\n", - " p = chain.getParams()\n", - "\n", - " chains.append(chain)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "param_list = [\n", - " \"OMEGA_M\",\n", - " \"ombh2\",\n", - " \"h0\",\n", - " \"n_s\",\n", - " \"SIGMA_8\",\n", - " \"s_8_input\",\n", - " \"logt_agn\",\n", - " \"a\",\n", - " \"m1\",\n", - " \"bias_1\",\n", - " \"alpha\",\n", - " \"beta\",\n", - " \"omch2\",\n", - " \"m\",\n", - " \"a_planck\",\n", - "]\n", - "label_list = [\n", - " r\"\\Omega_m\",\n", - " r\"\\omega_b\",\n", - " \"h_0\",\n", - " \"n_s\",\n", - " r\"\\sigma_8\",\n", - " \"S_8\",\n", - " \"log T_{AGN}\",\n", - " \"A_{IA}\",\n", - " \"m_1\",\n", - " r\"\\Delta z_1\",\n", - " \"\\\\alpha_{PSF}\",\n", - " \"\\\\beta_{PSF}\",\n", - " r\"\\omega_c\",\n", - " \"M\",\n", - " \"A_{\\rm Planck}\",\n", - "]\n", - "\n", - "for chain in chains:\n", - " param_names = chain.getParamNames()\n", - " for name, label in zip(param_list, label_list):\n", - " if param_names.parWithName(name) is not None:\n", - " param_names.parWithName(name).label = label" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Extract the best fit parameters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "best_fit = {}\n", - "\n", - "for root, chain in zip(roots, chains):\n", - " print(root)\n", - " p = chain.getParams()\n", - "\n", - " best_fit[root] = chain_postprocessing.extract_best_fit_params(\n", - " chain, best_fit_method=\"2Dkde\"\n", - " )\n", - "\n", - " for param_name in best_fit[root].keys():\n", - " high_68, low_68, high_95, low_95 = chain_postprocessing.compute_limits(\n", - " chain, param_name\n", - " )\n", - " if param_name == \"S_8\":\n", - " print(f\"{best_fit[root][param_name]}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run `Cosmosis` in test mode to get the data vectors" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if not os.path.exists(path_ini_files + \"/values_empty.ini\"):\n", - " content = \"\"\"[cosmological_parameters]\n", - "\n", - "tau = 0.0544\n", - "w = -1.0\n", - "mnu = 0.06\n", - "omega_k = 0.0\n", - "wa = 0.0\n", - "\n", - "[halo_model_parameters]\n", - "\n", - "[intrinsic_alignment_parameters]\n", - "\n", - "[shear_calibration_parameters]\n", - "\n", - "[nofz_shifts]\n", - "\n", - "[psf_leakage_parameters]\n", - "\"\"\"\n", - "\n", - " with open(path_ini_files + \"/values_empty.ini\", \"w\") as f:\n", - " f.write(content)\n", - " f.close()\n", - "\n", - " print(\"File created successfully\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "section_map = {\n", - " \"omch2\": \"cosmological_parameters\",\n", - " \"ombh2\": \"cosmological_parameters\",\n", - " \"h0\": \"cosmological_parameters\",\n", - " \"n_s\": \"cosmological_parameters\",\n", - " \"tau\": \"cosmological_parameters\",\n", - " \"s_8_input\": \"cosmological_parameters\",\n", - " \"logt_agn\": \"halo_model_parameters\",\n", - " \"a\": \"intrinsic_alignment_parameters\",\n", - " \"m1\": \"shear_calibration_parameters\",\n", - " \"bias_1\": \"nofz_shifts\",\n", - " \"alpha\": \"psf_leakage_parameters\",\n", - " \"beta\": \"psf_leakage_parameters\",\n", - " \"m\": \"supernova_params\",\n", - " \"a_planck\": \"planck\",\n", - "}\n", - "\n", - "best_fit[\"SP_v1.4.6.3_B_no_ia_config\"][\"a\"] = 0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "env = os.environ.copy()\n", - "env[\"LD_LIBRARY_PATH\"] = (\n", - " \"/home/guerrini/.conda/envs/sp_validation/lib/python3.9/site-packages/cosmosis/datablock:\"\n", - " + env.get(\"LD_LIBRARY_PATH\", \"\")\n", - ")\n", - "\n", - "for i, root in enumerate(roots):\n", - " print(root)\n", - " config = configparser.ConfigParser()\n", - " config.optionxform = str # Preserve case sensitivity of option names\n", - "\n", - " for param, section in section_map.items():\n", - " # Check if this parameter exists for the current root\n", - " if param in best_fit[root]:\n", - " value = best_fit[root][param]\n", - "\n", - " if section not in config:\n", - " config.add_section(section)\n", - "\n", - " config[section][param] = str(value)\n", - "\n", - " with open(path_ini_files + \"/values_empty.ini\", \"w\") as configfile:\n", - " config.write(configfile)\n", - "\n", - " # Modify the ini file to run in test mode at the best fit\n", - " config = configparser.ConfigParser()\n", - " config.optionxform = str # Preserve case sensitivity of option names\n", - "\n", - " ini_file = path_ini_files + \"config_space_v1.4.6.3_fiducial/pipeline/{}.ini\".format(\n", - " ini_roots[i]\n", - " )\n", - " config.read(ini_file)\n", - "\n", - " sampler = config[\"runtime\"][\"sampler\"]\n", - " config[\"runtime\"][\"sampler\"] = \"test\"\n", - " values = config[\"pipeline\"][\"values\"]\n", - " config[\"pipeline\"][\"values\"] = path_ini_files + \"/values_empty.ini\"\n", - " config[\"DEFAULT\"][\"FITS_FILE\"] = (\n", - " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_versions[0]}/cosmosis_{catalog_sub_versions[i]}.fits\"\n", - " )\n", - " config[\"test\"][\"save_dir\"] = root_dir + \"{}/best_fit\".format(root)\n", - "\n", - " with open(ini_file, \"w\") as configfile:\n", - " config.write(configfile)\n", - "\n", - " # Run cosmosis\n", - " result = subprocess.run(\n", - " [\"cosmosis\", ini_file], env=env, capture_output=True, text=True\n", - " )\n", - " print(f\"STDOUT:\\n{result.stdout}\")\n", - " print(f\"STDERR:\\n{result.stderr}\")\n", - "\n", - " # Modify the ini file to the previous one\n", - " config[\"pipeline\"][\"values\"] = values\n", - " config[\"runtime\"][\"sampler\"] = sampler\n", - "\n", - " with open(ini_file, \"w\") as configfile:\n", - " config.write(configfile)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Compute the $\\chi^2$" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "metrics = {}\n", - "\n", - "for idx, root in enumerate(roots):\n", - " print(root)\n", - " match = re.search(r\"corr_([A-Za-z])\", root)\n", - " if match:\n", - " blind = match.group(1)\n", - "\n", - " add_xi_sys = properties[root][\"add_xi_sys\"]\n", - " print(f\"add_xi_sys: {add_xi_sys}\")\n", - " lower_bound_xi_plus = properties[root][\"lower_bound_xi_plus\"]\n", - " upper_bound_xi_plus = properties[root][\"upper_bound_xi_plus\"]\n", - " lower_bound_xi_minus = properties[root][\"lower_bound_xi_minus\"]\n", - " upper_bound_xi_minus = properties[root][\"upper_bound_xi_minus\"]\n", - "\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder + \"{}/best_fit/shear_xi_plus/theta.txt\".format(root)\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " shear_xi_plus = np.loadtxt(\n", - " output_folder + \"{}/best_fit/shear_xi_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " shear_xi_minus = np.loadtxt(\n", - " output_folder + \"{}/best_fit/shear_xi_minus/bin_1_1.txt\".format(root)\n", - " )\n", - "\n", - " if add_xi_sys == \"T\":\n", - " xi_sys_plus = np.loadtxt(\n", - " output_folder + \"{}/best_fit/xi_sys/shear_xi_plus.txt\".format(root)\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " output_folder + \"{}/best_fit/xi_sys/shear_xi_minus.txt\".format(root)\n", - " )\n", - "\n", - " theta_tau = np.loadtxt(\n", - " output_folder + \"{}/best_fit/tau_0_plus/theta.txt\".format(root)\n", - " )\n", - " theta_tau_arcmin = theta_tau * 180 * 60 / np.pi\n", - " tau_0_model = np.loadtxt(\n", - " output_folder + \"{}/best_fit/tau_0_plus/bin_1_1.txt\".format(root)\n", - " )\n", - " tau_2_model = np.loadtxt(\n", - " output_folder + \"{}/best_fit/tau_2_plus/bin_1_1.txt\".format(root)\n", - " )\n", - "\n", - " data = fits.open(\n", - " f\"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_versions[0]}/cosmosis_{catalog_sub_versions[idx]}.fits\"\n", - " )\n", - "\n", - " tau_0_data = data[\"TAU_0_PLUS\"].data[\"VALUE\"]\n", - " tau_2_data = data[\"TAU_2_PLUS\"].data[\"VALUE\"]\n", - "\n", - " theta_data = data[\"XI_PLUS\"].data[\"ANG\"]\n", - " xi_plus_data = data[\"XI_PLUS\"].data[\"VALUE\"]\n", - " xi_minus_data = data[\"XI_MINUS\"].data[\"VALUE\"]\n", - "\n", - " # Load the covariance\n", - " cov = data[\"COVMAT\"].data\n", - " cov_xi = cov[0 : 2 * len(xi_plus_data), 0 : 2 * len(xi_plus_data)]\n", - " cov_tau = cov[2 * len(xi_plus_data) :, 2 * len(xi_plus_data) :]\n", - "\n", - " # interpolate the model\n", - " interp_xi_plus = interp1d(\n", - " theta_arcmin, shear_xi_plus, kind=\"cubic\", fill_value=\"extrapolate\"\n", - " )\n", - " interp_xi_minus = interp1d(\n", - " theta_arcmin, shear_xi_minus, kind=\"cubic\", fill_value=\"extrapolate\"\n", - " )\n", - "\n", - " xi_plus_model = interp_xi_plus(theta_data)\n", - " if add_xi_sys:\n", - " xi_plus_model += xi_sys_plus\n", - " xi_minus_model = interp_xi_minus(theta_data)\n", - " if add_xi_sys:\n", - " xi_minus_model += xi_sys_minus\n", - "\n", - " # Concatenate the data vector\n", - " xi_data = np.concatenate((xi_plus_data, xi_minus_data))\n", - " xi_model = np.concatenate((xi_plus_model, xi_minus_model))\n", - "\n", - " tau_data = np.concatenate((tau_0_data, tau_2_data))\n", - " tau_model = np.concatenate((tau_0_model, tau_2_model))\n", - "\n", - " # Apply scale cuts\n", - " mask_xi_plus = (theta_data > lower_bound_xi_plus) & (\n", - " theta_data < upper_bound_xi_plus\n", - " )\n", - " mask_xi_minus = (theta_data > lower_bound_xi_minus) & (\n", - " theta_data < upper_bound_xi_minus\n", - " )\n", - " mask = np.concatenate((mask_xi_plus, mask_xi_minus))\n", - "\n", - " xi_data = xi_data[mask]\n", - " xi_model = xi_model[mask]\n", - " cov_xi = cov_xi[mask][:, mask]\n", - "\n", - " cov_xi_plus = cov[0 : len(xi_plus_data), 0 : len(xi_plus_data)]\n", - " cov_xi_plus = cov_xi_plus[mask_xi_plus][:, mask_xi_plus]\n", - " cov_xi_minus = cov[\n", - " len(xi_plus_data) : 2 * len(xi_minus_data),\n", - " len(xi_plus_data) : 2 * len(xi_minus_data),\n", - " ]\n", - " cov_xi_minus = cov_xi_minus[mask_xi_minus][:, mask_xi_minus]\n", - "\n", - " xi_plus_chi2 = np.dot(\n", - " (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]),\n", - " np.dot(\n", - " np.linalg.inv(cov_xi_plus),\n", - " (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]),\n", - " ),\n", - " )\n", - " xi_minus_chi2 = np.dot(\n", - " (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]),\n", - " np.dot(\n", - " np.linalg.inv(cov_xi_minus),\n", - " (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]),\n", - " ),\n", - " )\n", - " xi_chi2 = np.dot(\n", - " (xi_model - xi_data), np.dot(np.linalg.inv(cov_xi), (xi_model - xi_data))\n", - " )\n", - " tau_chi2 = np.dot(\n", - " (tau_model - tau_data), np.dot(np.linalg.inv(cov_tau), (tau_model - tau_data))\n", - " )\n", - " n_dof_xi_plus = np.sum(mask_xi_plus)\n", - " n_dof_xi_minus = np.sum(mask_xi_minus)\n", - " n_dof_tau = len(tau_0_data) + len(tau_2_data)\n", - " p_value_xi_plus = 1 - stats.chi2.cdf(xi_plus_chi2, n_dof_xi_plus)\n", - " p_value_xi_minus = 1 - stats.chi2.cdf(xi_minus_chi2, n_dof_xi_minus)\n", - " p_value_xi = 1 - stats.chi2.cdf(xi_chi2, n_dof_xi_plus + n_dof_xi_minus)\n", - " p_value_tau = 1 - stats.chi2.cdf(tau_chi2, n_dof_tau)\n", - " chi2_tot = xi_plus_chi2 + xi_minus_chi2 + tau_chi2\n", - " n_dof_tot = n_dof_xi_plus + n_dof_xi_minus + n_dof_tau\n", - " p_value_tot = 1 - stats.chi2.cdf(chi2_tot, n_dof_tot)\n", - "\n", - " metrics[root] = {\n", - " \"chi2_xi_plus\": xi_plus_chi2,\n", - " \"n_dof_xi_plus\": n_dof_xi_plus,\n", - " \"p_value_xi_plus\": p_value_xi_plus,\n", - " \"chi2_xi_minus\": xi_minus_chi2,\n", - " \"n_dof_xi_minus\": n_dof_xi_minus,\n", - " \"p_value_xi_minus\": p_value_xi_minus,\n", - " \"chi2_xi\": xi_chi2,\n", - " \"p_value_xi\": p_value_xi,\n", - " \"chi2_tau\": tau_chi2,\n", - " \"n_dof_tau\": n_dof_tau,\n", - " \"p_value_tau\": p_value_tau,\n", - " \"chi2_tot\": chi2_tot,\n", - " \"n_dof_tot\": n_dof_tot,\n", - " \"p_value_tot\": p_value_tot,\n", - " }\n", - " print(\"Done!\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def get_latex_table(metrics):\n", - " latex_lines = [\n", - " r\"\\begin{tabular}{lccc|ccc|ccc}\",\n", - " r\"\\hline\",\n", - " r\"Root & $\\chi^2_{\\xi^+}$/dof & $p_{\\xi^+}$ & $\\chi^2_{\\xi^-}$/dof & $p_{\\xi^+}$ & $\\chi^2_{\\xi}$/dof & $p_{\\xi}$ &\"\n", - " r\"$\\chi^2_\\tau$/dof & $p_\\tau$ & $\\chi^2_{\\text{tot}}$/dof & $p_{\\text{tot}}$ \\\\\",\n", - " r\"\\hline\",\n", - " ]\n", - "\n", - " for root, vals in metrics.items():\n", - " escaped = root.replace(\"_\", r\"\\_\")\n", - " line = (\n", - " f\"{escaped} & \"\n", - " f\"{vals['chi2_xi_plus']:.2f}/{vals['n_dof_xi_plus']} & {vals['p_value_xi_plus']:.3g} & \"\n", - " f\"{vals['chi2_xi_minus']:.2f}/{vals['n_dof_xi_minus']} & {vals['p_value_xi_minus']:.3g} & \"\n", - " f\"{vals['chi2_xi']:.2f}/{vals['n_dof_xi_plus'] + vals['n_dof_xi_minus']} & {vals['p_value_xi']:.3g} &\"\n", - " f\"{vals['chi2_tau']:.2f}/{vals['n_dof_tau']} & {vals['p_value_tau']:.3g} & \"\n", - " f\"{vals['chi2_tot']:.2f}/{vals['n_dof_tot']} & {vals['p_value_tot']:.3g} \\\\\\\\\"\n", - " )\n", - " latex_lines.append(line)\n", - "\n", - " latex_lines.append(r\"\\hline\")\n", - " latex_lines.append(r\"\\end{tabular}\")\n", - "\n", - " # Print LaTeX table\n", - " print(\"\\n\".join(latex_lines))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "get_latex_table(metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def display_markdown(metrics):\n", - " # Build Markdown table\n", - " header = (\n", - " \"| Root | $\\\\chi^2$ (ξ⁺) / dof | p-val (ξ⁺) |$\\\\chi^2$ (ξ-) / dof | p-val (ξ-) | $\\\\chi^2$ (ξ) / dof | p-val (ξ) | $\\\\chi^2$ (τ) / dof | p-val (τ) | $\\\\chi^2$ (tot) / dof | p-val (tot) |\\n\"\n", - " \"|------|----------------|------------|----------------|------------|------------|---------------|------------|------------|------------------|--------------|\\n\"\n", - " )\n", - "\n", - " rows = []\n", - " for root, vals in metrics.items():\n", - " row = f\"| `{root}` \"\n", - " row += f\"| {vals['chi2_xi_plus']:.2f} / {vals['n_dof_xi_plus']} \"\n", - " row += f\"| {vals['p_value_xi_plus']:.5f} \"\n", - " row += f\"| {vals['chi2_xi_minus']:.2f} / {vals['n_dof_xi_minus']} \"\n", - " row += f\"| {vals['p_value_xi_minus']:.5f} \"\n", - " row += f\"| {vals['chi2_xi']:.2f} / {vals['n_dof_xi_minus'] + vals['n_dof_xi_plus']} \"\n", - " row += f\"| {vals['p_value_xi']:.5f} \"\n", - " row += f\"| {vals['chi2_tau']:.2f} / {vals['n_dof_tau']} \"\n", - " row += f\"| {vals['p_value_tau']:.5f} \"\n", - " row += f\"| {vals['chi2_tot']:.2f} / {vals['n_dof_tot']} \"\n", - " row += f\"| {vals['p_value_tot']:.5f} |\"\n", - " rows.append(row)\n", - "\n", - " # Display in Jupyter\n", - " display(Markdown(header + \"\\n\".join(rows)))\n", - " return header + \"\\n\".join(rows)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "markdown_source = display_markdown(metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "my_env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2_glass_mock.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2_glass_mock.ipynb deleted file mode 100644 index ddd66cd0..00000000 --- a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_chi2_glass_mock.ipynb +++ /dev/null @@ -1,565 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import configparser\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "# Make the plot\n", - "import seaborn as sns\n", - "from astropy.io import fits\n", - "from getdist import plots\n", - "from scipy.interpolate import interp1d\n", - "from scipy.stats import chi2\n", - "\n", - "sys.path.append(\"/home/guerrini/sp_validation/cosmo_inference/scripts\")\n", - "\n", - "import chain_postprocessing\n", - "\n", - "%matplotlib inline\n", - "\n", - "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", - "\n", - "plt.rcParams[\"axes.labelsize\"] = 18\n", - "plt.rcParams[\"xtick.labelsize\"] = 18\n", - "plt.rcParams[\"ytick.labelsize\"] = 18\n", - "\n", - "plt.rcParams[\"text.usetex\"] = True\n", - "\n", - "g = plots.get_subplot_plotter(width_inch=30)\n", - "g.settings.axes_fontsize = 30\n", - "g.settings.axes_labelsize = 30\n", - "g.settings.alpha_filled_add = 0.7\n", - "g.settings.legend_fontsize = 40\n", - "\n", - "# #SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE\n", - "\n", - "root_dir = \"/n09data/guerrini/glass_mock_chains/\"\n", - "\n", - "# Version of the glass mock chain run\n", - "chain_version = \"v6\"\n", - "\n", - "# Path to the glass mock data vectors\n", - "root_glass_dv = (\n", - " f\"/home/guerrini/sp_validation/cosmo_inference/data/glass_mocks/{chain_version}/\"\n", - ")\n", - "\n", - "# Choose the best-fit method\n", - "best_fit_method = \"2Dkde\"\n", - "\n", - "# Create the list of mocks\n", - "max_sim = 350\n", - "failed_simulations = [82, 83, 281, 282, 283, 284, 285, 286, 287]\n", - "roots = [f\"glass_mock_{chain_version}_{str(i).zfill(5)}\" for i in range(1, max_sim + 1)]\n", - "roots = [root for root in roots if int(root.split(\"_\")[-1]) not in failed_simulations]\n", - "\n", - "catalog_versions = [\n", - " \"SP_v1.4.6.3_config/SP_v1.4.6.3_A\",\n", - "]\n", - "\n", - "output_folder_chains = \"/n23data1/n06data/lgoh/scratch/temp/\"\n", - "path_ini_files = \"/home/guerrini/sp_validation/cosmo_inference/cosmosis_config/\"\n", - "output_fig_path = (\n", - " \"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/notebooks/Plots/\"\n", - ")\n", - "\n", - "ini_root = \"blind_A/fiducial\"\n", - "\n", - "lower_bound_xi = 12\n", - "upper_bound_xi = 83" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Retrieve the chains" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# READ CHAIN\n", - "\n", - "chains = []\n", - "best_fit = {}\n", - "\n", - "for i, root in enumerate(roots):\n", - " burnin = 0\n", - "\n", - " if os.path.isfile(f\"{root_dir}/{root}/{root}/getdist_{root}.txt\") == True:\n", - " chain = g.samples_for_root(\n", - " f\"{root_dir}/{root}/{root}/getdist_{root}\",\n", - " cache=False,\n", - " settings={\n", - " \"ignore_rows\": burnin,\n", - " \"smooth_scale_2D\": 0.5,\n", - " \"smooth_scale_1D\": 0.5,\n", - " },\n", - " )\n", - " p = chain.getParams()\n", - "\n", - " best_fit[root] = chain_postprocessing.extract_best_fit_params(\n", - " chain, best_fit_method=\"2Dkde\"\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "param_list = [\n", - " \"OMEGA_M\",\n", - " \"ombh2\",\n", - " \"h0\",\n", - " \"n_s\",\n", - " \"SIGMA_8\",\n", - " \"s_8_input\",\n", - " \"logt_agn\",\n", - " \"a\",\n", - " \"m1\",\n", - " \"bias_1\",\n", - " \"alpha\",\n", - " \"beta\",\n", - " \"omch2\",\n", - " \"m\",\n", - " \"a_planck\",\n", - "]\n", - "label_list = [\n", - " r\"\\Omega_m\",\n", - " r\"\\omega_b\",\n", - " \"h_0\",\n", - " \"n_s\",\n", - " r\"\\sigma_8\",\n", - " \"S_8\",\n", - " \"log T_{AGN}\",\n", - " \"A_{IA}\",\n", - " \"m_1\",\n", - " r\"\\Delta z_1\",\n", - " \"\\\\alpha_{PSF}\",\n", - " \"\\\\beta_{PSF}\",\n", - " r\"\\omega_c\",\n", - " \"M\",\n", - " \"A_{\\rm Planck}\",\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run `Cosmosis` in test mode to get the data vectors" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if not os.path.exists(path_ini_files + \"/values_empty.ini\"):\n", - " content = \"\"\"[cosmological_parameters]\n", - "\n", - "tau = 0.0544\n", - "w = -1.0\n", - "mnu = 0.06\n", - "omega_k = 0.0\n", - "wa = 0.0\n", - "\n", - "[halo_model_parameters]\n", - "\n", - "[intrinsic_alignment_parameters]\n", - "\n", - "[shear_calibration_parameters]\n", - "\n", - "[nofz_shifts]\n", - "\n", - "[psf_leakage_parameters]\n", - "\"\"\"\n", - "\n", - " with open(path_ini_files + \"/values_empty.ini\", \"w\") as f:\n", - " f.write(content)\n", - " f.close()\n", - "\n", - " print(\"File created successfully\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "section_map = {\n", - " \"omch2\": \"cosmological_parameters\",\n", - " \"ombh2\": \"cosmological_parameters\",\n", - " \"h0\": \"cosmological_parameters\",\n", - " \"n_s\": \"cosmological_parameters\",\n", - " \"s_8_input\": \"cosmological_parameters\",\n", - " \"logt_agn\": \"halo_model_parameters\",\n", - " \"a\": \"intrinsic_alignment_parameters\",\n", - " \"m1\": \"shear_calibration_parameters\",\n", - " \"bias_1\": \"nofz_shifts\",\n", - " \"alpha\": \"psf_leakage_parameters\",\n", - " \"beta\": \"psf_leakage_parameters\",\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "env = os.environ.copy()\n", - "env[\"LD_LIBRARY_PATH\"] = (\n", - " \"/home/guerrini/.conda/envs/sp_validation/lib/python3.9/site-packages/cosmosis/datablock:\"\n", - " + env.get(\"LD_LIBRARY_PATH\", \"\")\n", - ")\n", - "for i, root in enumerate(roots):\n", - " print(root)\n", - " config = configparser.ConfigParser()\n", - " config.optionxform = str # Preserve case sensitivity of option names\n", - "\n", - " for param, section in section_map.items():\n", - " # Check if this parameter exists for the current root\n", - " if param in best_fit[root]:\n", - " value = best_fit[root][param]\n", - "\n", - " if section not in config:\n", - " config.add_section(section)\n", - "\n", - " config[section][param] = str(value)\n", - "\n", - " with open(path_ini_files + \"/values_empty.ini\", \"w\") as configfile:\n", - " config.write(configfile)\n", - "\n", - " # Modify the ini file to run in test mode at the best fit\n", - " config = configparser.ConfigParser()\n", - " config.optionxform = str # Preserve case sensitivity of option names\n", - "\n", - " ini_file = (\n", - " path_ini_files + f\"config_space_v1.4.6.3_fiducial/pipeline/{ini_root}.ini\"\n", - " )\n", - " config.read(ini_file)\n", - "\n", - " sampler = config[\"runtime\"][\"sampler\"]\n", - " config[\"runtime\"][\"sampler\"] = \"test\"\n", - " values = config[\"pipeline\"][\"values\"]\n", - " config[\"pipeline\"][\"values\"] = path_ini_files + \"/values_empty.ini\"\n", - " config[\"DEFAULT\"][\"FITS_FILE\"] = (\n", - " f\"{root_glass_dv}/glass_mock_{root[-5:]}/cosmosis_glass_mock_v6_{root[-5:]}.fits\"\n", - " )\n", - " config[\"test\"][\"save_dir\"] = output_folder_chains + f\"{root}/best_fit_config\"\n", - "\n", - " with open(ini_file, \"w\") as configfile:\n", - " config.write(configfile)\n", - "\n", - " # Run cosmosis\n", - " result = subprocess.run(\n", - " [\"cosmosis\", ini_file], env=env, capture_output=True, text=True\n", - " )\n", - " # print(f\"STDOUT:\\n{result.stdout}\")\n", - " # print(f\"STDERR:\\n{result.stderr}\")\n", - "\n", - " # Modify the ini file to the previous one\n", - " config[\"pipeline\"][\"values\"] = values\n", - " config[\"runtime\"][\"sampler\"] = sampler\n", - "\n", - " with open(ini_file, \"w\") as configfile:\n", - " config.write(configfile)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "xi_plus_chi2s = np.array([])\n", - "xi_minus_chi2s = np.array([])\n", - "xi_chi2s = np.array([])\n", - "tau_chi2s = np.array([])\n", - "chi2_tots = np.array([])\n", - "\n", - "\n", - "for idx, root in enumerate(roots):\n", - " print(root)\n", - "\n", - " data = fits.open(\n", - " f\"{root_glass_dv}/glass_mock_{root[-5:]}/cosmosis_glass_mock_v6_{root[-5:]}.fits\"\n", - " )\n", - "\n", - " tau_0_data = data[\"TAU_0_PLUS\"].data[\"VALUE\"]\n", - " tau_2_data = data[\"TAU_2_PLUS\"].data[\"VALUE\"]\n", - "\n", - " theta_data = data[\"XI_PLUS\"].data[\"ANG\"]\n", - " xi_plus_data = data[\"XI_PLUS\"].data[\"VALUE\"]\n", - " xi_minus_data = data[\"XI_MINUS\"].data[\"VALUE\"]\n", - " xi_data = np.concatenate((xi_plus_data, xi_minus_data))\n", - "\n", - " tau_data = np.concatenate((tau_0_data, tau_2_data))\n", - "\n", - " # Apply scale cuts\n", - " mask_xi_plus = (theta_data > lower_bound_xi) & (theta_data < upper_bound_xi)\n", - " mask_xi_minus = (theta_data > lower_bound_xi) & (theta_data < upper_bound_xi)\n", - " mask = np.concatenate((mask_xi_plus, mask_xi_minus))\n", - " # Load the covariance\n", - " cov = data[\"COVMAT\"].data\n", - " cov_xi = cov[0 : 2 * len(xi_plus_data), 0 : 2 * len(xi_plus_data)]\n", - " cov_tau = cov[\n", - " 2 * len(xi_plus_data) : 4 * len(xi_plus_data),\n", - " 2 * len(xi_plus_data) : 4 * len(xi_plus_data),\n", - " ]\n", - " xi_data = xi_data[mask]\n", - " cov_xi = cov_xi[mask][:, mask]\n", - "\n", - " cov_xi_plus = cov[0 : len(xi_plus_data), 0 : len(xi_plus_data)]\n", - " cov_xi_plus = cov_xi_plus[mask_xi_plus][:, mask_xi_plus]\n", - " cov_xi_minus = cov[\n", - " len(xi_plus_data) : 2 * len(xi_minus_data),\n", - " len(xi_plus_data) : 2 * len(xi_minus_data),\n", - " ]\n", - " cov_xi_minus = cov_xi_minus[mask_xi_minus][:, mask_xi_minus]\n", - "\n", - " # Read the results\n", - " theta = np.loadtxt(\n", - " output_folder_chains + f\"{root}/best_fit_config/shear_xi_plus/theta.txt\"\n", - " )\n", - " theta_arcmin = theta * 180 * 60 / np.pi\n", - " shear_xi_plus = np.loadtxt(\n", - " output_folder_chains + f\"{root}/best_fit_config/shear_xi_plus/bin_1_1.txt\"\n", - " )\n", - " shear_xi_minus = np.loadtxt(\n", - " output_folder_chains + f\"{root}/best_fit_config/shear_xi_minus/bin_1_1.txt\"\n", - " )\n", - "\n", - " xi_sys_plus = np.loadtxt(\n", - " output_folder_chains + f\"{root}/best_fit_config/xi_sys/shear_xi_plus.txt\"\n", - " )\n", - " xi_sys_minus = np.loadtxt(\n", - " output_folder_chains + f\"{root}/best_fit_config/xi_sys/shear_xi_minus.txt\"\n", - " )\n", - "\n", - " theta_tau = np.loadtxt(\n", - " output_folder_chains + f\"{root}/best_fit_config/tau_0_plus/theta.txt\"\n", - " )\n", - " theta_tau_arcmin = theta_tau * 180 * 60 / np.pi\n", - " tau_0_model = np.loadtxt(\n", - " output_folder_chains + f\"{root}/best_fit_config/tau_0_plus/bin_1_1.txt\"\n", - " )\n", - " tau_2_model = np.loadtxt(\n", - " output_folder_chains + f\"{root}/best_fit_config/tau_2_plus/bin_1_1.txt\"\n", - " )\n", - "\n", - " # interpolate the model\n", - " interp_xi_plus = interp1d(\n", - " theta_arcmin, shear_xi_plus, kind=\"cubic\", fill_value=\"extrapolate\"\n", - " )\n", - " interp_xi_minus = interp1d(\n", - " theta_arcmin, shear_xi_minus, kind=\"cubic\", fill_value=\"extrapolate\"\n", - " )\n", - "\n", - " xi_plus_model = interp_xi_plus(theta_data)\n", - " xi_plus_model += xi_sys_plus\n", - " xi_minus_model = interp_xi_minus(theta_data)\n", - " xi_minus_model += xi_sys_minus\n", - "\n", - " xi_model = np.concatenate((xi_plus_model, xi_minus_model))\n", - " tau_model = np.concatenate((tau_0_model, tau_2_model))\n", - " xi_model = xi_model[mask]\n", - "\n", - " xi_plus_chi2 = np.dot(\n", - " (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]),\n", - " np.dot(\n", - " np.linalg.inv(cov_xi_plus),\n", - " (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]),\n", - " ),\n", - " )\n", - " xi_minus_chi2 = np.dot(\n", - " (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]),\n", - " np.dot(\n", - " np.linalg.inv(cov_xi_minus),\n", - " (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]),\n", - " ),\n", - " )\n", - " xi_chi2 = np.dot(\n", - " (xi_model - xi_data), np.dot(np.linalg.inv(cov_xi), (xi_model - xi_data))\n", - " )\n", - " tau_chi2 = np.dot(\n", - " (tau_model - tau_data), np.dot(np.linalg.inv(cov_tau), (tau_model - tau_data))\n", - " )\n", - " chi2_tot = xi_plus_chi2 + xi_minus_chi2 + tau_chi2\n", - "\n", - " xi_plus_chi2s = np.append(xi_plus_chi2s, xi_plus_chi2)\n", - " xi_minus_chi2s = np.append(xi_minus_chi2s, xi_minus_chi2)\n", - " xi_chi2s = np.append(xi_chi2s, xi_chi2)\n", - " tau_chi2s = np.append(tau_chi2s, tau_chi2)\n", - " chi2_tots = np.append(chi2_tots, chi2_tot)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "fig, [ax1, ax2] = plt.subplots(2, 1, figsize=(7, 10))\n", - "chi2_fiducial = -2 * -37.560916821678894\n", - "dof, loc, scale = chi2.fit(chi2_tots, floc=0)\n", - "\n", - "print(f\"Best-fit dof: {dof:.3e}\")\n", - "counts, bin_edges = np.histogram(chi2_tots, bins=25, density=True)\n", - "\n", - "sns.histplot(\n", - " chi2_tots,\n", - " ax=ax1,\n", - " kde=False,\n", - " bins=bin_edges,\n", - " stat=\"density\",\n", - " label=r\"$\\chi^2$ for \\texttt{GLASS} mocks best-fits\",\n", - " color=\"green\",\n", - " alpha=0.3,\n", - ")\n", - "\n", - "# Compute the p-value\n", - "\n", - "# 1. Get in which bin the chi2 of the fiducial falls\n", - "bin_index = np.digitize(chi2_fiducial, bin_edges)\n", - "\n", - "# 2. Compute the p-value as the integral of the tail of the histogram\n", - "p_value = np.sum(counts[bin_index:]) * np.diff(bin_edges)[0]\n", - "\n", - "print(f\"P-value: {p_value}\")\n", - "\n", - "ax1.axvline(chi2_fiducial, color=\"red\", label=r\"$\\chi^2$ of the fiducial\", lw=2)\n", - "\n", - "mantissa, exponent = np.frexp(p_value)\n", - "pte_string = rf\"${{\\rm PTE}} = {p_value:.4f}$\"\n", - "print(f\"mantissa: {mantissa}, exponent: {exponent}\")\n", - "x_text = 78\n", - "y_text = max(counts) * 0.95\n", - "ax1.text(\n", - " x_text,\n", - " y_text,\n", - " pte_string,\n", - " fontsize=15,\n", - " bbox=dict(facecolor=\"wheat\", alpha=0.8, edgecolor=\"black\"),\n", - ")\n", - "\n", - "chi2_string = rf\"${{\\rm Eff. dof}}= {dof:.1f}$\"\n", - "y_text = max(counts) * 0.85\n", - "ax1.text(\n", - " x_text,\n", - " y_text,\n", - " chi2_string,\n", - " fontsize=15,\n", - " bbox=dict(facecolor=\"wheat\", alpha=0.8, edgecolor=\"black\"),\n", - ")\n", - "\n", - "ax1.set_xlabel(r\"$\\chi^2_{\\rm tot}$\")\n", - "ax1.set_ylabel(\"Density\")\n", - "\n", - "chi2_fiducial = 9.5\n", - "dof, loc, scale = chi2.fit(xi_chi2s, floc=0)\n", - "\n", - "print(f\"Best-fit dof: {dof:.3e}\")\n", - "counts, bin_edges = np.histogram(xi_chi2s, bins=25, density=True)\n", - "\n", - "sns.histplot(\n", - " xi_chi2s,\n", - " ax=ax2,\n", - " kde=False,\n", - " bins=bin_edges,\n", - " stat=\"density\",\n", - " label=r\"$\\chi^2$ for \\texttt{GLASS} mocks best-fits\",\n", - " color=\"pink\",\n", - " alpha=0.5,\n", - ")\n", - "\n", - "# Compute the p-value\n", - "\n", - "# 1. Get in which bin the chi2 of the fiducial falls\n", - "bin_index = np.digitize(chi2_fiducial, bin_edges)\n", - "\n", - "# 2. Compute the p-value as the integral of the tail of the histogram\n", - "p_value = np.sum(counts[bin_index:]) * np.diff(bin_edges)[0]\n", - "\n", - "print(f\"P-value: {p_value}\")\n", - "\n", - "ax2.axvline(chi2_fiducial, color=\"red\", label=r\"$\\chi^2$ of the fiducial\", lw=2)\n", - "\n", - "mantissa, exponent = np.frexp(p_value)\n", - "print(f\"mantissa: {mantissa}, exponent: {exponent}\")\n", - "pte_string = rf\"${{\\rm PTE}} = {p_value:.4f}$\"\n", - "# rf\"${{\\rm PTE}} = {mantissa:.2f} \\times 10^{{{exponent}}}$\" if exponent != 0 else\n", - "x_text = 17.5\n", - "y_text = max(counts) * 0.95\n", - "ax2.text(\n", - " x_text,\n", - " y_text,\n", - " pte_string,\n", - " fontsize=15,\n", - " bbox=dict(facecolor=\"wheat\", alpha=0.8, edgecolor=\"black\"),\n", - ")\n", - "\n", - "chi2_string = rf\"${{\\rm Eff. dof}}= {dof:.1f}$\"\n", - "y_text = max(counts) * 0.85\n", - "ax2.text(\n", - " x_text,\n", - " y_text,\n", - " chi2_string,\n", - " fontsize=15,\n", - " bbox=dict(facecolor=\"wheat\", alpha=0.8, edgecolor=\"black\"),\n", - ")\n", - "\n", - "ax2.set_xlabel(r\"$\\chi^2 (\\xi_\\pm)$\")\n", - "ax2.set_ylabel(\"Density\")\n", - "fig.savefig(f\"{output_fig_path}/chi2_glass_mocks_p_value_xi_tau.pdf\")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "my_env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_prior_psf_leakage.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_prior_psf_leakage.ipynb deleted file mode 100644 index ad6f5fd1..00000000 --- a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/get_prior_psf_leakage.ipynb +++ /dev/null @@ -1,261 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "0", - "metadata": {}, - "source": [ - "# Covariance matrix and PSF leakage\n", - "\n", - "This notebook plots the combined covariance matrix, and samples and plots the 2D marginalised posteriors of the PSF leakage parameters $\\alpha$ and $\\beta$." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "if not os.path.exists(\"./Plots\"):\n", - " os.makedirs(\"./Plots\")\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import seaborn as sns\n", - "from astropy.io import fits\n", - "from getdist import MCSamples, plots\n", - "from shear_psf_leakage.rho_tau_stat import PSFErrorFit, RhoStat, TauStat\n", - "\n", - "# Use paper style and seaborn with husl palette\n", - "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", - "# Set default palette - will be updated per plot as needed\n", - "sns.set_palette(\"husl\")\n", - "%matplotlib inline\n", - "\n", - "g = plots.get_subplot_plotter(width_inch=30)\n", - "g.settings.axes_fontsize = 30\n", - "g.settings.axes_labelsize = 30\n", - "g.settings.alpha_filled_add = 0.7\n", - "g.settings.legend_fontsize = 25\n", - "\n", - "ver = \"v1.4.6.3\"\n", - "blind = \"B\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ - "data_path = f\"/home/guerrini/sp_validation/cosmo_inference/data/SP_{ver}_config/\"\n", - "\n", - "path_cosmo_val = \"/home/guerrini/sp_validation/cosmo_val/output/\"\n", - "\n", - "roots = [f\"SP_{ver}_{blind}\", f\"SP_{ver}_leak_corr_{blind}\"]\n", - "\n", - "labels = [f\"SP_{ver}_{blind}\", f\"SP_{ver}_leak_corr_{blind}\"]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "data_vectors = []\n", - "\n", - "for root in roots:\n", - " data_vectors.append(\n", - " fits.open(data_path + f\"SP_{ver}_{blind}/cosmosis_{root}_masked.fits\")\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "def cov_to_corr(cov):\n", - " \"\"\"Convert a covariance matrix to a correlation matrix.\"\"\"\n", - " d = np.sqrt(np.diag(cov))\n", - " corr = cov / np.outer(d, d)\n", - " corr[cov == 0] = 0\n", - " return corr" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5", - "metadata": {}, - "outputs": [], - "source": [ - "# Print the covariance matrix for each root\n", - "for i, root in enumerate(roots):\n", - " print(f\"Covariance matrix for {labels[i]}:\")\n", - " cov = data_vectors[i][\"COVMAT\"].data\n", - "\n", - " n_bins = cov.shape[0] // 4\n", - "\n", - " fig, ax = plt.subplots(figsize=(10, 8))\n", - "\n", - " im = ax.imshow(cov_to_corr(cov), vmin=-1, vmax=1, cmap=\"seismic\")\n", - " ax.set_aspect(\"equal\")\n", - " ax.set_yticks(np.array([10, 30, 50, 70]))\n", - " ax.set_yticklabels(\n", - " [\n", - " r\"$\\xi_+(\\vartheta)$\",\n", - " r\"$\\xi_-(\\vartheta)$\",\n", - " r\"$\\tau_0(\\vartheta)$\",\n", - " r\"$\\tau_2(\\vartheta)$\",\n", - " ]\n", - " )\n", - " ax.set_xticks(np.array([10, 30, 50, 70]))\n", - " ax.set_xticklabels(\n", - " [\n", - " r\"$\\xi_+(\\vartheta)$\",\n", - " r\"$\\xi_-(\\vartheta)$\",\n", - " r\"$\\tau_0(\\vartheta)$\",\n", - " r\"$\\tau_2(\\vartheta)$\",\n", - " ],\n", - " rotation=45,\n", - " )\n", - " fig.colorbar(im, ax=ax)\n", - "\n", - " plt.savefig(f\"./Plots/cov_matrix_{root}.png\", bbox_inches=\"tight\", dpi=300)\n", - " plt.show()\n", - " print(\"\\n\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "# Create dummy rho and tau stat handler.\n", - "\n", - "# Inference of the xi_sys parameters\n", - "sep_units = \"arcmin\"\n", - "coord_units = \"degrees\"\n", - "theta_min = 1.0\n", - "theta_max = 250\n", - "nbins = 20\n", - "\n", - "\n", - "TreeCorrConfig_xi = {\n", - " \"ra_units\": coord_units,\n", - " \"dec_units\": coord_units,\n", - " \"min_sep\": theta_min,\n", - " \"max_sep\": theta_max,\n", - " \"sep_units\": sep_units,\n", - " \"nbins\": nbins,\n", - " \"var_method\": \"jackknife\",\n", - "}\n", - "\n", - "rho_stats_handler = RhoStat(output=\".\", treecorr_config=TreeCorrConfig_xi, verbose=True)\n", - "\n", - "tau_stats_handler = TauStat(\n", - " catalogs=rho_stats_handler.catalogs,\n", - " output=\".\",\n", - " treecorr_config=TreeCorrConfig_xi,\n", - " verbose=True,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [ - "# Create a PSFErrorFit instance\n", - "psf_fitter = PSFErrorFit(\n", - " rho_stats_handler,\n", - " tau_stats_handler,\n", - " path_cosmo_val + \"rho_tau_stats/\",\n", - " use_eta=False,\n", - ")\n", - "\n", - "g = plots.get_subplot_plotter(width_inch=30)\n", - "\n", - "g.settings.axes_fontsize = 30\n", - "g.settings.axes_labelsize = 30\n", - "g.settings.alpha_filled_add = 0.7\n", - "g.settings.legend_fontsize = 40\n", - "\n", - "chains = []\n", - "\n", - "# Load rho-, tau-statistics, and cov_tau from the data_vector\n", - "for i, root in enumerate(roots):\n", - " print(\"Sampling PSF parameters for \", labels[i])\n", - " path_rho = f\"rho_stats_{root}.fits\"\n", - " path_tau = f\"tau_stats_{root}.fits\"\n", - " path_cov_rho = f\"cov_rho_{root}.npy\"\n", - " path_cov_tau = f\"cov_tau_{root}_th.npy\"\n", - " psf_fitter.load_rho_stat(path_rho)\n", - " psf_fitter.load_tau_stat(path_tau)\n", - " psf_fitter.load_covariance(path_cov_rho, cov_type=\"rho\")\n", - " psf_fitter.load_covariance(path_cov_tau, cov_type=\"tau\")\n", - " samples_lq, _, _ = psf_fitter.get_least_squares_params_samples(\n", - " npatch=None, apply_debias=False\n", - " )\n", - "\n", - " samples_gd = MCSamples(\n", - " samples=samples_lq, names=[r\"\\alpha\", r\"\\beta\"], labels=[r\"\\alpha\", r\"\\beta\"]\n", - " )\n", - "\n", - " chains.append(samples_gd)\n", - "\n", - "g.triangle_plot(\n", - " chains,\n", - " filled=True,\n", - " legend_labels=labels,\n", - " legend_loc=\"upper right\",\n", - ")\n", - "\n", - "# plt.savefig(f\"./Plots/psf_leakage_params.png\", bbox_inches='tight', dpi=300)\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "my_env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/glass_mock_hist.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/glass_mock_hist.ipynb deleted file mode 100644 index 32f89a18..00000000 --- a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/glass_mock_hist.ipynb +++ /dev/null @@ -1,586 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "0", - "metadata": { - "lines_to_next_cell": 2 - }, - "outputs": [], - "source": [ - "import IPython\n", - "\n", - "ipython = IPython.get_ipython()\n", - "\n", - "if ipython is not None:\n", - " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", - " ipython.run_line_magic(\"autoreload\", \"2\")\n", - "\n", - "import os\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import seaborn as sns\n", - "from getdist import plots\n", - "from tqdm import tqdm\n", - "\n", - "g = plots.get_subplot_plotter(width_inch=7)\n", - "g.settings.axes_fontsize = 15\n", - "g.settings.axes_labelsize = 15\n", - "g.settings.alpha_filled_add = 0.7\n", - "g.settings.legend_fontsize = 15\n", - "\n", - "if os.path.exists(\"/home/guerrini/matplotlib_config/paper.mplstyle\"):\n", - " plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", - "\n", - "# Set default palette - will be updated per plot as needed\n", - "sns.set_palette(\"husl\")\n", - "\n", - "if ipython is not None:\n", - " ipython.run_line_magic(\"matplotlib\", \"inline\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1", - "metadata": { - "lines_to_next_cell": 1 - }, - "outputs": [], - "source": [ - "root_dir = \"/n09data/guerrini/glass_mock_chains/\"\n", - "chain_version = \"v6\"\n", - "num_sims = 350\n", - "\n", - "roots = [f\"glass_mock_{chain_version}_{i + 1:05d}\" for i in range(num_sims)]\n", - "\n", - "\n", - "# # %%\n", - "def load_samples_and_write_paramames(root_dir, root, chain_type=\"configuration\"):\n", - " assert chain_type in [\"configuration\", \"harmonic\"], (\n", - " \"chain_type must be 'configuration' or 'harmonic'\"\n", - " )\n", - "\n", - " if chain_type == \"configuration\":\n", - " path_samples = root_dir + \"{}/{}/samples_{}.txt\".format(\"/\" + root, root, root)\n", - " path_paramnames = root_dir + \"{}/{}/getdist_{}.paramnames\".format(\n", - " \"/\" + root, root, root\n", - " )\n", - " else:\n", - " path_samples = root_dir + \"{}/{}/samples_{}_cell.txt\".format(\n", - " \"/\" + root, root, root\n", - " )\n", - " path_paramnames = root_dir + \"{}/{}/getdist_{}_cell.paramnames\".format(\n", - " \"/\" + root, root, root\n", - " )\n", - "\n", - " with open(path_samples, \"r\") as file:\n", - " params = file.readline()[1:].split(\"\\t\")[:-4]\n", - " file.close()\n", - "\n", - " with open(path_paramnames, \"w\") as file:\n", - " for i in range(len(params)):\n", - " if len(params[i].split(\"--\")) > 1:\n", - " file.write(params[i].split(\"--\")[1] + \"\\n\")\n", - " else:\n", - " file.write(params[i].split(\"--\")[0] + \"\\n\")\n", - " file.close()\n", - "\n", - "\n", - "def write_samples_getdist_format(root_dir, root, chain_type=\"configuration\"):\n", - " assert chain_type in [\"configuration\", \"harmonic\"], (\n", - " \"chain_type must be 'configuration' or 'harmonic'\"\n", - " )\n", - "\n", - " if chain_type == \"configuration\":\n", - " path_samples = root_dir + \"{}/{}/samples_{}.txt\".format(\"/\" + root, root, root)\n", - " path_gd_samples = root_dir + \"{}/{}/getdist_{}.txt\".format(\n", - " \"/\" + root, root, root\n", - " )\n", - " path_gd = root_dir + \"{}/{}/getdist_{}\".format(root, root, root)\n", - " else:\n", - " path_samples = root_dir + \"{}/{}/samples_{}_cell.txt\".format(\n", - " \"/\" + root, root, root\n", - " )\n", - " path_gd_samples = root_dir + \"{}/{}/getdist_{}_cell.txt\".format(\n", - " \"/\" + root, root, root\n", - " )\n", - " path_gd = root_dir + \"{}/{}/getdist_{}_cell\".format(root, root, root)\n", - "\n", - " samples = np.loadtxt(\n", - " path_samples,\n", - " )\n", - " if \"nautilus\" in root:\n", - " samples = np.column_stack(\n", - " (np.exp(samples[:, -3]), samples[:, -1] - samples[:, -2], samples[:, 0:-3])\n", - " )\n", - " else:\n", - " samples = np.column_stack((samples[:, -1], samples[:, -2], samples[:, 0:-4]))\n", - " np.savetxt(path_gd_samples, samples)\n", - "\n", - " chain = g.samples_for_root(\n", - " path_gd,\n", - " cache=False,\n", - " settings={\"ignore_rows\": 0.0, \"smooth_scale_2D\": 0.5, \"smooth_scale_1D\": 0.5},\n", - " )\n", - "\n", - " return chain\n", - "\n", - "\n", - "def extract_param_chain(chain, param_names):\n", - " margestats = chain.getMargeStats()\n", - " likestats = chain.getLikeStats()\n", - "\n", - " param_values = {}\n", - " for param_name in param_names:\n", - " if param_name not in chain.getParamNames().list():\n", - " raise ValueError(f\"Parameter {param_name} not found in chain.\")\n", - "\n", - " param_stats = margestats.parWithName(param_name)\n", - " param_values[param_name] = {\n", - " \"mean\": param_stats.mean,\n", - " \"1sigma_minus\": param_stats.mean - param_stats.limits[0].lower,\n", - " \"1sigma_plus\": param_stats.limits[0].upper - param_stats.mean,\n", - " \"2sigma_minus\": param_stats.mean - param_stats.limits[1].lower,\n", - " \"2sigma_plus\": param_stats.limits[1].upper - param_stats.mean,\n", - " }\n", - "\n", - " param_stats = likestats.parWithName(param_name)\n", - " param_names_getdist = chain.getParamNames()\n", - " par = param_names_getdist.parWithName(param_name)\n", - " kde = chain.get1DDensity(par, num_bins=1000)\n", - " kde_map = kde.x[np.argmax(kde.P)]\n", - " param_values[param_name].update(\n", - " {\n", - " \"MAP\": kde_map,\n", - " }\n", - " )\n", - "\n", - " par = chain.getParamNames().parWithName(\"S_8\")\n", - " par_om = chain.getParamNames().parWithName(\"OMEGA_M\")\n", - " kde = chain.get2DDensity(par, par_om, fine_bins_2D=1000)\n", - " s8_kde_map = kde.x[np.unravel_index(np.argmax(kde.P), kde.P.shape)[1]]\n", - " om_kde_map = kde.y[np.unravel_index(np.argmax(kde.P), kde.P.shape)[0]]\n", - " param_values[\"S_8\"].update(\n", - " {\n", - " \"MAP_2D\": s8_kde_map,\n", - " }\n", - " )\n", - " param_values[\"OMEGA_M\"].update(\n", - " {\n", - " \"MAP_2D\": om_kde_map,\n", - " }\n", - " )\n", - "\n", - " return param_values\n", - "\n", - "\n", - "def concatenate_param_stats(name, param_values, verbose=False):\n", - " output = [name]\n", - " for key in param_values.keys():\n", - " param_stat = param_values[key]\n", - " if verbose:\n", - " print(\n", - " f\"{name} - {key}: {param_stat['mean']:.4f} +{param_stat['1sigma_plus']:.4f}/-{param_stat['1sigma_minus']:.4f} (1σ), +{param_stat['2sigma_plus']:.4f}/-{param_stat['2sigma_minus']:.4f} (2σ)\"\n", - " )\n", - "\n", - " param_list = [\n", - " param_stat[\"mean\"],\n", - " param_stat[\"1sigma_minus\"],\n", - " param_stat[\"1sigma_plus\"],\n", - " param_stat[\"2sigma_minus\"],\n", - " param_stat[\"2sigma_plus\"],\n", - " param_stat[\"MAP\"],\n", - " ]\n", - "\n", - " if key == \"S_8\":\n", - " param_list.append(param_stat[\"MAP_2D\"])\n", - "\n", - " if key == \"OMEGA_M\":\n", - " param_list.append(param_stat[\"MAP_2D\"])\n", - "\n", - " output += param_list\n", - "\n", - " return output\n", - "\n", - "\n", - "def merge_param_stats(params_configuration, params_harmonic):\n", - " merged_params = {}\n", - " for key in params_configuration.keys():\n", - " if key in params_harmonic:\n", - " merged_params[key] = {\n", - " \"configuration\": params_configuration[key],\n", - " \"harmonic\": params_harmonic[key],\n", - " }\n", - " return merged_params\n", - "\n", - "\n", - "def concatenate_merge_params(name, merged_params, verbose=False):\n", - " output = [name]\n", - " for key in merged_params.keys():\n", - " param_config = merged_params[key][\"configuration\"]\n", - " param_harm = merged_params[key][\"harmonic\"]\n", - "\n", - " if verbose:\n", - " print(\n", - " f\"{name} - {key} (Configuration): {param_config['mean']:.4f} +{param_config['1sigma_plus']:.4f}/-{param_config['1sigma_minus']:.4f} (1σ), +{param_config['2sigma_plus']:.4f}/-{param_config['2sigma_minus']:.4f} (2σ)\"\n", - " )\n", - " print(\n", - " f\"{name} - {key} (Harmonic): {param_harm['mean']:.4f} +{param_harm['1sigma_plus']:.4f}/-{param_harm['1sigma_minus']:.4f} (1σ), +{param_harm['2sigma_plus']:.4f}/-{param_harm['2sigma_minus']:.4f} (2σ)\"\n", - " )\n", - "\n", - " param_list = [\n", - " param_config[\"mean\"],\n", - " param_config[\"1sigma_minus\"],\n", - " param_config[\"1sigma_plus\"],\n", - " param_config[\"2sigma_minus\"],\n", - " param_config[\"2sigma_plus\"],\n", - " param_config[\"MAP\"],\n", - " param_harm[\"mean\"],\n", - " param_harm[\"1sigma_minus\"],\n", - " param_harm[\"1sigma_plus\"],\n", - " param_harm[\"2sigma_minus\"],\n", - " param_harm[\"2sigma_plus\"],\n", - " param_harm[\"MAP\"],\n", - " ]\n", - "\n", - " output += param_list\n", - "\n", - " return output" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": { - "lines_to_next_cell": 0 - }, - "outputs": [], - "source": [ - "chain_harmonic = []\n", - "chain_config = []\n", - "\n", - "for i, root in enumerate(tqdm(roots)):\n", - " if os.path.isfile(f\"{root_dir}/{root}/{root}/getdist_{root}.txt\"):\n", - " # Load samples and write paramnames for harmonic space\n", - " load_samples_and_write_paramames(root_dir, root, chain_type=\"harmonic\")\n", - " write_samples_getdist_format(root_dir, root, chain_type=\"harmonic\")\n", - " chain_harm = g.samples_for_root(\n", - " root_dir + f\"/{root}/{root}/getdist_{root}_cell\",\n", - " cache=False,\n", - " settings={\n", - " \"ignore_rows\": 0.0,\n", - " \"smooth_scale_2D\": 0.5,\n", - " \"smooth_scale_1D\": 0.5,\n", - " },\n", - " )\n", - " chain_harmonic.append(chain_harm)\n", - "\n", - " # Load samples and write paramnames for harmonic space\n", - " load_samples_and_write_paramames(root_dir, root, chain_type=\"configuration\")\n", - " write_samples_getdist_format(root_dir, root, chain_type=\"configuration\")\n", - " chain_conf = g.samples_for_root(\n", - " root_dir + f\"/{root}/{root}/getdist_{root}\",\n", - " cache=False,\n", - " settings={\n", - " \"ignore_rows\": 0.0,\n", - " \"smooth_scale_2D\": 0.5,\n", - " \"smooth_scale_1D\": 0.5,\n", - " },\n", - " )\n", - " chain_config.append(chain_conf)\n", - "# # %%\n", - "param_names = [\"S_8\", \"OMEGA_M\", \"SIGMA_8\", \"a\"]\n", - "\n", - "output_mocks_harm = np.array(\n", - " [\n", - " \"Name\",\n", - " \"S8_mean\",\n", - " \"S8_1sigma_minus\",\n", - " \"S8_1sigma_plus\",\n", - " \"S8_2sigma_minus\",\n", - " \"S8_2sigma_plus\",\n", - " \"S8_MAP\",\n", - " \"S8_MAP_2D\",\n", - " \"OMEGA_M_mean\",\n", - " \"OMEGA_M_1sigma_minus\",\n", - " \"OMEGA_M_1sigma_plus\",\n", - " \"OMEGA_M_2sigma_minus\",\n", - " \"OMEGA_M_2sigma_plus\",\n", - " \"OMEGA_M_MAP\",\n", - " \"OMEGA_M_MAP_2D\",\n", - " \"SIGMA_8_mean\",\n", - " \"SIGMA_8_1sigma_minus\",\n", - " \"SIGMA_8_1sigma_plus\",\n", - " \"SIGMA_8_2sigma_minus\",\n", - " \"SIGMA_8_2sigma_plus\",\n", - " \"SIGMA_8_MAP\",\n", - " \"a_mean\",\n", - " \"a_1sigma_minus\",\n", - " \"a_1sigma_plus\",\n", - " \"a_2sigma_minus\",\n", - " \"a_2sigma_plus\",\n", - " \"a_MAP\",\n", - " ]\n", - ")\n", - "\n", - "output_mocks_config = np.array(\n", - " [\n", - " \"Name\",\n", - " \"S8_mean\",\n", - " \"S8_1sigma_minus\",\n", - " \"S8_1sigma_plus\",\n", - " \"S8_2sigma_minus\",\n", - " \"S8_2sigma_plus\",\n", - " \"S8_MAP\",\n", - " \"S8_MAP_2D\",\n", - " \"OMEGA_M_mean\",\n", - " \"OMEGA_M_1sigma_minus\",\n", - " \"OMEGA_M_1sigma_plus\",\n", - " \"OMEGA_M_2sigma_minus\",\n", - " \"OMEGA_M_2sigma_plus\",\n", - " \"OMEGA_M_MAP\",\n", - " \"OMEGA_M_MAP_2D\",\n", - " \"SIGMA_8_mean\",\n", - " \"SIGMA_8_1sigma_minus\",\n", - " \"SIGMA_8_1sigma_plus\",\n", - " \"SIGMA_8_2sigma_minus\",\n", - " \"SIGMA_8_2sigma_plus\",\n", - " \"SIGMA_8_MAP\",\n", - " \"a_mean\",\n", - " \"a_1sigma_minus\",\n", - " \"a_1sigma_plus\",\n", - " \"a_2sigma_minus\",\n", - " \"a_2sigma_plus\",\n", - " \"a_MAP\",\n", - " ]\n", - ")\n", - "\n", - "for i, root in enumerate(tqdm(roots[:-1])):\n", - " param_values_harm = extract_param_chain(chain_harmonic[i], param_names)\n", - "\n", - " param_harm = concatenate_param_stats(root, param_values_harm, verbose=False)\n", - "\n", - " output_mocks_harm = np.vstack((output_mocks_harm, param_harm))\n", - "\n", - " param_values_config = extract_param_chain(chain_config[i], param_names)\n", - "\n", - " param_config = concatenate_param_stats(root, param_values_config, verbose=False)\n", - "\n", - " output_mocks_config = np.vstack((output_mocks_config, param_config))\n", - "\n", - "np.savetxt(\n", - " f\"summary_parameter_constraints_harmonic_space_{chain_version}.txt\",\n", - " output_mocks_harm,\n", - " fmt=\"%s\",\n", - " delimiter=\";\",\n", - ")\n", - "np.savetxt(\n", - " f\"summary_parameter_constraints_configuration_space_{chain_version}.txt\",\n", - " output_mocks_config,\n", - " fmt=\"%s\",\n", - " delimiter=\";\",\n", - ")\n", - "print(\n", - " f\"Saved summary of parameter constraints for harmonic space in summary_parameter_constraints_harmonic_space_{chain_version}.txt\"\n", - ")\n", - "print(\n", - " f\"Saved summary of parameter constraints for configuration space in summary_parameter_constraints_configuration_space_{chain_version}.txt\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": { - "lines_to_next_cell": 0 - }, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "output_df_harm = pd.read_csv(\n", - " f\"summary_parameter_constraints_harmonic_space_{chain_version}.txt\",\n", - " delimiter=\";\",\n", - " skiprows=1,\n", - " names=output_mocks_harm[0],\n", - ")\n", - "\n", - "output_df_config = pd.read_csv(\n", - " f\"summary_parameter_constraints_configuration_space_{chain_version}.txt\",\n", - " delimiter=\";\",\n", - " skiprows=1,\n", - " names=output_mocks_config[0],\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "# Define the true value of the parameters\n", - "from astropy.cosmology import Planck18 as planck\n", - "\n", - "Omega_m_fid = planck.Om0\n", - "sigma_8_fid = 0.8102\n", - "s8_fid = sigma_8_fid * (Omega_m_fid / 0.3) ** 0.5\n", - "h = planck.h\n", - "Omega_b_fig = planck.Ob0\n", - "n_s_fid = 0.9665\n", - "print(\n", - " f\"Fiducial values: Omega_m = {Omega_m_fid}, sigma_8 = {sigma_8_fid}, S_8 = {s8_fid}\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5", - "metadata": {}, - "outputs": [], - "source": [ - "sns.histplot(\n", - " output_df_harm[\"S8_mean\"] - output_df_config[\"S8_mean\"],\n", - " kde=True,\n", - " bins=30,\n", - " label=\"Mean\",\n", - ")\n", - "# sns.histplot(\n", - "# output_df_harm[\"S8_MAP\"]-output_df_config[\"S8_MAP\"],\n", - "# kde=True,\n", - "# bins=20,\n", - "# label=\"MAP\",\n", - "# )\n", - "sns.histplot(\n", - " output_df_harm[\"S8_MAP_2D\"] - output_df_config[\"S8_MAP_2D\"],\n", - " kde=True,\n", - " bins=30,\n", - " label=\"2D Mode\",\n", - " alpha=0.5,\n", - ")\n", - "plt.axvline(0, color=\"black\", linestyle=\"--\")\n", - "plt.legend(fontsize=12)\n", - "\n", - "plt.xlabel(r\"$\\Delta S_8$\")\n", - "plt.savefig(\n", - " \"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/notebooks/Plots/S8_comparison_harmonic_vs_configuration.pdf\",\n", - " bbox_inches=\"tight\",\n", - ")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "output_df_config[\"S8_MAP_2D\"].shape\n", - "output_df_harm[\"S8_MAP_2D\"].shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": { - "lines_to_next_cell": 2 - }, - "outputs": [], - "source": [ - "# Create JointGrid\n", - "g = sns.JointGrid(\n", - " x=output_df_config[\"OMEGA_M_MAP_2D\"],\n", - " y=output_df_config[\"S8_MAP_2D\"],\n", - " height=7,\n", - " ratio=5,\n", - " space=0,\n", - ")\n", - "\n", - "# Main 2D histogram\n", - "sns.histplot(\n", - " x=output_df_config[\"OMEGA_M_MAP_2D\"],\n", - " y=output_df_config[\"S8_MAP_2D\"],\n", - " bins=25,\n", - " cmap=\"Greens\",\n", - " cbar=False,\n", - " ax=g.ax_joint,\n", - ")\n", - "\n", - "# Marginal histograms\n", - "sns.histplot(\n", - " x=output_df_config[\"OMEGA_M_MAP_2D\"], bins=25, color=\"#2ca25f\", ax=g.ax_marg_x\n", - ")\n", - "sns.histplot(y=output_df_config[\"S8_MAP_2D\"], bins=25, color=\"#2ca25f\", ax=g.ax_marg_y)\n", - "\n", - "# Add dashed reference lines\n", - "g.ax_joint.axvline(Omega_m_fid, color=\"k\", linestyle=\"--\")\n", - "g.ax_joint.axhline(s8_fid, color=\"k\", linestyle=\"--\")\n", - "\n", - "# Labels\n", - "g.set_axis_labels(\n", - " r\"$\\Omega_m$ estimated from mocks (Configuration space)\",\n", - " r\"$S_8$ estimated from mocks (Configuration space)\",\n", - ")\n", - "\n", - "# Optional styling tweaks\n", - "g.ax_joint.tick_params(labelsize=12)\n", - "plt.savefig(\n", - " \"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/notebooks/Plots/S8_vs_OmegaM_configuration_space_mocks.pdf\",\n", - " bbox_inches=\"tight\",\n", - ")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "jupytext": { - "cell_metadata_filter": "-all", - "main_language": "python", - "notebook_metadata_filter": "-all" - }, - "kernelspec": { - "display_name": "my_env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/masking.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/masking.ipynb deleted file mode 100644 index b0dd4bbc..00000000 --- a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/masking.ipynb +++ /dev/null @@ -1,132 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Covmat mask analysis\n", - "\n", - "This notebook creates the plots to look at the ratio of the covaraiance matrices when applying the mask or not" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "import healpy as hp\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import seaborn as sns\n", - "\n", - "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", - "\n", - "plt.rcParams[\"axes.labelsize\"] = 18\n", - "plt.rcParams[\"xtick.labelsize\"] = 18\n", - "plt.rcParams[\"ytick.labelsize\"] = 18\n", - "\n", - "plt.rcParams[\"text.usetex\"] = True\n", - "sns.set_palette(\"husl\")\n", - "\n", - "cat_dir = \"/n17data/UNIONS/WL/v1.4.x/\"\n", - "catalog_ver = \"v1.4.6.3\"\n", - "blind = \"B\"\n", - "\n", - "nside = 8192\n", - "npix = hp.nside2npix(nside)\n", - "\n", - "data_dir = \"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/data/\"\n", - "curr_dir = os.getcwd()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# PLOT 2D MAP OF COVMAT masked vs unmasked RATIOS\n", - "nbins = 20\n", - "ndata = nbins * 2\n", - "full_ratio = np.zeros((ndata, ndata))\n", - "\n", - "cov = np.loadtxt(data_dir + f\"/covs/cov_SP_{catalog_ver}_{blind}.txt\")\n", - "cov_masked = np.loadtxt(data_dir + f\"/covs/cov_masked_SP_{catalog_ver}_{blind}.txt\")\n", - "\n", - "for i in range(ndata):\n", - " for j in range(ndata):\n", - " full_ratio[i][j] = cov_masked[i][j] / cov[i][j]\n", - "\n", - "fig = plt.figure()\n", - "ax = fig.add_subplot(1, 1, 1)\n", - "extent = (0, ndata, ndata, 0)\n", - "\n", - "vmin, vmax = np.percentile(full_ratio, [1, 99])\n", - "\n", - "im3 = ax.imshow(full_ratio, cmap=\"RdBu_r\", vmin=vmin, vmax=vmax, extent=extent)\n", - "\n", - "cbar = fig.colorbar(im3, ax=ax, fraction=0.046, pad=0.04)\n", - "\n", - "ax.text(int(ndata / 4), ndata + 5, r\"$\\xi_+$\", fontsize=15)\n", - "ax.text(3 * int(ndata / 4), ndata + 5, r\"$\\xi_-$\", fontsize=15)\n", - "ax.text(-8, int(ndata / 4), r\"$\\xi_+$\", fontsize=15, rotation=90)\n", - "ax.text(-8, 3 * int(ndata / 4), r\"$\\xi_-$\", fontsize=15, rotation=90)\n", - "ax.set_xticks([0, 10, 20, 30, 40])\n", - "ax.set_yticks([0, 10, 20, 30, 40])\n", - "ax.set_yticklabels([\"1'\", \"125'\", \"250'\", \"125'\", \"250'\"])\n", - "ax.set_xticklabels([\"1'\", \"125'\", \"250'\", \"125'\", \"250'\"])\n", - "plt.axvline(x=int(ndata / 2), color=\"white\", linewidth=1.0)\n", - "plt.axhline(y=int(ndata / 2), color=\"white\", linewidth=1.0)\n", - "\n", - "plt.savefig(\n", - " f\"{curr_dir}/../Plots/covmat_masked_unmasked_ratio_{catalog_ver}_{blind}.pdf\",\n", - " bbox_inches=\"tight\",\n", - ")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "theta = np.linspace(1, 250, 20)\n", - "plt.axhline(y=1, color=\"k\", ls=\"--\")\n", - "plt.plot(theta, np.diag(cov_masked)[:20] / np.diag(cov)[:20], label=r\"$\\xi_+$\")\n", - "plt.plot(theta, np.diag(cov_masked)[20:] / np.diag(cov)[20:], label=r\"$\\xi_-$\")\n", - "\n", - "plt.xlabel(r\"$\\theta$ (arcmin)\")\n", - "plt.ylabel(\"Cov masked / Cov unmasked\")\n", - "plt.legend(fontsize=20)\n", - "plt.savefig(\n", - " f\"{curr_dir}/../Plots/covmat_masked_unmasked_ratio_diag.pdf\", bbox_inches=\"tight\"\n", - ")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "my_env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/nonlin_k_analysis.ipynb b/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/nonlin_k_analysis.ipynb deleted file mode 100644 index 4ab9c3c5..00000000 --- a/cosmo_inference/notebooks/2D_cosmic_shear_configuration_plots/nonlin_k_analysis.ipynb +++ /dev/null @@ -1,174 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Nonlinear $k$ contributions\n", - "\n", - "This notebook plots the 2D heatmap of ratio of scale contributions to the $\\xi_\\pm$ 2PCF given angular scale $\\theta$ and wavenumber $k$." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "import matplotlib.pylab as plt\n", - "import numpy as np\n", - "import seaborn as sns\n", - "\n", - "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", - "\n", - "plt.rcParams[\"text.usetex\"] = True\n", - "\n", - "plt.rcParams.update(\n", - " {\n", - " \"font.size\": 20,\n", - " \"axes.titlesize\": 21,\n", - " \"axes.labelsize\": 20,\n", - " \"xtick.labelsize\": 20,\n", - " \"ytick.labelsize\": 20,\n", - " \"legend.fontsize\": 20,\n", - " \"figure.titlesize\": 21,\n", - " }\n", - ")\n", - "sns.set_palette(\"husl\")\n", - "\n", - "blind = \"B\"\n", - "ver = \"v1.4.6.3\"\n", - "\n", - "%matplotlib inline\n", - "\n", - "data_dir = \"/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/data/\"\n", - "curr_dir = os.getcwd()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Plotting from script" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Read the 2D array from the text file\n", - "\n", - "file_headers = [\"xip_%s_%s\" % (ver, blind), \"xim_%s_%s\" % (ver, blind)]\n", - "\n", - "for f in file_headers:\n", - " xis = np.loadtxt(data_dir + f\"theta_k_{f}.txt\")\n", - " xis_reshaped = xis.reshape(-1, 201)\n", - " sorted_xis = xis_reshaped[np.argsort(xis_reshaped[:, 0])]\n", - "\n", - " np.savetxt(data_dir + f\"theta_k_{f}_sorted.txt\", sorted_xis)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "fig, axs = plt.subplots(2, 1, figsize=(8, 10))\n", - "\n", - "# --- k grid ---\n", - "h = 0.6766\n", - "k_plot = np.logspace(-4, 2, 200)\n", - "\n", - "file_header = \"%s_%s\" % (ver, blind)\n", - "\n", - "xi_thetas = np.loadtxt(data_dir + f\"theta_k_xip_{file_header}_sorted.txt\")\n", - "thetas = xi_thetas[:, 0]\n", - "xis = xi_thetas[:, 1:]\n", - "\n", - "# normalise\n", - "xi_plot = xis / np.max(xis, axis=1, keepdims=True)\n", - "\n", - "T, K = np.meshgrid(thetas, k_plot)\n", - "\n", - "axs[0].contour(T, K, xi_plot.T, levels=[0.9], colors=\"red\", linewidths=1.7)\n", - "pcm = axs[0].pcolormesh(T, K, xi_plot.T, shading=\"auto\", cmap=\"viridis\")\n", - "pcm.set_rasterized(True)\n", - "\n", - "axs[0].axvline(5, color=\"k\", ls=\"dashed\", lw=1.2)\n", - "axs[0].axvline(12, color=\"white\", ls=\"dashed\", lw=1.6)\n", - "axs[0].axhline(1, color=\"k\", ls=\"dashed\", lw=1.2) # converted to h/Mpc space if needed\n", - "axs[0].axhline(0.425, color=\"white\", ls=\"dashed\", lw=1.6)\n", - "\n", - "axs[0].set_yscale(\"log\")\n", - "axs[0].set_xlabel(r\"$\\theta\\ \\mathrm{(arcmin)}$\")\n", - "axs[0].set_ylabel(r\"$k\\ (h$ Mpc$^{-1})$\")\n", - "\n", - "axs[0].set_title(r\"$\\xi_+$\")\n", - "\n", - "xi_thetas = np.loadtxt(data_dir + f\"theta_k_xim_{file_header}_sorted.txt\")\n", - "thetas = xi_thetas[:, 0]\n", - "xis = xi_thetas[:, 1:]\n", - "\n", - "xi_plot = xis / np.max(xis, axis=1, keepdims=True)\n", - "\n", - "T, K = np.meshgrid(thetas, k_plot)\n", - "\n", - "axs[1].contour(T, K, xi_plot.T, levels=[0.9], colors=\"red\", linewidths=1.7)\n", - "pcm = axs[1].pcolormesh(T, K, xi_plot.T, shading=\"nearest\", cmap=\"viridis\")\n", - "pcm.set_rasterized(True)\n", - "\n", - "axs[1].axvline(12, color=\"white\", ls=\"dashed\", lw=1.6)\n", - "axs[1].axhline(2.85, color=\"white\", ls=\"dashed\", lw=1.6)\n", - "\n", - "\n", - "axs[1].set_yscale(\"log\")\n", - "axs[1].set_xlabel(r\"$\\theta\\ \\mathrm{(arcmin)}$\")\n", - "axs[1].set_ylabel(r\"$k\\ (h$ Mpc$^{-1})$\")\n", - "axs[1].set_title(r\"$\\xi_-$\")\n", - "\n", - "\n", - "fig.tight_layout()\n", - "\n", - "cbar_ax = fig.add_axes([0.99, 0.15, 0.02, 0.7])\n", - "cbar = fig.colorbar(pcm, cax=cbar_ax)\n", - "\n", - "fig.savefig(\n", - " curr_dir + f\"/../Plots/theta_k_xip_xim_{ver}_{blind}.pdf\", bbox_inches=\"tight\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "my_env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_unblinding/unblinding_party_plots.py b/cosmo_inference/notebooks/2D_cosmic_shear_unblinding/unblinding_party_plots.py deleted file mode 100644 index 13af7ed2..00000000 --- a/cosmo_inference/notebooks/2D_cosmic_shear_unblinding/unblinding_party_plots.py +++ /dev/null @@ -1,894 +0,0 @@ -# %% -import os -import sys -import warnings - -# Append any useful folder in the path -sys.path.append("/home/guerrini/sp_validation/cosmo_inference/scripts/") -sys.path.append( - "/home/guerrini/sp_validation/cosmo_inference/notebooks/2D_cosmic_shear_unblinding/" -) - -import matplotlib.pyplot as plt -import matplotlib.scale as mscale -import numpy as np -import seaborn as sns -from astropy.io import fits -from getdist import plots -from IPython.display import Markdown, display - -from sp_validation.rho_tau import SquareRootScale - -mscale.register_scale(SquareRootScale) - -import IPython - -ipython = IPython.get_ipython() - -if ipython is not None: - ipython.run_line_magic("load_ext", "autoreload") - ipython.run_line_magic("autoreload", "2") - -if ipython is not None: - ipython.run_line_magic("matplotlib", "inline") - -import chain_postprocessing as cp -import utils - -plt.style.use( - "/home/guerrini/sp_validation/papers/harmonic/matplotlib_config/paper.mplstyle" -) - -plt.rcParams["text.usetex"] = True - -sns.set_palette("husl") - -g = plots.get_subplot_plotter(width_inch=30) -g.settings.axes_fontsize = 30 -g.settings.axes_labelsize = 30 -g.settings.alpha_filled_add = 0.7 -g.settings.legend_fontsize = 40 - -# Directory where the chains are located -root_dir = "/n09data/guerrini/output_chains" - -# THE BLIND TO USE FOR THE PLOTS -blind = "B" # Options are "A", "B", or "C" -catalog_version = "SP_v1.4.6.3" -fiducial_root_cell = f"SP_v1.4.6.3_leak_corr_{blind}" -label_fiducial_cell = r"UNIONS $C_{\ell}$" -fiducial_root_xi_data = f"SP_v1.4.6.3_leak_corr_{blind}_masked" -fiducial_root_xi_chains = f"SP_v1.4.6.3_{blind}_fiducial_config" -label_fiducial_xi = r"UNIONS $\xi_{\pm}$" - -# Path to the ini files used -path_ini_files = "/home/guerrini/sp_validation/cosmo_inference/cosmosis_config" -path_datavectors = "/home/guerrini/sp_validation/cosmo_inference/data/" -path_output_chains = "/n09data/guerrini/output_chains/" - -# %% -# 0. Do a funny print with emojis for the unblinding -display(Markdown("## 🎉 Let the Unblinding Party begin 🎉")) -# %% -# 1. Plot the datavectors without best-fit -display(Markdown("### 1.a. Plot the datavectors without best-fit")) - -# Plot Cells EE -data = fits.open( - os.path.join( - path_datavectors, f"{fiducial_root_cell}/cosmosis_{fiducial_root_cell}.fits" - ) -) -cell_ee = data["CELL_EE"].data -cov_mat = data["COVMAT"].data - -# Plot hyperparameter -loc_legend = "lower center" -bbox_to_anchor = (0.685, 0.70) - -fig, ax = plt.subplots(1, 1, figsize=(8, 5)) - -ell, cell = cell_ee["ANG"], cell_ee["VALUE"] -ax.errorbar( - ell, - ell * cell, - yerr=ell * np.sqrt(np.diag(cov_mat)), - fmt="o", - label=r"UNIONS $C_{\ell}$ data", - color="black", - capsize=2, -) - -# Plot the scale cuts for different k_max -ax.axvline(x=1800, color="black", linestyle="--", alpha=0.5) -ax.axvline(x=2048, color="black", linestyle="--", alpha=1.0) -ax.axvline(x=500, color="black", linestyle="--", alpha=0.3) - -ymin = ax.get_ylim()[0] -ymax = ax.get_ylim()[1] -# Shadowing cut scaled -ax.fill_betweenx( - y=[ymin, ymax], - x1=0, - x2=300, - color="gray", - alpha=0.2, - label=r"$B$-mode informed scale cut", -) -ax.fill_betweenx(y=[ymin, ymax], x1=1600, x2=2048, color="gray", alpha=0.2) - -ax.set_ylim(ymin, ymax) - -# Add labels directly under the tick -ax.text( - 1740, - 0.90, - r"$k_\mathrm{max} = 3 h$ Mpc$^{-1}$", - transform=ax.get_xaxis_transform(), - ha="center", - va="top", - fontsize=10, - rotation=90, -) - -ax.text( - 1978, - 0.90, - r"$k_\mathrm{max} = 5 h$ Mpc$^{-1}$", - transform=ax.get_xaxis_transform(), - ha="center", - va="top", - fontsize=10, - rotation=90, -) - -ax.text( - 470, - 0.90, - r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", - transform=ax.get_xaxis_transform(), - ha="center", - va="top", - fontsize=10, - rotation=90, -) - -ell, cell = cell_ee["ANG"], cell_ee["VALUE"] -ax.set_ylabel(r"$\ell C_\ell$", fontsize=16) -ax.set_xlabel(r"$\ell$", fontsize=16) -ax.set_xlim(ell.min() - 10, ell.max() + 100) -ax.set_xscale("squareroot") -ax.set_xticks(np.array([100, 400, 900, 1600])) -ax.minorticks_on() -ax.tick_params(axis="x", which="minor", length=2, width=0.8) -minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] -ax.xaxis.set_ticks(minor_ticks, minor=True) -ax.tick_params(axis="both", which="major", labelsize=14) -ax.tick_params(axis="both", which="minor", labelsize=10) -ax.yaxis.get_offset_text().set_fontsize(14) - -plt.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor, fontsize=12) - -plt.show() - -# Plots xi_+ and xi_- - -# TODO: add the plot for xi_+ and xi_- -display(Markdown(r"### 1.b. Plot the datavectors without best-fit ($\xi_\pm$)")) - -# Plot xi_pm's -data = fits.open( - os.path.join( - path_datavectors, - f"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}/cosmosis_{fiducial_root_xi_data}.fits", - ) -) -xi_p_data = data["XI_PLUS"].data -xi_m_data = data["XI_MINUS"].data -cov_mat = data["COVMAT"].data - -# Plot hyperparameter -loc_legend = "lower center" -bbox_to_anchor_xip = (0.685, 0.03) -bbox_to_anchor_xim = (0.3, 0.65) - -fig, [ax, ax2] = plt.subplots(2, 1, figsize=(8, 9)) - -theta, xi_p = xi_p_data["ANG"], xi_p_data["VALUE"] -ax.errorbar( - theta, - theta * xi_p, - yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])), - fmt="o", - label=r"UNIONS $\xi_+$ data", - color="black", - capsize=2, -) - -# Plot the scale cuts for different k_max -ax.axvline(x=3.2, color="black", linestyle="--", alpha=0.3) - -ymin = ax.get_ylim()[0] -ymax = ax.get_ylim()[1] -# Shadowing cut scaled -ax.fill_betweenx( - y=[ymin, ymax], - x1=0, - x2=12, - color="gray", - alpha=0.2, - label=r"$B$-mode informed scale cut", -) -ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color="gray", alpha=0.2) - -ax.set_ylim(ymin, ymax) - -# Add labels directly under the tick -ax.text( - 3, - 1e-4, - r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", - # transform=ax.get_xaxis_transform(), - ha="center", - va="top", - fontsize=10, - rotation=90, -) - -ax.set_ylabel(r"$\theta \xi_+$", fontsize=16) -# ax.set_xlabel('$\theta$', fontsize=16) -# ax.set_xlim([theta.min()-0.1, theta.max()+20]) -ax.set_xscale("log") -ax.set_xticks(np.array([1, 10, 100])) -ax.tick_params(axis="x", which="minor", length=2, width=0.8) -ax.tick_params(axis="both", which="major", labelsize=14) -ax.tick_params(axis="both", which="minor", labelsize=10) -ax.yaxis.get_offset_text().set_fontsize(14) -ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) -ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=12) - -theta, xi_m = xi_p_data["ANG"], xi_m_data["VALUE"] -ax2.errorbar( - theta, - theta * xi_m, - yerr=theta - * np.sqrt( - np.diag(cov_mat[len(theta) : 2 * len(theta), len(theta) : 2 * len(theta)]) - ), - fmt="o", - label=r"UNIONS $\xi_-$ data", - color="black", - capsize=2, -) - -# Plot the scale cuts for different k_max -ax2.axvline(x=24, color="black", linestyle="--", alpha=0.3) - -ymin = ax2.get_ylim()[0] -ymax = ax2.get_ylim()[1] -# Shadowing cut scaled -ax2.fill_betweenx( - y=[ymin, ymax], - x1=0, - x2=12, - color="gray", - alpha=0.2, - label=r"$B$-mode informed scale cut", -) -ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color="gray", alpha=0.2) - -ax2.set_ylim(ymin, ymax) - -# Add labels directly under the tick -ax2.text( - 22.3, - 9e-5, - r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", - # transform=ax.get_xaxis_transform(), - ha="center", - va="top", - fontsize=10, - rotation=90, -) - -ax2.set_ylabel(r"$\theta÷ \xi_-$", fontsize=16) -ax2.set_xlabel("$\theta$", fontsize=16) -ax2.set_xlim([theta.min() - 0.1, theta.max() + 20]) -ax2.set_xscale("log") -ax2.set_xticks(np.array([1, 10, 100])) -ax2.tick_params(axis="x", which="minor", length=2, width=0.8) -ax2.tick_params(axis="both", which="major", labelsize=14) -ax2.tick_params(axis="both", which="minor", labelsize=10) -ax2.yaxis.get_offset_text().set_fontsize(14) -ax2.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) -ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=12) - -plt.show() - -# %% -# 2. Plot the best-fit datavectors -display(Markdown("### 2. Plot the best-fit datavectors")) - -# Perform the computation for the fiducial of Cell -path_samples_fiducial_cell = os.path.join( - path_output_chains, - fiducial_root_cell, - fiducial_root_cell, - f"samples_{fiducial_root_cell}_cell.txt", -) -path_gd_fiducial_cell = os.path.join( - path_output_chains, - fiducial_root_cell, - fiducial_root_cell, - f"getdist_{fiducial_root_cell}_cell", -) -cp.load_samples_and_write_paramnames( - path_samples_fiducial_cell, path_gd_fiducial_cell + ".paramnames" -) -cp.write_samples_getdist_format( - path_samples_fiducial_cell, path_gd_fiducial_cell + ".txt", chain_type="polychord" -) - -chain_fiducial_cell = cp.load_chain(path_gd_fiducial_cell, smoothing_scale=0.3) - -best_fit_params_fiducial_cell = cp.extract_best_fit_params(chain_fiducial_cell) - -cp.compute_best_fit( - path_ini_files, - best_fit_params_fiducial_cell, - fiducial_root_cell, - is_harmonic=True, - blind=blind, -) - -# Perform the computation for the fiducial of xi -path_samples_fiducial_xi = os.path.join( - path_output_chains, - fiducial_root_xi_chains, - f"samples_{fiducial_root_xi_chains}.txt", -) -path_gd_fiducial_xi = os.path.join( - path_output_chains, fiducial_root_xi_chains, f"getdist_{fiducial_root_xi_chains}" -) -cp.load_samples_and_write_paramnames( - path_samples_fiducial_xi, path_gd_fiducial_xi + ".paramnames" -) -cp.write_samples_getdist_format( - path_samples_fiducial_xi, path_gd_fiducial_xi + ".txt", chain_type="polychord" -) - -chain_fiducial_xi = cp.load_chain(path_gd_fiducial_xi, smoothing_scale=0.3) - -best_fit_params_fiducial_xi = cp.extract_best_fit_params(chain_fiducial_xi) - -ini_file_root = os.path.join( - path_ini_files, - f"config_space_v1.4.6.3_fiducial/pipeline/blind_{blind}/fiducial.ini", -) -cp.compute_best_fit( - path_ini_files, - best_fit_params_fiducial_xi, - fiducial_root_xi_chains, - is_harmonic=False, - blind=blind, - ini_file_root=ini_file_root, -) - -# %% -# Make the plot for the best-fit datavector for Cell EE -root_to_plot = [fiducial_root_cell, fiducial_root_xi_chains] - -labels = [r"UNIONS $C_\ell$", r"UNIONS $\xi_\pm(\vartheta)$"] - -line_args = [ - {"color": "royalblue", "linestyle": "-"}, - {"color": "orange", "linestyle": "-"}, -] - -properties = {} - -properties = utils.update_properties_w_roots( - properties, fiducial_root_cell, path_ini_files, with_configuration=False -) -properties = utils.update_properties_w_roots( - properties, - fiducial_root_xi_chains, - path_ini_files, - with_configuration=True, - path_to_this_ini=ini_file_root, -) - -utils.plot_best_fit( - fiducial_root_cell, - root_to_plot, - path_output_chains, - line_args, - savefile=None, - labels=labels, - loc_legend=loc_legend, - bbox_to_anchor=bbox_to_anchor, - properties=properties, -) - -# TODO: add the plot for xi -# %% -# Plot best-fit xi_+ and xi_- (also from C_ell's) - -path_best_fit_xi_theta = os.path.join( - path_output_chains, fiducial_root_xi_chains, "best_fit/shear_xi_plus/theta.txt" -) -theta_rad = np.loadtxt(path_best_fit_xi_theta) - -cp.compute_best_fit_xi_from_cell( - path_output_chains, fiducial_root_cell, best_fit_params_fiducial_cell, theta_rad -) - - -xi_data_path = os.path.join( - path_datavectors, - f"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}/cosmosis_{fiducial_root_xi_data}.fits", -) -utils.plot_best_fit_config( - xi_data_path, - root_to_plot, - path_output_chains, - line_args, - savefile=None, - labels=labels, - loc_legend=loc_legend, - bbox_to_anchor_xip=bbox_to_anchor_xip, - bbox_to_anchor_xim=bbox_to_anchor_xim, - properties=properties, -) - -# %% -# 3. Do a whisker plot with external experiments and our constraints -display(Markdown("### 🥸 Time to look at the whisker plot 🥸")) -colour_blind = {"A": "royalblue", "B": "crimson", "C": "forestgreen"} - -roots = [ - f"SP_v1.4.6.3_leak_corr_{blind}", - f"SP_v1.4.6.3_{blind}_fiducial_config", - "Planck18", - "DES_Y3", - "DES_Y3_cell", - "KiDS-1000", - "KiDS-1000_cosebis", - "KiDS-1000_bp", - "DES+KiDS", - "HSC_Y3", - "HSC_Y3_cell", -] - -legend_labels = [ - r"UNIONS $C_\ell$, unblind", - r"UNIONS $\xi_\pm(\vartheta)$, unblind", - r"\textit{Planck} 2018", - r"DES Y3 $\xi_\pm(\vartheta)$", - r"DES Y3 $C_\ell$", - r"KiDS-1000 $\xi_\pm(\vartheta)$", - r"KiDS-1000 $E_n$", - r"KiDS-1000 $C_E$", - r"DES Y3 + KiDS-1000 combined", - r"HSC Y3 $\xi_\pm(\vartheta)$", - r"HSC Y3 $C_\ell$", -] - -colours = [ - colour_blind[blind], - colour_blind[blind], - "violet", - "black", - "black", - "black", - "black", - "black", - "black", - "black", - "black", -] - -categories = [ - "harmonic", - "configuration", - "external", - "external", - "external", - "external", - "external_compute_sample", - "external_compute_sample", - "external", - "external", - "external_compute_sample", -] - -for bl in ["A", "B", "C"]: - if bl != blind: - roots.append(f"SP_v1.4.6.3_leak_corr_{bl}") - roots.append(f"SP_v1.4.6.3_{bl}_fiducial_config") - legend_labels.append(rf"UNIONS $C_\ell$, Blind {bl}") - legend_labels.append(rf"UNIONS $\xi_\pm(\vartheta)$, Blind {bl}") - colours.append(colour_blind[bl]) - colours.append(colour_blind[bl]) - categories.append("harmonic") - categories.append("configuration") - -# Loop on all versions to load the chain -chains = [] -for i, root in enumerate(roots): - category = categories[i] - if category != "external": - if category == "configuration": - path_samples = os.path.join( - path_output_chains, f"{root}/samples_{root}.txt" - ) - path_getdist = os.path.join(path_output_chains, f"{root}/getdist_{root}") - elif category == "harmonic": - path_samples = os.path.join( - path_output_chains, f"{root}/{root}/samples_{root}_cell.txt" - ) - path_getdist = os.path.join( - path_output_chains, f"{root}/{root}/getdist_{root}" - ) - elif category == "external_compute_sample": - path_samples = os.path.join( - path_output_chains, f"ext_data/{root}/samples_{root}.txt" - ) - path_getdist = os.path.join( - path_output_chains, f"ext_data/{root}/getdist_{root}" - ) - else: - raise ValueError(f"The category, {category}, of {root} is not correct") - - cp.load_samples_and_write_paramnames(path_samples, path_getdist + ".paramnames") - cp.write_samples_getdist_format(path_samples, path_getdist + ".txt") - chains.append(cp.load_chain(path_getdist, smoothing_scale=0.5)) - else: - path_getdist = os.path.join( - path_output_chains, f"ext_data/{root}/getdist_{root}" - ) - chains.append(cp.load_chain(path_getdist)) - -# Give labels for the chains -name_list = [ - "OMEGA_M", - "ombh2", - "h0", - "n_s", - "SIGMA_8", - "S_8", - "s_8_input", - "logt_agn", - "a", - "m1", - "bias_1", -] -label_list = [ - r"\Omega_{\rm m}", - r"\omega_b h^2", - r"h_0", - r"n_s", - r"\sigma_8", - r"S_8", - r"S_8", - r"\log T_{\rm AGN}", - r"A_{\rm IA}", - r"m_1", - r"\Delta z_1", -] - -for i, chain in enumerate(chains): - print(legend_labels[i]) - param_names = chain.getParamNames() - for name, label in zip(name_list, label_list): - try: - param_names.parWithName(name).label = label - except Exception: - warnings.warn(f"Parameter {name} not found in chain {roots[i]}.") - -# Account for the missing parameter conventions -# OMEGA_M not in DES_Y3_cell -idx = roots.index("DES_Y3_cell") -cp.adjust_paramname_chain(chains[idx], "omega_m", "OMEGA_M", r"\Omega_{\rm m}") -cp.derive_parameter_S8(chains[idx]) - -# OMEGA_M not in KiDS-1000 -idx = roots.index("KiDS-1000") -cp.adjust_paramname_chain(chains[idx], "omega_m", "OMEGA_M", r"\Omega_{\rm m}") - -# OMEGA_M not in DES+KiDS -idx = roots.index("DES+KiDS") -cp.adjust_paramname_chain(chains[idx], "omega_m", "OMEGA_M", r"\Omega_{\rm m}") - -# OMEGA_M not in HSC_Y3_cell -idx = roots.index("HSC_Y3_cell") -cp.adjust_paramname_chain(chains[idx], "omega_m", "OMEGA_M", r"\Omega_{\rm m}") - -# Build an array containing the parameter values -param_values = np.array( - [ - "# Expt", - "Colour", - "S8_Mean", - "S8_low", - "S8_high", - "sigma_8_Mean", - "sigma_8_low", - "sigma_8_high", - "Omega_m_Mean", - "Omega_m_low", - "Omega_m_high", - ] -) -escaped = np.char.replace(legend_labels, "\\", "\\\\") -for i, chain in enumerate(chains): - print(chain.root) - margestats = chain.getMargeStats() - likestats = chain.getLikeStats() - - s8_stats = margestats.parWithName("S_8") - sigma8_stats = margestats.parWithName("SIGMA_8") - omegam_stats = margestats.parWithName("OMEGA_M") - - param_values = np.vstack( - ( - param_values, - [ - escaped[i], - colours[i], - s8_stats.mean, - s8_stats.mean - s8_stats.limits[0].lower, - s8_stats.limits[0].upper - s8_stats.mean, - sigma8_stats.mean, - sigma8_stats.mean - sigma8_stats.limits[0].lower, - sigma8_stats.limits[0].upper - sigma8_stats.mean, - omegam_stats.mean, - omegam_stats.mean - omegam_stats.limits[0].lower, - omegam_stats.limits[0].upper - omegam_stats.mean, - ], - ) - ) -print(param_values) -np.savetxt( - "./param_values.txt", param_values, fmt=["%s" for i in range(11)], delimiter=";" -) - -# Reload the table -# Load the value of the parameters -cosmo = np.loadtxt( - "./param_values.txt", - dtype={ - "names": ( - "Expt", - "colour", - "s8_mean", - "s8_low", - "s8_high", - "sigma8_mean", - "sigma8_low", - "sigma8_high", - "omegam_mean", - "omegam_low", - "omegam_high", - ), - "formats": ( - "U250", - "U20", - "U20", - "U20", - "U20", - "U20", - "U20", - "U20", - "U20", - "U20", - "U20", - ), - }, - skiprows=1, - delimiter=";", -) -expt = np.char.replace(cosmo["Expt"], "\\\\", "\\") -colours = cosmo["colour"] -s8_mean = cosmo["s8_mean"].astype(np.float64) -s8_low = cosmo["s8_low"].astype(np.float64) -s8_high = cosmo["s8_high"].astype(np.float64) -sigma8_mean = cosmo["sigma8_mean"].astype(np.float64) -sigma8_low = cosmo["sigma8_low"].astype(np.float64) -sigma8_high = cosmo["sigma8_high"].astype(np.float64) -omegam_mean = cosmo["omegam_mean"].astype(np.float64) -omegam_low = cosmo["omegam_low"].astype(np.float64) -omegam_high = cosmo["omegam_high"].astype(np.float64) - -# %% -# Perform the plot -from matplotlib.gridspec import GridSpec - -fig = plt.figure(figsize=(10, 6)) -gs = GridSpec(1, 3, width_ratios=[1, 0.5, 0.5]) -ax1 = fig.add_subplot(gs[0]) -ax2 = fig.add_subplot(gs[1], sharey=ax1) -ax3 = fig.add_subplot(gs[2], sharey=ax1) - -axs = [ax1, ax2, ax3] - -params = [ - (s8_mean, s8_low, s8_high, r"$S_8$"), - (sigma8_mean, sigma8_low, sigma8_high, r"$\sigma_8$"), - (omegam_mean, omegam_low, omegam_high, r"$\Omega_{\rm m}$"), -] -reference = r"UNIONS $C_\ell$, unblind" -separation_after = [ - r"UNIONS $\xi_\pm(\vartheta)$, unblind", - r"HSC Y3 $C_\ell$", -] -list_section_index = [r"(ii)", r"(iii)", r"(iv)", r"(v)", r"(vi)", r"(vii)"] - -preliminary_watermark = False -blind_axes = False -row_spacing = 0.1 - -index_ref = np.where(expt == reference)[0][0] - -y = np.arange(len(expt)) -for ax, param in zip(axs, params): - means, lows, highs, label = param - for i, mean, low, high, color in zip(y, means, lows, highs, colours): - ax.errorbar( - mean, - 0.05 + i * row_spacing, - xerr=np.array([low, high])[:, None], - fmt="o", - color=color, - ecolor=color, - elinewidth=2, - capsize=3, - ) - ax.set_xlabel(label, fontsize=14) - - ax.grid(False) - ax.tick_params(axis="y", left=False, labelleft=False) - if label == r"$S_8$": - ax.axvspan( - s8_mean[index_ref] - s8_low[index_ref], - s8_mean[index_ref] + s8_high[index_ref], - color=colours[index_ref], - alpha=0.2, - ) - ax.set_xlim(0.25, 1.1) - if blind_axes: - ref_tick = np.mean(s8_mean[:4]) - ax.set_xticks([ref_tick + i * 0.1 for i in range(-5, 5)], labels=[]) - elif label == r"$\sigma_8$": - ax.axvspan( - sigma8_mean[index_ref] - sigma8_low[index_ref], - sigma8_mean[index_ref] + sigma8_high[index_ref], - color=colours[index_ref], - alpha=0.2, - ) - ax.set_xlim(0.5, 1.2) - if blind_axes: - ref_tick = np.mean(sigma8_mean[:4]) - ax.set_xticks([ref_tick + i * 0.2 for i in range(-2, 2)], labels=[]) - elif label == r"$\Omega_{\rm m}$": - ax.axvspan( - omegam_mean[index_ref] - omegam_low[index_ref], - omegam_mean[index_ref] + omegam_high[index_ref], - color=colours[index_ref], - alpha=0.2, - ) - ax.set_xlim(0.1, 0.5) - if blind_axes: - ref_tick = np.mean(omegam_mean[:4]) - ax.set_xticks([ref_tick + i * 0.1 for i in range(-2, 3)], labels=[]) - - -axs[0].set_yticks(0.05 + y * row_spacing) -axs[0].set_yticklabels([]) -for label, color in zip(expt, colours): - axs[0].text( - 0.26, - 0.05 + row_spacing * np.where(expt == label)[0][0], - label, - fontsize=12, - ha="left", - va="center", - color=color, - ) - if label != reference: - index = np.where(expt == label)[0][0] - s8_tension = cp.get_sigma_tension( - s8_mean[index], - s8_low[index], - s8_high[index], - s8_mean[index_ref], - s8_low[index_ref], - s8_high[index_ref], - ) - sign_str = "+" if s8_tension > 0 else "-" - axs[0].text( - 1.095, - 0.05 + row_spacing * index, - rf"${sign_str}{np.abs(s8_tension):.2f}" + r"\, \sigma$", - fontsize=10, - ha="right", - va="center", - color=color, - ) -# Add separation lines -for i, sep in enumerate(separation_after): - index_sep = np.where(expt == sep)[0][0] - for ax in axs: - ax.axhline( - row_spacing * (index_sep + 1), - color="black", - linestyle="dotted", - linewidth=1, - ) - axs[0].text( - 0.25, - 0.05 + row_spacing * (index_sep + 1), - list_section_index[i], - fontsize=14, - fontweight="bold", - va="center", - ha="right", - ) - - -# --- Add section labels (i), (ii)) --- -axs[0].text(0.25, 0.05, r"(i)", fontsize=14, fontweight="bold", va="center", ha="right") - -if preliminary_watermark: - plt.figtext( - 0.5, - 0.5, - "PRELIMINARY", - fontsize=50, - color="gray", - ha="center", - va="center", - alpha=0.3, - rotation=330, - ) - -plt.gca().invert_yaxis() - -plt.tight_layout() - -plt.show() - -# %% -# 4. Make a contour plots -display(Markdown(r"### Here comes $S_8$ and $\Omega_m$")) -colours = ["royalblue", "orange", "violet"] - -filled = [True, True, False, False, False, False, False, False, False, False, False] - -line_args = [dict(color=col, ls="solid") for col in colours] - -g = plots.get_single_plotter(width_inch=30) -g.settings.axes_fontsize = 60 -g.settings.axes_labelsize = 60 -g.settings.alpha_filled_add = 0.7 -g.settings.legend_fontsize = 45 -g.settings.figure_legend_ncol = 3 -g.settings.legend_frame = False - -g.plot_2d( - chains[:-4], - "OMEGA_M", - "S_8", - filled=filled, - line_args=line_args, - contour_colors=colours, -) - -g.add_legend( - legend_labels[:-4], - legend_loc="upper center", - bbox_to_anchor=(0.5, 1.20), # moves legend above the axes -) - -plt.show() -# %% diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_unblinding/utils.py b/cosmo_inference/notebooks/2D_cosmic_shear_unblinding/utils.py deleted file mode 100644 index 8c2fb9e5..00000000 --- a/cosmo_inference/notebooks/2D_cosmic_shear_unblinding/utils.py +++ /dev/null @@ -1,442 +0,0 @@ -""" -Useful scripts to perform the plots for the unblinding party. -""" - -import configparser -import os -import sys - -# Append any useful folder in the path -sys.path.append("/home/guerrini/sp_validation/cosmo_inference/scripts/") - -import matplotlib.pyplot as plt -import matplotlib.scale as mscale -import numpy as np -from astropy.io import fits - -from sp_validation.rho_tau import SquareRootScale - -mscale.register_scale(SquareRootScale) - - -def read_config(path_ini_files, root, thisfile=None): - config = configparser.ConfigParser() - config.optionxform = str - if thisfile is not None: - read_path = thisfile - else: - read_path = os.path.join(path_ini_files, f"{root}.ini") - config.read(read_path) - return config - - -def update_properties_w_roots( - properties, root, path_ini_files, path_to_this_ini=None, with_configuration=False -): - config = read_config(path_ini_files, root, thisfile=path_to_this_ini) - - try: - lower_bound_cell_ee, upper_bound_cell_ee = map( - float, config["2pt_like"]["angle_range_CELL_EE_1_1"].split() - ) - properties[root].update( - { - "lower_bound_cell_ee": lower_bound_cell_ee, - "upper_bound_cell_ee": upper_bound_cell_ee, - } - ) - except KeyError: - properties[root] = {"lower_bound_cell_ee": 0.0, "upper_bound_cell_ee": 2048} - - if with_configuration: - # Also save the scale cuts in theta for xi - add_xi_sys = config["2pt_like"]["add_xi_sys"] - add_xi_sys = add_xi_sys == "T" - lower_bound_xi_plus, upper_bound_xi_plus = map( - float, config["2pt_like"]["angle_range_XI_PLUS_1_1"].split() - ) - lower_bound_xi_minus, upper_bound_xi_minus = map( - float, config["2pt_like"]["angle_range_XI_MINUS_1_1"].split() - ) - - properties[root].update( - { - "add_xi_sys": add_xi_sys, - "lower_bound_xi_plus": lower_bound_xi_plus, - "upper_bound_xi_plus": upper_bound_xi_plus, - "lower_bound_xi_minus": lower_bound_xi_minus, - "upper_bound_xi_minus": upper_bound_xi_minus, - } - ) - return properties - - -def plot_best_fit( - data_points, - root_to_plot, - output_folder, - line_args, - savefile, - ell_min=10.0, - ell_max=2048.0, - multiply_ell=True, - loc_legend="best", - bbox_to_anchor=None, - label_data="Fiducial data", - labels=None, - properties=None, - paths_to_bestfit=None, -): - data = fits.open( - f"/home/guerrini/sp_validation/cosmo_inference/data/{data_points}/cosmosis_{data_points}.fits" - ) - cell_ee = data["CELL_EE"].data - cov_mat = data["COVMAT"].data - - if labels is None: - labels = root_to_plot - - fig, ax = plt.subplots(1, 1, figsize=(8, 5)) - - ell, cell = cell_ee["ANG"], cell_ee["VALUE"] - ax.errorbar( - ell, - ell * cell, - yerr=ell * np.sqrt(np.diag(cov_mat)), - fmt="o", - label=label_data, - color="black", - capsize=2, - ) - - for idx, (label, root) in enumerate(zip(labels, root_to_plot)): - # Read the results - if paths_to_bestfit is None: - ell = np.loadtxt( - output_folder - + "{}/best_fit/shear_cl/ell.txt".format( - root, - ) - ) - shear_cl = np.loadtxt( - output_folder - + "{}/best_fit/shear_cl/bin_1_1.txt".format( - root, - ) - ) - else: - ell = np.loadtxt(paths_to_bestfit[idx] + "best_fit/shear_cl/ell.txt") - shear_cl = np.loadtxt( - paths_to_bestfit[idx] + "best_fit/shear_cl/bin_1_1.txt" - ) - - mask = (ell > ell_min) & (ell < ell_max) - - ax.plot( - ell[mask], - ell[mask] * shear_cl[mask] if multiply_ell else shear_cl[mask], - label=label, - **line_args[idx], - ) - - # Plot the scale cuts for different k_max - ax.axvline(x=1800, color="black", linestyle="--", alpha=0.5) - ax.axvline(x=2048, color="black", linestyle="--", alpha=1.0) - ax.axvline(x=500, color="black", linestyle="--", alpha=0.3) - - ymin = ax.get_ylim()[0] - ymax = ax.get_ylim()[1] - # Shadowing cut scaled - ax.fill_betweenx( - y=[ymin, ymax], - x1=0, - x2=300, - color="gray", - alpha=0.2, - label=r"$B$-mode informed scale cut", - ) - ax.fill_betweenx(y=[ymin, ymax], x1=1600, x2=2048, color="gray", alpha=0.2) - - ax.set_ylim(ymin, ymax) - - # Add labels directly under the tick - ax.text( - 1740, - 0.90, - r"$k_\mathrm{max} = 3 h$ Mpc$^{-1}$", - transform=ax.get_xaxis_transform(), - ha="center", - va="top", - fontsize=14, - rotation=90, - ) - - ax.text( - 1978, - 0.90, - r"$k_\mathrm{max} = 5 h$ Mpc$^{-1}$", - transform=ax.get_xaxis_transform(), - ha="center", - va="top", - fontsize=14, - rotation=90, - ) - - ax.text( - 470, - 0.90, - r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", - transform=ax.get_xaxis_transform(), - ha="center", - va="top", - fontsize=14, - rotation=90, - ) - - ell, cell = cell_ee["ANG"], cell_ee["VALUE"] - ax.set_ylabel(r"$\ell C_\ell \times 10^{-7}$", fontsize=20) - ax.set_xlabel(r"Multipole $\ell$", fontsize=20) - ax.set_xlim(ell.min() - 10, ell.max() + 100) - ax.set_xscale("squareroot") - ax.set_xticks(np.array([100, 400, 900, 1600])) - ax.minorticks_on() - ax.tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax.xaxis.set_ticks(minor_ticks, minor=True) - ax.tick_params(axis="both", which="major", labelsize=14) - ax.tick_params(axis="both", which="minor", labelsize=10) - ax.yaxis.get_offset_text().set_visible(False) - - plt.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor, fontsize=11) - - if savefile is not None: - plt.savefig(savefile, bbox_inches="tight") - - plt.show() - - -def plot_best_fit_config( - data, - root_to_plot, - output_folder, - line_args, - savefile, - theta_min=1.0, - theta_max=250.0, - multiply_theta=True, - loc_legend="best", - bbox_to_anchor_xip=None, - bbox_to_anchor_xim=None, - label_data="Fiducial data", - labels=None, - properties=None, - paths_to_bestfit=None, -): - - data = fits.open(data) - - xi_p_data = data["XI_PLUS"].data - xi_m_data = data["XI_MINUS"].data - cov_mat = data["COVMAT"].data - - # Plot hyperparameter - loc_legend = "lower center" - - fig, [ax, ax2] = plt.subplots(2, 1, figsize=(8, 9)) - - theta, xi_p, xi_m = xi_p_data["ANG"], xi_p_data["VALUE"], xi_m_data["VALUE"] - ax.errorbar( - theta, - theta * xi_p, - yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])), - fmt="o", - label=r"UNIONS $\xi_+$ data", - color="black", - capsize=2, - ) - ax2.errorbar( - theta, - theta * xi_m, - yerr=theta - * np.sqrt( - np.diag(cov_mat[len(theta) : 2 * len(theta), len(theta) : 2 * len(theta)]) - ), - fmt="o", - label=r"UNIONS $\xi_-$ data", - color="black", - capsize=2, - ) - - for idx, (label, root) in enumerate(zip(labels, root_to_plot)): - # Read the results - if paths_to_bestfit is None: - theta = ( - ( - np.loadtxt( - output_folder - + "{}/best_fit/shear_xi_plus/theta.txt".format(root) - ) - ) - * 180 - / np.pi - * 60 - ) - xi_plus = np.loadtxt( - output_folder + "{}/best_fit/shear_xi_plus/bin_1_1.txt".format(root) - ) - xi_minus = np.loadtxt( - output_folder + "{}/best_fit/shear_xi_minus/bin_1_1.txt".format(root) - ) - if r"$C_\ell$" not in label: - xi_sys_plus = np.loadtxt( - output_folder + "{}/best_fit/xi_sys/shear_xi_plus.txt".format(root) - ) - xi_sys_minus = np.loadtxt( - output_folder + "{}/best_fit/xi_sys/shear_xi_minus.txt".format(root) - ) - theta_xi_sys = ( - np.loadtxt( - output_folder + "{}/best_fit/xi_sys/theta.txt".format(root) - ) - * 180 - / np.pi - * 60 - ) - xi_plus += np.interp(theta, theta_xi_sys, xi_sys_plus) - xi_minus += np.interp(theta, theta_xi_sys, xi_sys_minus) - else: - theta = ( - (np.loadtxt(paths_to_bestfit[idx] + "best_fit/shear_xi_plus/theta.txt")) - * 180 - / np.pi - * 60 - ) - xi_plus = np.loadtxt( - paths_to_bestfit[idx] + "best_fit/shear_xi_plus/bin_1_1.txt" - ) - xi_minus = np.loadtxt( - paths_to_bestfit[idx] + "best_fit/shear_xi_minus/bin_1_1.txt" - ) - if r"$C_\ell$" not in label: - xi_sys_plus = np.loadtxt( - output_folder + "{}/best_fit/xi_sys/shear_xi_plus.txt".format(root) - ) - xi_sys_minus = np.loadtxt( - output_folder + "{}/best_fit/xi_sys/shear_xi_minus.txt".format(root) - ) - theta_xi_sys = ( - np.loadtxt( - output_folder + "{}/best_fit/xi_sys/theta.txt".format(root) - ) - * 180 - / np.pi - * 60 - ) - xi_plus += np.interp(theta, theta_xi_sys, xi_sys_plus) - xi_minus += np.interp(theta, theta_xi_sys, xi_sys_minus) - - mask = (theta > theta_min) & (theta < theta_max) - theta = theta[mask] - ax.plot( - theta, - theta * xi_plus[mask] if multiply_theta else xi_plus[mask], - label=label, - **line_args[idx], - ) - ax2.plot( - theta, - theta * xi_minus[mask] if multiply_theta else xi_minus[mask], - label=label, - **line_args[idx], - ) - - # XI PLUS PLOT SETTINGS - - # Plot the scale cuts for different k_max - ax.axvline(x=3.2, color="black", linestyle="--", alpha=0.7) - - ymin = ax.get_ylim()[0] - ymax = ax.get_ylim()[1] - # Shadowing cut scaled - ax.fill_betweenx( - y=[ymin, ymax], - x1=0, - x2=12, - color="gray", - alpha=0.2, - label=r"$B$-mode informed scale cut", - ) - ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color="gray", alpha=0.2) - - ax.set_ylim(ymin, ymax) - - # Add labels directly under the tick - ax.text( - 2.9, - 1.23e-4, - r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", - ha="center", - va="top", - fontsize=14, - rotation=90, - ) - - # ax.set_ylabel('$\theta \xi_+$', fontsize=16) - # ax.set_xlabel('$\theta$', fontsize=16) - ax.set_xlim([theta.min() - 0.1, theta.max() + 20]) - ax.set_xscale("log") - ax.set_xticks(np.array([1, 10, 100])) - ax.tick_params(axis="x", which="minor", length=2, width=0.8) - ax.tick_params(axis="both", which="major", labelsize=14) - ax.tick_params(axis="both", which="minor", labelsize=10) - ax.yaxis.get_offset_text().set_fontsize(14) - ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) - ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=12) - - # XI_MINUS PLOT SETTINGS - - # Plot the scale cuts for different k_max - ax2.axvline(x=24, color="black", linestyle="--", alpha=0.7) - - ymin = ax2.get_ylim()[0] - ymax = ax2.get_ylim()[1] - # Shadowing cut scaled - ax2.fill_betweenx( - y=[ymin, ymax], - x1=0, - x2=12, - color="gray", - alpha=0.2, - label=r"$B$-mode informed scale cut", - ) - ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color="gray", alpha=0.2) - - ax2.set_ylim(ymin, ymax) - - # Add labels directly under the tick - ax2.text( - 21.8, - 1.15e-4, - r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", - ha="center", - va="top", - fontsize=14, - rotation=90, - ) - - ax2.set_ylabel(r"$\theta \xi_-$", fontsize=16) - ax2.set_xlabel(r"$\theta$", fontsize=16) - ax2.set_xlim([theta.min() - 0.1, theta.max() + 20]) - ax2.set_xscale("log") - ax2.set_xticks(np.array([1, 10, 100])) - ax2.tick_params(axis="x", which="minor", length=2, width=0.8) - ax2.tick_params(axis="both", which="major", labelsize=14) - ax2.tick_params(axis="both", which="minor", labelsize=10) - ax2.yaxis.get_offset_text().set_fontsize(14) - ax2.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) - ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=12) - - if savefile is not None: - plt.savefig(savefile, bbox_inches="tight") - - plt.show() diff --git a/cosmo_inference/notebooks/cfis_analysis.ipynb b/cosmo_inference/notebooks/cfis_analysis.ipynb deleted file mode 100644 index ee93f4ec..00000000 --- a/cosmo_inference/notebooks/cfis_analysis.ipynb +++ /dev/null @@ -1,1065 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "0", - "metadata": {}, - "source": [ - "# Analysis of a CFIS shear catalogue\n", - "First steps. Analysing both ShapePipe and Lensfit catalogues, for all blinds A,B and C" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import ipywidgets as widgets\n", - "import matplotlib.pylab as plt\n", - "import numpy as np\n", - "import pandas as pd\n", - "import pyccl as ccl\n", - "import treecorr\n", - "from astropy.io import fits\n", - "from ipywidgets import interact\n", - "\n", - "%matplotlib inline\n", - "plt.rcParams.update({\"font.size\": 20, \"figure.figsize\": [12, 10]})\n", - "plt.rc(\"mathtext\", fontset=\"stix\")\n", - "plt.rc(\"font\", family=\"sans-serif\")\n", - "\n", - "# SPECIFY DIRECTORIES AND CATALOGUE PATHS\n", - "work_dir = \"/home/mkilbing/astro/data/UNIONS/v1.x/ShapePipe\"\n", - "\n", - "cat_dict = {\n", - " 1: {\n", - " \"dir\": work_dir + \"/Lensfit/lensfit_goldshape_2022v1.fits\",\n", - " \"label\": \"LF_full\",\n", - " \"e1_bias\": 0,\n", - " \"e2_bias\": 0,\n", - " \"ls\": \"solid\",\n", - " \"colour\": \"g\",\n", - " },\n", - " 2: {\n", - " \"dir\": work_dir + \"/ShapePipe/unions_shapepipe_2022_v1.0.fits\",\n", - " \"label\": \"SP_full\",\n", - " \"e1_bias\": 0,\n", - " \"e2_bias\": 0,\n", - " \"ls\": \"solid\",\n", - " \"colour\": \"b\",\n", - " },\n", - " 3: {\n", - " \"dir\": work_dir + \"/matched_LF_SP/masked_matched_lensfit_goldshape_2022v1.fits\",\n", - " \"label\": \"LF_matched_SP\",\n", - " \"e1_bias\": 3.939e-4,\n", - " \"e2_bias\": 6.482e-5,\n", - " \"ls\": \"dotted\",\n", - " \"colour\": \"g\",\n", - " },\n", - " 4: {\n", - " \"dir\": work_dir\n", - " + \"/matched_LF_SP/masked_matched_unions_shapepipe_extended_2022_v1.0.fits\",\n", - " \"label\": \"SP_matched_LF\",\n", - " \"e1_bias\": -5.6726e-5,\n", - " \"e2_bias\": 8.218e-4,\n", - " \"ls\": \"dotted\",\n", - " \"colour\": \"b\",\n", - " },\n", - " 5: {\n", - " \"dir\": work_dir + \"/matched_LF_SP/matched_footprint_shapepipe.fits\",\n", - " \"label\": \"SP Match LF Footprint\",\n", - " \"e1_bias\": 0,\n", - " \"e2_bias\": 0,\n", - " \"ls\": \"dashed\",\n", - " \"colour\": \"b\",\n", - " },\n", - " 6: {\n", - " \"dir\": work_dir + \"/cfis-shapepipe.parquet\",\n", - " \"label\": \"SP Match MegaPipe\",\n", - " \"e1_bias\": 0,\n", - " \"e2_bias\": 0,\n", - " \"ls\": \"dashdot\",\n", - " \"colour\": \"b\",\n", - " },\n", - " 7: {\n", - " \"dir\": work_dir + \"/ShapePipe/shapepipe_1500_goldshape_v1.fits\",\n", - " \"label\": \"SP_1500\",\n", - " \"e1_bias\": 7.156105098141909e-06,\n", - " \"e2_bias\": -6.00816359759969e-06,\n", - " \"ls\": \"dotted\",\n", - " \"colour\": \"b\",\n", - " },\n", - " 8: {\n", - " \"dir\": work_dir + \"/ShapePipe/unions_shapepipe_2022_v1.0.4.fits\",\n", - " \"label\": \"SP_cut_Fabian\",\n", - " \"e1_bias\": 0.0,\n", - " \"e2_bias\": 0.0,\n", - " \"ls\": \"dashdot\",\n", - " \"colour\": \"pink\",\n", - " },\n", - " 9: {\n", - " \"dir\": work_dir + \"/ShapePipe/unions_shapepipe_psf_2022_v1.0.2.fits\",\n", - " \"label\": \"SP_PSF\",\n", - " \"e1_bias\": 0.0,\n", - " \"e2_bias\": 0.0,\n", - " \"ls\": \"dashdot\",\n", - " \"colour\": \"b\",\n", - " },\n", - " 10: {\n", - " \"dir\": work_dir + \"/unions_shapepipe_2022_v1.3.fits\",\n", - " \"label\": \"SP_v1.3\",\n", - " \"e1_bias\": 0.0,\n", - " \"e2_bias\": 0.0,\n", - " \"ls\": \"dashdot\",\n", - " \"colour\": \"r\",\n", - " },\n", - " 11: {\n", - " \"dir\": work_dir + \"/ShapePipe/unions_shapepipe_star_2022_v1.3.fits\",\n", - " \"label\": \"SP_v1.3\",\n", - " \"e1_bias\": 0.0,\n", - " \"e2_bias\": 0.0,\n", - " \"ls\": \"dashdot\",\n", - " \"colour\": \"b\",\n", - " },\n", - " 12: {\n", - " \"dir\": work_dir + \"/unions_shapepipe_2024_v1.4.1.fits\",\n", - " \"label\": \"SP_v1.4.1\",\n", - " \"e1_bias\": 0.0,\n", - " \"e2_bias\": 0.0,\n", - " \"ls\": \"dashdot\",\n", - " \"colour\": \"b\",\n", - " },\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ - "# CATALOGUE OPTIONS:\n", - "# 1: LensFit Full\n", - "# 2: ShapePipe Full\n", - "# 3: LF Match SP\n", - "# 4: SP Match LF\n", - "# 5: SP Matched in LF footprint\n", - "# 6: SP Match MegaPipe\n", - "# 7: SP 1500deg2 (Axel's)\n", - "# 8: SP cut on large gals\n", - "# 12: SP psfex v1.4.1\n", - "\n", - "cat_options = [10, 12]\n", - "\n", - "dfs = []\n", - "\n", - "for cat_option in cat_options:\n", - " if cat_option == 6:\n", - " df = pd.read_parquet(cat_dict[cat_option][\"dir\"], engine=\"pyarrow\")\n", - " df = df.replace([np.inf, -np.inf], np.nan).dropna(axis=0)\n", - " else:\n", - " with fits.open(cat_dict[cat_option][\"dir\"]) as data:\n", - " df = pd.DataFrame(data[1].data)\n", - " if cat_option == 7:\n", - " df = df.rename(columns={\"g1\": \"e1\", \"g2\": \"e2\"})\n", - " if cat_option == 8 or cat_option == 10:\n", - " df = df.rename(columns={\"RA\": \"ra\", \"Dec\": \"dec\"})\n", - " if cat_option == 12:\n", - " df = df.rename(\n", - " columns={\"RA\": \"ra\", \"Dec\": \"dec\", \"e1\": \"e1_prev\", \"e2\": \"e2_prev\"}\n", - " )\n", - " df = df.rename(columns={\"e1_noleakage\": \"e1\", \"e2_noleakage\": \"e2\"})\n", - " dfs.append(df)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "3", - "metadata": {}, - "source": [ - "## Catalogue Analysis" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "for cat in cat_options:\n", - " plt.plot(df[\"ra\"], df[\"dec\"], \".\", label=cat_dict[cat][\"label\"])\n", - "plt.xlabel(\"RA [deg]\")\n", - "plt.ylabel(\"DEC [deg]\")\n", - "plt.legend(loc=\"upper right\")\n", - "# plt.savefig('plots/3500deg^2_plot.pdf',dpi=100)\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5", - "metadata": {}, - "outputs": [], - "source": [ - "# Ellipticity histograms\n", - "plt.rcParams.update({\"font.size\": 20, \"figure.figsize\": [22, 7]})\n", - "\n", - "fig, axs = plt.subplots(1, 2)\n", - "nbins = 200\n", - "\n", - "for idx, cat in enumerate(cat_options):\n", - " (n, bins, _) = axs[0].hist(\n", - " dfs[idx][\"e1\"],\n", - " bins=nbins,\n", - " density=True,\n", - " histtype=\"step\",\n", - " weights=dfs[idx][\"w\"],\n", - " label=\"e1 %s\" % cat_dict[cat][\"label\"],\n", - " )\n", - "axs[0].set_xlabel(r\"$e_1$\")\n", - "axs[0].legend()\n", - "axs[0].set_xlim([-1.5, 1.5])\n", - "\n", - "# axs[0].set_ylim([0,2e4])\n", - "\n", - "for idx, cat in enumerate(cat_options):\n", - " (n, bins, _) = axs[1].hist(\n", - " dfs[idx][\"e2\"],\n", - " bins=nbins,\n", - " density=True,\n", - " histtype=\"step\",\n", - " weights=dfs[idx][\"w\"],\n", - " label=\"e2 {}\".format(cat_dict[cat][\"label\"]),\n", - " )\n", - " print(\n", - " \"e1 sigma {}: {}\".format(\n", - " cat_dict[cat][\"label\"], np.std(dfs[idx][\"e1_noleakage\"])\n", - " )\n", - " )\n", - " print(\n", - " \"e2 sigma {}: {}\".format(\n", - " cat_dict[cat][\"label\"], np.std(dfs[idx][\"e2_noleakage\"])\n", - " )\n", - " )\n", - " print(\n", - " \"e1 bias {}: {}\".format(\n", - " cat_dict[cat][\"label\"],\n", - " np.average(\n", - " np.array(dfs[idx][\"e1_noleakage\"]), weights=np.array(dfs[idx][\"w\"])\n", - " ),\n", - " )\n", - " )\n", - " print(\n", - " \"e2 bias {}: {}\".format(\n", - " cat_dict[cat][\"label\"],\n", - " np.average(\n", - " np.array(dfs[idx][\"e2_noleakage\"]), weights=np.array(dfs[idx][\"w\"])\n", - " ),\n", - " )\n", - " )\n", - "axs[1].set_xlabel(r\"$e_2$\")\n", - "axs[1].legend()\n", - "axs[1].set_xlim([-1.5, 1.5])\n", - "# axs[1].set_ylim([0,2e4])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "# Mag histograms\n", - "\n", - "plt.rcParams.update({\"font.size\": 20, \"figure.figsize\": [15, 10]})\n", - "\n", - "for idx, cat in enumerate(cat_options):\n", - " (n, bins, _) = plt.hist(\n", - " dfs[idx][\"mag\"],\n", - " bins=200,\n", - " density=False,\n", - " histtype=\"step\",\n", - " weights=dfs[idx][\"w\"],\n", - " label=\"Mag %s\" % cat_dict[cat][\"label\"],\n", - " )\n", - "\n", - "plt.xlim([19, 26])\n", - "plt.xlabel(\"Mag\")\n", - "plt.legend(loc=\"upper left\")" - ] - }, - { - "cell_type": "markdown", - "id": "7", - "metadata": { - "tags": [] - }, - "source": [ - "## Plot n(z)'s from file\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8", - "metadata": {}, - "outputs": [], - "source": [ - "# nz_lf = fits.open(work_dir + '/nz/blind_nz_cfis_lensfit_goldshape_2022v1.fits')[1].data\n", - "nz = fits.open(work_dir + \"/nz/blind_nz_cfis_shapepipe_2022v1.fits\")[1].data\n", - "\n", - "# nz_lf_matched = fits.open(work_dir + '/nz/nz_masked_matched_lensfit_goldshape_2022v1.fits')[1].data\n", - "# nz_sp_matched = fits.open(work_dir + '/nz/nz_masked_matched_unions_shapepipe_extended_2022_v1.0.fits')[1].data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# FULL CATALOGUE NZ'S\n", - "from matplotlib.ticker import StrMethodFormatter\n", - "\n", - "blinds = [\"A\", \"B\", \"C\"]\n", - "\n", - "# for blind in blinds:\n", - "# z1 = nz_lf['Z_%s' %blind]\n", - "\n", - "# (n,bins,_)= plt.hist(z1, bins=200, range=(0,5.0), density=True, histtype='step', weights=None,label='LensFit Blind %s' %blind)\n", - "# # n_lf.append(list(n))\n", - "# # bins_lf.append(list(bins))\n", - "\n", - "# plt.xlabel('Redshifts')\n", - "# plt.ylabel('n(z)')\n", - "# print(\"zmin = \",min(z1))\n", - "# print(\"zmax = \",max(z1))\n", - "# plt.legend(fontsize=20)\n", - "# # plt.savefig('plots/Lensfit_nz_all_blinds.pdf' )\n", - "# plt.show()\n", - "#####################################################################################################\n", - "for blind in blinds:\n", - " z = nz[\"Z_%s\" % blind]\n", - " bins = np.linspace(0, 5, 100)\n", - "\n", - " y, edges = np.histogram(z, bins, density=True, weights=nz[\"som_w\"])\n", - " centers = 0.5 * (edges[1:] + edges[:-1])\n", - " plt.plot(centers, y, \"-o\", markersize=4, label=\"Blind %s\" % blind, alpha=0.7)\n", - "\n", - " # (n,bins,_)= plt.hist(z2, bins=50, range=(0,5.0), density=True, histtype='step',weights=nz['som_w'],label='Blind %s' %blind,alpha=0.5)\n", - " # n_sp.append(list(n))\n", - " # bins_sp.append(list(bins))\n", - "\n", - " plt.xlabel(r\"$z$\")\n", - " plt.ylabel(r\"$n(z)$\")\n", - " plt.ylim([0, 1.7])\n", - " plt.xlim([0, 5])\n", - " plt.grid(True)\n", - " plt.gca().xaxis.set_major_formatter(StrMethodFormatter(\"{x:,.1f}\"))\n", - " # print(\"zmin = \",min(z))\n", - " # print(\"zmax = \",max(z))\n", - " plt.legend(fontsize=20)\n", - " plt.savefig(\"../plots/unions_nz.pdf\", bbox_inches=\"tight\")\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "id": "10", - "metadata": { - "tags": [] - }, - "source": [ - "## Compute shear-shear correlation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11", - "metadata": {}, - "outputs": [], - "source": [ - "# Create TreeCorr catalogue\n", - "treecorr.set_omp_threads(8)\n", - "\n", - "sep_units = \"arcmin\"\n", - "theta_min = 1\n", - "theta_max = 200\n", - "\n", - "TreeCorrConfig = {\n", - " \"ra_units\": \"degrees\",\n", - " \"dec_units\": \"degrees\",\n", - " \"max_sep\": str(theta_max),\n", - " \"min_sep\": str(theta_min),\n", - " \"sep_units\": sep_units,\n", - " \"nbins\": 20,\n", - " \"var_method\": \"jackknife\",\n", - "}\n", - "\n", - "cat_ggs = []\n", - "for idx, cat in enumerate(cat_options):\n", - " cat_gal = treecorr.Catalog(\n", - " ra=dfs[idx][\"ra\"],\n", - " dec=dfs[idx][\"dec\"],\n", - " g1=dfs[idx][\"e1\"] - cat_dict[cat][\"e1_bias\"],\n", - " g2=dfs[idx][\"e2\"] - cat_dict[cat][\"e2_bias\"],\n", - " w=dfs[idx][\"w\"],\n", - " ra_units=\"degrees\",\n", - " dec_units=\"degrees\",\n", - " npatch=50,\n", - " )\n", - " gg = treecorr.GGCorrelation(TreeCorrConfig)\n", - " gg.process(cat_gal)\n", - " cat_ggs.append(gg)\n", - " print(\"done for cat %s\" % cat)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "12", - "metadata": {}, - "outputs": [], - "source": [ - "plt.rcParams.update({\"font.size\": 25, \"figure.figsize\": [10, 7]})\n", - "\n", - "ax1 = plt.subplot(111)\n", - "for idx, cat in enumerate(cat_options):\n", - " ax1.plot(\n", - " cat_ggs[idx].meanr,\n", - " cat_ggs[idx].npairs,\n", - " label=r\"$n_{pairs}$ %s\" % (cat_dict[cat][\"label\"]),\n", - " ls=cat_dict[cat][\"ls\"],\n", - " color=cat_dict[cat][\"colour\"],\n", - " )\n", - "ax1.set_xlabel(rf\"$\\theta$ [{sep_units}]\")\n", - "ax1.set_ylabel(r\"$n_{pairs}$\")\n", - "plt.show()\n", - "\n", - "ax2 = plt.subplot(111)\n", - "for idx, cat in enumerate(cat_options):\n", - " ax2.errorbar(\n", - " cat_ggs[idx].meanr,\n", - " cat_ggs[idx].xip,\n", - " yerr=np.sqrt(cat_ggs[idx].varxip),\n", - " label=r\"$\\xi_+$ %s\" % (cat_dict[cat][\"label\"]),\n", - " ls=cat_dict[cat][\"ls\"],\n", - " color=cat_dict[cat][\"colour\"],\n", - " )\n", - " ax2.axvspan(0, 10, color=\"gray\", alpha=0.3)\n", - " # ax2.axvspan(100,200,color='gray', alpha=0.3)\n", - "\n", - "ax2.text(\n", - " 0.85,\n", - " 0.88,\n", - " \"1,1\",\n", - " transform=ax2.transAxes,\n", - " bbox=dict(facecolor=\"white\", edgecolor=\"black\", boxstyle=\"round\", pad=0.5),\n", - ")\n", - "ax2.set_xscale(\"log\")\n", - "ax2.set_yscale(\"log\")\n", - "ax2.set_xlabel(rf\"$\\theta$ [{sep_units}]\")\n", - "ax2.set_xlim([0, 200])\n", - "_ = ax2.set_ylabel(r\"$\\xi_+(\\theta)$\")\n", - "ax2.legend(loc=\"lower left\")\n", - "# plt.savefig('../plots/xi_plus_%s.pdf' %cat_dict[cat]['label'],bbox_inches='tight')\n", - "plt.show()\n", - "\n", - "ax3 = plt.subplot(111)\n", - "for idx, cat in enumerate(cat_options):\n", - " ax3.errorbar(\n", - " cat_ggs[idx].meanr,\n", - " cat_ggs[idx].xim,\n", - " yerr=np.sqrt(cat_ggs[idx].varxim),\n", - " label=r\"$\\xi_-$ %s\" % (cat_dict[cat][\"label\"]),\n", - " ls=\"dotted\",\n", - " color=cat_dict[cat][\"colour\"],\n", - " )\n", - " ax3.axvspan(0, 20, color=\"gray\", alpha=0.3)\n", - " # ax3.axvspan(100,200,color='gray', alpha=0.3)\n", - "\n", - "ax3.text(\n", - " 0.85,\n", - " 0.88,\n", - " \"1,1\",\n", - " transform=ax3.transAxes,\n", - " bbox=dict(facecolor=\"white\", edgecolor=\"black\", boxstyle=\"round\", pad=0.5),\n", - ")\n", - "ax3.set_xscale(\"log\")\n", - "ax3.set_yscale(\"log\")\n", - "ax3.set_xlabel(rf\"$\\theta$ [{sep_units}]\")\n", - "ax3.set_xlim([0, 200])\n", - "ax3.legend(loc=\"lower left\")\n", - "_ = ax3.set_ylabel(r\"$\\xi_-(\\theta)$\")\n", - "# plt.savefig('../plots/xi_minus_%s.pdf' %cat_dict[cat]['label'],bbox_inches='tight')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "id": "13", - "metadata": { - "tags": [] - }, - "source": [ - "## Comparison with theory PyCCL" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "14", - "metadata": {}, - "outputs": [], - "source": [ - "nz = np.loadtxt(\n", - " \"/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/v1.0/nz/dndz_SP_v1.0_A.txt\",\n", - " usecols=1,\n", - ")\n", - "bins = np.loadtxt(\n", - " \"/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/v1.0/nz/dndz_SP_v1.0_A.txt\",\n", - " usecols=0,\n", - ")\n", - "\n", - "\n", - "def theory_cls(Omega_c, Omega_b, h, n_s, sigma_8):\n", - " # Set cosmology\n", - " cosmo = ccl.Cosmology(Omega_c, Omega_b, h, n_s, sigma_8)\n", - "\n", - " ell = np.arange(2, 2000)\n", - " theta_deg = np.logspace(\n", - " np.log10(theta_min / 60), np.log10(theta_max / 60), num=20\n", - " ) # Theta is in degrees\n", - " # CALCULATION OF THEORY XI_PM\n", - " xi_plus_lf = []\n", - " xi_minus_lf = []\n", - "\n", - " for i in range(len(nz)):\n", - " bias_ia = 0 * np.ones_like(bins[i][:-1])\n", - " lens_ia = ccl.WeakLensingTracer(\n", - " cosmo,\n", - " dndz=(np.array(bins[i][:-1]), np.array(nz[i])),\n", - " ia_bias=(np.array(bins[i][:-1]), bias_ia),\n", - " )\n", - " cl = ccl.angular_cl(cosmo, lens_ia, lens_ia, ell)\n", - "\n", - " xi_plus_lf.append(\n", - " list(\n", - " ccl.correlation(cosmo, ell, cl, theta_deg, type=\"GG+\", method=\"FFTLog\")\n", - " )\n", - " )\n", - " xi_minus_lf.append(\n", - " list(\n", - " ccl.correlation(cosmo, ell, cl, theta_deg, type=\"GG-\", method=\"FFTLog\")\n", - " )\n", - " )\n", - "\n", - " style = [\":\", \"--\", \"-.\"]\n", - " plt.errorbar(\n", - " gg.meanr,\n", - " gg.xip,\n", - " yerr=np.sqrt(gg.varxip),\n", - " ls=\"\",\n", - " label=r\"$\\xi_+$ TreeCorr (LF)\",\n", - " capsize=5,\n", - " marker=\"o\",\n", - " color=\"b\",\n", - " )\n", - " plt.errorbar(\n", - " gg.meanr,\n", - " gg.xim,\n", - " yerr=np.sqrt(gg.varxim),\n", - " ls=\"\",\n", - " label=r\"$\\xi_-$ TreeCorr (LF)\",\n", - " capsize=5,\n", - " marker=\"o\",\n", - " color=\"g\",\n", - " )\n", - "\n", - " for i in range(len(blinds)):\n", - " plt.plot(\n", - " theta_deg * 60,\n", - " xi_plus_lf[i],\n", - " color=\"b\",\n", - " ls=style[i],\n", - " label=r\"$\\xi_+$ PyCCL (LF) blind %s\" % blinds[i],\n", - " )\n", - " plt.plot(\n", - " theta_deg * 60,\n", - " xi_minus_lf[i],\n", - " color=\"g\",\n", - " ls=style[i],\n", - " label=r\"$\\xi_-$ PyCCL (LF) blind %s\" % blinds[i],\n", - " )\n", - "\n", - " plt.xscale(\"log\")\n", - " # plt.yscale('log')\n", - " plt.legend(fontsize=20)\n", - " plt.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n", - " plt.xlim([1, 200])\n", - " plt.ylim([0, 10e-5])\n", - " plt.ylabel(r\"$\\xi_\\pm(\\theta)$\")\n", - " plt.xlabel(r\"$\\theta$ [arcmin]\")\n", - " # plt.savefig('plots/pyccl_comparison_lensfit.pdf')\n", - "\n", - "\n", - "interact(\n", - " theory_cls,\n", - " Omega_c=widgets.FloatSlider(\n", - " value=0.26, min=0.01, max=0.5, step=0.01, description=r\"$\\Omega_c$\"\n", - " ),\n", - " Omega_b=widgets.FloatSlider(\n", - " value=0.04, min=0.001, max=0.07, step=0.001, description=r\"$\\Omega_b$\"\n", - " ),\n", - " h=widgets.FloatSlider(value=0.7, min=0.3, max=0.9, step=0.01, description=r\"$h$\"),\n", - " n_s=widgets.FloatSlider(\n", - " value=0.96, min=0.6, max=1.1, step=0.01, description=r\"$n_s$\"\n", - " ),\n", - " sigma_8=widgets.FloatSlider(\n", - " value=0.8, min=0.3, max=1.2, step=0.01, description=r\"$\\sigma_8$\"\n", - " ),\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "15", - "metadata": { - "tags": [] - }, - "source": [ - "## Plot varxipm's\n", - "Error bars are computed by treecorr, either through the 'shot' or 'jackknife' method." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "16", - "metadata": {}, - "outputs": [], - "source": [ - "for idx, cat in enumerate(cat_options):\n", - " blind = \"A\"\n", - " label = \"SP_v1.4\"\n", - "\n", - " cc = \"/n23data1/n06data/lgoh/scratch/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/data/{}/covs/cov_{}\".format(\n", - " label + \"_{}\".format(blind), label\n", - " )\n", - "\n", - " cc_var = np.diag(np.loadtxt(cc + \".txt\"))\n", - " cc_varxip = cc_var[:20]\n", - " cc_varxim = cc_var[20:]\n", - "\n", - " cc_var = np.diag(np.loadtxt(cc + \"_g.txt\"))\n", - " cc_varxip_g = cc_var[:20]\n", - " cc_varxim_g = cc_var[20:]\n", - "\n", - " plt.loglog(\n", - " cat_ggs[idx].meanr,\n", - " cat_ggs[idx].varxip,\n", - " \"-k\",\n", - " label=r\"$\\sigma(\\xi_+)$ TreeCorr jackknife %s\" % cat_dict[cat][\"label\"],\n", - " )\n", - " plt.loglog(\n", - " cat_ggs[idx].meanr,\n", - " cc_varxip,\n", - " ls=\"--\",\n", - " c=\"%s\" % cat_dict[cat][\"colour\"],\n", - " label=r\"$\\sigma(\\xi_+)$ CosmoCov %s\" % cat_dict[cat][\"label\"],\n", - " )\n", - " plt.loglog(\n", - " cat_ggs[idx].meanr,\n", - " cc_varxip_g,\n", - " ls=\":\",\n", - " c=\"%s\" % cat_dict[cat][\"colour\"],\n", - " label=r\"$\\sigma(\\xi_+)$ CosmoCov Gaussian %s\" % cat_dict[cat][\"label\"],\n", - " )\n", - " plt.grid()\n", - " plt.xlim([cat_ggs[idx].meanr[0], cat_ggs[idx].meanr[-1]])\n", - " plt.legend(fontsize=15)\n", - " plt.xlabel(rf\"$\\theta$ [{sep_units}]\")\n", - " plt.ylabel(r\"$\\sigma(\\xi_+)$\")\n", - " plt.show()\n", - " # plt.savefig()\n", - "\n", - " plt.loglog(\n", - " cat_ggs[idx].meanr,\n", - " cat_ggs[idx].varxim,\n", - " \"-k\",\n", - " label=r\"$\\sigma(\\xi_-)$ TreeCorr jackknife %s\" % cat_dict[cat][\"label\"],\n", - " )\n", - " plt.loglog(\n", - " cat_ggs[idx].meanr,\n", - " cc_varxim,\n", - " ls=\"--\",\n", - " c=\"%s\" % cat_dict[cat][\"colour\"],\n", - " label=r\"$\\sigma(\\xi_-)$ CosmoCov (SP) %s\" % cat_dict[cat][\"label\"],\n", - " )\n", - " plt.loglog(\n", - " cat_ggs[idx].meanr,\n", - " cc_varxim_g,\n", - " ls=\":\",\n", - " c=\"%s\" % cat_dict[cat][\"colour\"],\n", - " label=r\"$\\sigma(\\xi_-)$ CosmoCov (SP) Gaussian %s\" % cat_dict[cat][\"label\"],\n", - " )\n", - " plt.grid()\n", - " plt.xlim([cat_ggs[idx].meanr[0], cat_ggs[idx].meanr[-1]])\n", - " plt.legend(fontsize=15)\n", - " plt.xlabel(rf\"$\\theta$ [{sep_units}]\")\n", - " plt.ylabel(r\"$\\sigma(\\xi_-)$\")\n", - " plt.show()\n", - " # plt.savefig()" - ] - }, - { - "cell_type": "markdown", - "id": "17", - "metadata": { - "jp-MarkdownHeadingCollapsed": true, - "tags": [] - }, - "source": [ - "## Run systematic tests" - ] - }, - { - "cell_type": "markdown", - "id": "18", - "metadata": { - "jp-MarkdownHeadingCollapsed": true, - "tags": [] - }, - "source": [ - "### C_sys" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "19", - "metadata": {}, - "outputs": [], - "source": [ - "# CALCULATE XI_SYS FOR SHAPEPIPE\n", - "\n", - "sep_units = \"arcmin\"\n", - "theta_min = 1\n", - "theta_max = 200\n", - "\n", - "TreeCorrConfig = {\n", - " \"ra_units\": \"degrees\",\n", - " \"dec_units\": \"degrees\",\n", - " \"max_sep\": str(theta_max),\n", - " \"min_sep\": str(theta_min),\n", - " \"sep_units\": sep_units,\n", - " \"nbins\": 20,\n", - " \"var_method\": \"jackknife\",\n", - "}\n", - "\n", - "with fits.open(cat_dict[11][\"dir\"]) as data:\n", - " df_psf = pd.DataFrame(data[1].data)\n", - "\n", - "cat_psf = treecorr.Catalog(\n", - " ra=df_psf[\"RA\"],\n", - " dec=df_psf[\"DEC\"],\n", - " g1=df_psf[\"HSM_G1_PSF\"],\n", - " g2=df_psf[\"HSM_G2_PSF\"],\n", - " ra_units=\"degrees\",\n", - " dec_units=\"degrees\",\n", - " npatch=50,\n", - ")\n", - "\n", - "gg_psf = treecorr.GGCorrelation(TreeCorrConfig)\n", - "gg_psf.process(cat_psf)\n", - "\n", - "ggs_psf_star = []\n", - "for idx, cat in enumerate(cat_options):\n", - " cat_gal = treecorr.Catalog(\n", - " ra=dfs[idx][\"ra\"],\n", - " dec=dfs[idx][\"dec\"],\n", - " g1=dfs[idx][\"e1\"] - cat_dict[cat][\"e1_bias\"],\n", - " g2=dfs[idx][\"e2\"] - cat_dict[cat][\"e2_bias\"],\n", - " w=dfs[idx][\"w\"],\n", - " ra_units=\"degrees\",\n", - " dec_units=\"degrees\",\n", - " npatch=50,\n", - " )\n", - " gg_psf_star = treecorr.GGCorrelation(TreeCorrConfig)\n", - " gg_psf_star.process(cat_gal, cat_psf)\n", - " ggs_psf_star.append(gg_psf_star)\n", - "\n", - " print(\"done for cat %s\" % cat)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "20", - "metadata": {}, - "outputs": [], - "source": [ - "for idx, cat in enumerate(cat_options):\n", - " C_sys_xip = gg_psf.xip\n", - " C_sys_xim = gg_psf.xim\n", - "\n", - " # delta_C_sys_xip = C_sys_xip*np.sqrt((2*np.sqrt(ggs_psf_star[idx].varxip)/ggs_psf_star[idx].xip)**2+(np.sqrt(gg_psf.varxip)/gg_psf.xip)**2)\n", - " # delta_C_sys_xim = C_sys_xim*np.sqrt((2*np.sqrt(ggs_psf_star[idx].varxim)/ggs_psf_star[idx].xim)**2+(np.sqrt(gg_psf.varxim)/gg_psf.xim)**2)\n", - "\n", - " plt.errorbar(\n", - " ggs_psf_star[idx].meanr,\n", - " C_sys_xip,\n", - " yerr=0,\n", - " label=r\"$(\\xi^{sys}_+)$ Catalogue %s\" % cat_dict[cat][\"label\"],\n", - " ls=cat_dict[cat][\"ls\"],\n", - " color=cat_dict[cat][\"colour\"],\n", - " )\n", - " plt.legend()\n", - " plt.xlabel(r\"$\\theta[arcmin]$\")\n", - " plt.ylabel(r\"$\\xi^{sys}_\\pm$\")\n", - " # plt.ylim([-2e-8,2e-8])\n", - " plt.xscale(\"log\")\n", - " plt.ticklabel_format(style=\"sci\", axis=\"y\", scilimits=(0, 0))\n", - " plt.grid(True)\n", - "\n", - " # plt.errorbar(ggs_psf_star[idx].meanr, C_sys_xim, yerr=delta_C_sys_xim, label=r'$(\\xi^{sys}_-)$ Catalogue %s'%cat_dict[cat]['label'],color='g')\n", - " # plt.legend()\n", - " # plt.xlabel(r'$\\theta[arcmin]$')\n", - " # plt.ylabel(r'$\\xi^{sys}_\\pm$')\n", - " # plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))\n", - " # # plt.ylim([-2e-8,2e-8])\n", - " # plt.xscale('log')\n", - " # plt.grid(True)" - ] - }, - { - "cell_type": "markdown", - "id": "21", - "metadata": { - "jp-MarkdownHeadingCollapsed": true, - "tags": [] - }, - "source": [ - "### M_ap" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "22", - "metadata": {}, - "outputs": [], - "source": [ - "for idx, cat in enumerate(cat_options):\n", - " R = cat_ggs[idx].rnom\n", - "\n", - " (Map_lf, mapsq_im_lf, Mx_lf, mxsq_im_lf, varMapsq_lf) = cat_ggs[idx].calculateMapSq(\n", - " R=R, m2_uform=\"Schneider\"\n", - " )\n", - " (Map_sp, mapsq_im_sp, Mx_sp, mxsq_im_sp, varMapsq_sp) = cat_ggs[idx].calculateMapSq(\n", - " R=R, m2_uform=\"Schneider\"\n", - " )\n", - "\n", - " plt.errorbar(\n", - " R,\n", - " Map_lf,\n", - " yerr=np.sqrt(varMapsq_lf),\n", - " label=r\"$$ {}\".format(cat_dict[cat][\"label\"]),\n", - " ls=\":\",\n", - " color=\"b\",\n", - " )\n", - " plt.errorbar(\n", - " R,\n", - " Mx_lf,\n", - " yerr=np.sqrt(varMapsq_lf),\n", - " label=r\"$$ {}\".format(cat_dict[cat][\"label\"]),\n", - " ls=\":\",\n", - " color=\"r\",\n", - " )\n", - " plt.axhline(y=0, xmin=0, xmax=200, color=\"k\")\n", - " plt.xlabel(r\"$\\theta[arcmin]$\")\n", - " plt.ylabel(r\"$$\")\n", - " plt.xscale(\"log\")\n", - " plt.ylim([-2e-5, 1e-5])\n", - " # plt.xlim([1,200])\n", - " plt.grid(True)\n", - " plt.legend()\n", - "\n", - " plt.errorbar(\n", - " R,\n", - " Map_sp,\n", - " yerr=np.sqrt(varMapsq_sp),\n", - " label=r\"$$ {}\".format(cat_dict[cat][\"label\"]),\n", - " ls=\":\",\n", - " color=\"b\",\n", - " )\n", - " plt.errorbar(\n", - " R,\n", - " Mx_sp,\n", - " yerr=np.sqrt(varMapsq_sp),\n", - " label=r\"$$ {}\".format(cat_dict[cat][\"label\"]),\n", - " ls=\":\",\n", - " color=\"r\",\n", - " )\n", - " plt.axhline(y=0, xmin=0, xmax=200, color=\"k\")\n", - " plt.xscale(\"log\")\n", - " plt.xlabel(r\"$\\theta[arcmin]$\")\n", - " plt.ylabel(r\"$$\")\n", - " plt.ticklabel_format(style=\"sci\", axis=\"y\", scilimits=(0, 0))\n", - " plt.ylim([-2e-5, 1e-5])\n", - " plt.grid(True)\n", - " plt.legend()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "23", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "24", - "metadata": {}, - "source": [ - "## Plot Covariance Matrix" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "25", - "metadata": {}, - "outputs": [], - "source": [ - "from numpy import linalg as LA\n", - "\n", - "%matplotlib inline\n", - "\n", - "\n", - "def get_cov(filename):\n", - "\n", - " data = np.loadtxt(filename)\n", - " ndata = int(np.max(data[:, 0])) + 1\n", - "\n", - " print(\"Dimension of cov: %dx%d\" % (ndata, ndata))\n", - "\n", - " # ndata_min = int(np.min(data[:,0]))\n", - " cov_g = np.zeros((ndata, ndata))\n", - " cov_ng = np.zeros((ndata, ndata))\n", - " for i in range(0, data.shape[0]):\n", - " cov_g[int(data[i, 0]), int(data[i, 1])] = data[i, 8]\n", - " cov_g[int(data[i, 1]), int(data[i, 0])] = data[i, 8]\n", - " cov_ng[int(data[i, 0]), int(data[i, 1])] = data[i, 9]\n", - " cov_ng[int(data[i, 1]), int(data[i, 0])] = data[i, 9]\n", - "\n", - " return cov_g, cov_ng, ndata\n", - "\n", - "\n", - "covfile = \"/feynman/work/dap/lcs/lg268561/UNIONS/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/data/SP_cut_Fabian/covs/out_cov_ssss_+-_cov_Ntheta20_Ntomo1_3\"\n", - "\n", - "c_g, c_ng, ndata = get_cov(covfile)\n", - "cov = c_ng + c_g\n", - "cov_g = c_g\n", - "\n", - "b = np.sort(LA.eigvals(cov))\n", - "print(\"min+max eigenvalues cov: %e, %e\" % (np.min(b), np.max(b)))\n", - "if np.min(b) <= 0.0:\n", - " print(\"non-positive eigenvalue encountered! Covariance Invalid!\")\n", - " exit()\n", - "\n", - "print(\"Covariance is postive definite!\")\n", - "\n", - "pp_var = []\n", - "for i in range(ndata):\n", - " pp_var.append(cov[i][i])\n", - "\n", - "\n", - "cmap = \"seismic\"\n", - "\n", - "pp_norm = np.zeros((ndata, ndata))\n", - "for i in range(ndata):\n", - " for j in range(ndata):\n", - " pp_norm[i][j] = cov[i][j] / np.sqrt(cov[i][i] * cov[j][j])\n", - "\n", - "print(\"Plotting correlation matrix ...\")\n", - "\n", - "# plot_path = covfile+'_plot.pdf'\n", - "\n", - "fig = plt.figure()\n", - "ax = fig.add_subplot(1, 1, 1)\n", - "ax.xaxis.tick_top()\n", - "ax.xaxis.set_ticks(np.arange(0, 41, 1))\n", - "ax.yaxis.set_ticks(np.arange(0, 41, 1))\n", - "\n", - "\n", - "plt.axvline(x=19.5, color=\"black\", linewidth=1.5)\n", - "plt.axhline(y=19.5, color=\"black\", linewidth=1.5)\n", - "\n", - "\n", - "im3 = ax.imshow(pp_norm, cmap=cmap, vmin=-1, vmax=1)\n", - "ax.get_xaxis().set_ticklabels([])\n", - "ax.get_yaxis().set_ticklabels([])\n", - "cbar = fig.colorbar(im3, orientation=\"vertical\", shrink=0.6, ticks=[-1, 0, 1])\n", - "cbar.ax.tick_params(labelsize=15)\n", - "cbar.ax.set_yticklabels([r\"$-1$\", r\"$0$\", r\"$1$\"])\n", - "\n", - "ax.text(8, -2, r\"$\\xi_+^{ij}(\\theta)$\", fontsize=22)\n", - "ax.text(30, -2, r\"$\\xi_-^{ij}(\\theta)$\", fontsize=22)\n", - "ax.text(-6, 10, r\"$\\xi_+^{ij}(\\theta)$\", fontsize=22)\n", - "ax.text(-6, 30, r\"$\\xi_-^{ij}(\\theta)$\", fontsize=22)\n", - "# ax.set_title('Blind A',fontsize=15)\n", - "\n", - "plt.savefig(\"../plots/unions_covmat.pdf\", bbox_inches=\"tight\")\n", - "\n", - "\n", - "plt.show()\n", - "# print(\"Plot saved as %s\"%(plot_path))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "26", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "my_env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cosmo_inference/notebooks/cfis_mcmc.ipynb b/cosmo_inference/notebooks/cfis_mcmc.ipynb deleted file mode 100644 index 124eaf99..00000000 --- a/cosmo_inference/notebooks/cfis_mcmc.ipynb +++ /dev/null @@ -1,1546 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "0", - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "from getdist import plots\n", - "\n", - "# import uncertainties\n", - "\n", - "plt.rc(\"mathtext\", fontset=\"stix\")\n", - "plt.rc(\"font\", family=\"sans-serif\")\n", - "\n", - "g = plots.get_subplot_plotter(width_inch=30)\n", - "g.settings.axes_fontsize = 30\n", - "g.settings.axes_labelsize = 30\n", - "g.settings.alpha_filled_add = 0.7\n", - "g.settings.legend_fontsize = 40\n", - "\n", - "\n", - "# SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE\n", - "root_dir = \"/n09data/guerrini/output_chains/\"\n", - "\n", - "\"\"\" lower_bound = ['3.0', '3.0', '3.0', '3.0', '3.0', '10.0', '10.0']\n", - "upper_bound = ['200.0', '150.0', '100.0', '80.0', '60.0', '150.0', '60.0']\n", - "roots = [\n", - " f'SP_v1.4.5_leak_corr_sc_{lc}_{hc}_10.0_200.0' for lc, hc in zip(lower_bound, upper_bound)\n", - " ] \"\"\"\n", - "\n", - "roots = [\n", - " \"SP_v1.4.5_glass_mock_1\",\n", - " \"SP_v1.4.5_glass_mock_1_takahashi\",\n", - " \"SP_v1.4.5_glass_mock_1_HM_code\",\n", - "]\n", - "\n", - "roots = [\n", - " \"SP_v1.4.5_A\",\n", - " # \"SP_v1.4.5_A_no_IA\",\n", - " # \"SP_v1.4.5_A_no_dz\",\n", - " # \"SP_v1.4.5_A_no_m_bias\",\n", - " \"SP_v1.4.5_A_sc_3_150\",\n", - " \"SP_v1.4.5_A_sc_3_60\",\n", - " # \"SP_v1.4.5_A_sc_10_150\",\n", - " # \"SP_v1.4.5_A_sc_10_60\",\n", - " # \"SP_v1.4.5_A_sc_5_150\",\n", - " # \"SP_v1.4.5_A_sc_7_150\",\n", - " \"SP_v1.4.5_A_no_leakage\",\n", - " \"SP_v1.4.5_A_no_leakage_150\",\n", - " \"SP_v1.4.5_A_no_leakage_60\",\n", - "]\n", - "\n", - "\"\"\" roots = [\n", - " f\"SP_v1.4.5_glass_mock_{i}\" for i in range(1, 17)\n", - "] \"\"\"\n", - "\n", - "\"\"\" roots = [\n", - " \"SP_v1.4.5_glass_mock_A_IA_m5_5\",\n", - " \"SP_v1.4.5_glass_mock_A_IA_G_0.57_0.5\",\n", - "] \"\"\"\n", - "\n", - "\n", - "roots = [\n", - " f\"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_{int(i)}.0_80.0_10.0_80.0\"\n", - " for i in [3, 5, 7, 10, 11]\n", - "]\n", - "\n", - "roots = [\n", - " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0\",\n", - " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0_no_alpha_beta\",\n", - "]\n", - "\n", - "roots = [\n", - " \"SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1_sc_10.0_80.0_10.0_80.0\",\n", - " \"SP_v1.4.5_leak_corr_cell\",\n", - "]\n", - "\n", - "print(roots)" - ] - }, - { - "cell_type": "markdown", - "id": "1", - "metadata": {}, - "source": [ - "## Retrieve the chains" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ - "# MAKE PARAMNAMES FILE\n", - "\n", - "for root in roots:\n", - " with open(root_dir + \"{}/samples_{}.txt\".format(\"/\" + root, root), \"r\") as file:\n", - " params = file.readline()[1:].split(\"\\t\")[:-4]\n", - " file.close()\n", - "\n", - " with open(\n", - " root_dir + \"{}/getdist_{}.paramnames\".format(\"/\" + root, root), \"w\"\n", - " ) as file:\n", - " for i in range(len(params)):\n", - " if len(params[i].split(\"--\")) > 1:\n", - " file.write(params[i].split(\"--\")[1] + \"\\n\")\n", - " else:\n", - " file.write(params[i].split(\"--\")[0] + \"\\n\")\n", - " file.close()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "# READ CHAIN\n", - "\n", - "chains = []\n", - "\n", - "for root in roots:\n", - " samples = np.loadtxt(root_dir + \"{}/samples_{}.txt\".format(root, root))\n", - " print(len(samples))\n", - " if \"nautilus\" in root:\n", - " samples = np.column_stack(\n", - " (np.exp(samples[:, -3]), samples[:, -1] - samples[:, -2], samples[:, 0:-3])\n", - " )\n", - " else:\n", - " samples = np.column_stack((samples[:, -1], samples[:, -3], samples[:, 0:-4]))\n", - " np.savetxt(root_dir + \"{}/getdist_{}.txt\".format(root, root), samples)\n", - "\n", - " chain = g.samples_for_root(\n", - " root_dir + \"{}/getdist_{}\".format(root, root),\n", - " cache=False,\n", - " settings={\"ignore_rows\": 0, \"smooth_scale_2D\": 0.5, \"smooth_scale_1D\": 0.5},\n", - " )\n", - "\n", - " chains.append(chain)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "name_list = [\n", - " \"OMEGA_M\",\n", - " \"ombh2\",\n", - " \"h0\",\n", - " \"n_s\",\n", - " \"SIGMA_8\",\n", - " \"s_8_input\",\n", - " \"logt_agn\",\n", - " \"a\",\n", - " \"m1\",\n", - " \"bias_1\",\n", - "] # ,'alpha','beta']\n", - "label_list = [\n", - " r\"\\Omega_m\",\n", - " r\"\\omega_b h^2\",\n", - " \"h_0\",\n", - " \"n_s\",\n", - " r\"\\sigma_8\",\n", - " \"S_8\",\n", - " \"log T_{AGN}\",\n", - " \"A_{IA}\",\n", - " \"m_1\",\n", - " r\"\\Delta z_1\",\n", - "] # , '\\\\alpha_{PSF}', '\\\\beta_{PSF}']\n", - "\n", - "for chain in chains:\n", - " param_names = chain.getParamNames()\n", - " for name, label in zip(name_list, label_list):\n", - " param_names.parWithName(name).label = label" - ] - }, - { - "cell_type": "markdown", - "id": "5", - "metadata": {}, - "source": [ - "## Plot the chain" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "%matplotlib inline\n", - "\n", - "\"\"\" legend_labels = [\n", - " rf'$\\theta \\\\in$ [{lc}-{hc}]' for lc, hc in zip(lower_bound, upper_bound)\n", - "] \"\"\"\n", - "\n", - "legend_labels = [rf\"GLASS mock {i}\" for i in range(1, 17)]\n", - "\n", - "legend_labels = [\"GLASS mock 1\", \"GLASS mock 1 takahashi\", \"GLASS mock 1 HM code\"]\n", - "\n", - "legend_labels = [\n", - " \"SP_v1.4.5 blind A\",\n", - " # \"SP_v1.4.5 blind A no IA\",\n", - " # r\"SP_v1.4.5 blind A no $\\Delta z$\",\n", - " # r\"SP_v1.4.5 blind A no $m_1$\",\n", - " r\"SP_v1.4.5 blind A, $\\theta \\in [3-150]$\",\n", - " r\"SP_v1.4.5 blind A, $\\theta \\in [3-60]$\",\n", - " # r\"SP_v1.4.5 blind A, $\\theta \\in [10-150]$\",\n", - " # r\"SP_v1.4.5 blind A, $\\theta \\in [10-60]$\",\n", - " # r\"SP_v1.4.5 blind A, $\\theta \\in [5-150]$\",\n", - " # r\"SP_v1.4.5 blind A, $\\theta \\in [7-150]$\",\n", - " r\"SP_v1.4.5 blind A no leakage\",\n", - " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [3-150]$\",\n", - " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [3-60]$\",\n", - "]\n", - "\n", - "legend_labels = [\n", - " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [3-80]$\",\n", - " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [5-180]$\",\n", - " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [7-180]$\",\n", - " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [10-80]$\",\n", - " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [11-80]$\",\n", - "]\n", - "\n", - "legend_labels = [\n", - " r\"SP_v1.4.5 blind A no leakage, $\\theta \\in [10-80]$\",\n", - " r\"SP_v1.4.5 blind A no leakage, $C_\\ell$\",\n", - "]\n", - "\n", - "contour_colors = [\n", - " \"cornflowerblue\",\n", - " \"salmon\",\n", - " \"darkorange\",\n", - " \"forestgreen\",\n", - " \"turquoise\",\n", - " \"darkviolet\",\n", - " \"crimson\",\n", - " \"gold\",\n", - " \"lightcoral\",\n", - " \"mediumseagreen\",\n", - " \"lightsteelblue\",\n", - " \"black\",\n", - " \"silver\",\n", - " \"peru\",\n", - " \"maroon\",\n", - " \"olive\",\n", - "]\n", - "\n", - "\"\"\" legend_labels = [\n", - " r\"GLASS mock 3\",\n", - " r\"GLASS mock 3 no PSF\",\n", - " r\"GLASS mock 3 no PSF baryons\",\n", - "] \"\"\"\n", - "\n", - "marker = {\n", - " \"OMEGA_LAMBDA\": 0.7013160542257656,\n", - " \"ombh2\": 0.024499999999999997,\n", - " \"omch2\": 0.12249999999999998,\n", - " \"h0\": 0.70,\n", - " \"n_s\": 0.96,\n", - " \"SIGMA_8\": 0.793897,\n", - " \"s_8_input\": 0.79563645,\n", - " \"m1\": 0.0,\n", - " \"bias_1\": 0.0,\n", - " #'alpha': -0.0005,\n", - " #'beta': 0.0631,\n", - " \"a\": 0.0,\n", - "}\n", - "\n", - "marker = {\n", - " \"bias_1\": -0.045,\n", - " \"m1\": 0.0,\n", - " \"a\": 0.5,\n", - " #'alpha': 0.0169,\n", - " #'beta': 1.0789\n", - "}\n", - "g.triangle_plot(\n", - " chains,\n", - " [\n", - " \"OMEGA_M\",\n", - " \"ombh2\",\n", - " \"h0\",\n", - " \"n_s\",\n", - " \"SIGMA_8\",\n", - " \"s_8_input\",\n", - " \"logt_agn\",\n", - " \"a\",\n", - " \"m1\",\n", - " \"bias_1\",\n", - " \"alpha\",\n", - " \"beta\",\n", - " ],\n", - " legend_labels=legend_labels,\n", - " legend_loc=\"upper right\",\n", - " # param_limits={'bias_1':[-0.8,0.5]},\n", - " contour_colors=contour_colors,\n", - " line_args=[{\"color\": contour_colors[i], \"ls\": \"solid\"} for i in range(16)],\n", - " # title_limit=1,\n", - " filled=True,\n", - " markers=marker,\n", - ")\n", - "\n", - "g.export(\"contour_plot_unions_cell.png\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [ - "\"\"\" legend_labels = [\n", - " rf'$\\theta \\\\in$ [{lc}-{hc}]' for lc, hc in zip(lower_bound, upper_bound)\n", - "] \"\"\"\n", - "g.triangle_plot(\n", - " chains,\n", - " [\"OMEGA_M\", \"s_8_input\", \"SIGMA_8\", \"a\"],\n", - " legend_labels=legend_labels,\n", - " legend_loc=\"upper right\",\n", - " # param_limits={'bias_1':[-0.8,0.5]},\n", - " contour_colors=contour_colors,\n", - " line_args=[{\"color\": contour_colors[i], \"ls\": \"solid\"} for i in range(16)],\n", - " title_limit=1,\n", - " filled=True,\n", - " markers=marker,\n", - ")\n", - "\n", - "g.export(\"contour_plot_s8_unions_cell.png\")" - ] - }, - { - "cell_type": "markdown", - "id": "8", - "metadata": { - "jp-MarkdownHeadingCollapsed": true, - "tags": [] - }, - "source": [ - "### Output bestfit and sigma values" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9", - "metadata": {}, - "outputs": [], - "source": [ - "#########BESTFIT AND SIGMA VALS##########\n", - "params = [\n", - " \"OMEGA_M\",\n", - " \"omega_b\",\n", - " \"h0\",\n", - " \"n_s\",\n", - " \"a_s\",\n", - " \"SIGMA_8\",\n", - " \"S_8\",\n", - " \"logt_agn\",\n", - " \"a\",\n", - " \"m1\",\n", - " \"bias_1\",\n", - " \"alpha\",\n", - " \"beta\",\n", - " \"omch2\",\n", - " \"ombh2\",\n", - "]\n", - "latex_params = [\n", - " r\"$\\Omega_{\\rm m,0}$\",\n", - " r\"$\\Omega_{\\rm b,0}$\",\n", - " r\"$h$\",\n", - " r\"$n_{\\rm s}$\",\n", - " r\"$A_{\\rm s}$\",\n", - " r\"$\\sigma_8$\",\n", - " r\"$S_8$\",\n", - " r\"$\\log_{10}{T_{\\rm AGN}}$\",\n", - " r\"$\\mathcal{A}_rm IA}$\",\n", - " r\"$m_1$\",\n", - " r\"$\\Delta z$\",\n", - " r\"$\\alpha$\",\n", - " r\"$\\beta$\",\n", - " r\"$\\Omega_{\\rm c,0}$\",\n", - " r\"$\\Omega_{\\rm b,0}$\",\n", - "]\n", - "\n", - "for chain in chains:\n", - " margestats = chain.getMargeStats()\n", - " likestats = chain.getLikeStats()\n", - " p = chain.getParams()\n", - "\n", - " for no in range(len(latex_params)):\n", - " if hasattr(p, params[no]):\n", - " param_stats = margestats.parWithName(params[no])\n", - " a = np.array(\n", - " [\n", - " param_stats.mean,\n", - " param_stats.mean - param_stats.limits[0].lower,\n", - " param_stats.limits[0].upper - param_stats.mean,\n", - " ]\n", - " )\n", - " if \"%.2g\" % a[1] == \"%.2g\" % a[2]:\n", - " latex_params[no] += r\"&$%.3g\\pm%.2g$\" % (a[0], a[1])\n", - " else:\n", - " latex_params[no] += \"&$%.3g_{-%.2g}^{+%.2g}$\" % (a[0], a[1], a[2])\n", - " else:\n", - " latex_params[no] += \"&$-$\"\n", - "\n", - "\n", - "for param in latex_params:\n", - " param += r\"\\\\\"\n", - " print(param)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "10", - "metadata": {}, - "outputs": [], - "source": [ - "chain = chains[0]\n", - "\n", - "margestats = chain.getMargeStats()\n", - "likestats = chain.getLikeStats()\n", - "p = chain.getParams()\n", - "\n", - "for no in range(len(latex_params)):\n", - " if hasattr(p, params[no]):\n", - " param_stats = margestats.parWithName(params[no])\n", - " a = np.array([param_stats.mean])\n", - " print(params[no], a[0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11", - "metadata": {}, - "outputs": [], - "source": [ - "#########BESTFIT AND SIGMA VALS##########\n", - "params = [\n", - " \"OMEGA_M\",\n", - " \"omega_b\",\n", - " \"h0\",\n", - " \"n_s\",\n", - " \"a_s\",\n", - " \"SIGMA_8\",\n", - " \"S_8\",\n", - " \"logt_agn\",\n", - " \"a\",\n", - " \"m1\",\n", - " \"bias_1\",\n", - " \"alpha\",\n", - " \"beta\",\n", - " \"omch2\",\n", - " \"ombh2\",\n", - "]\n", - "latex_params = [\n", - " r\"$\\Omega_{\\rm m,0}$\",\n", - " r\"$\\Omega_{\\rm b,0}$\",\n", - " r\"$h$\",\n", - " r\"$n_{\\rm s}$\",\n", - " r\"$A_{\\rm s}$\",\n", - " r\"$\\sigma_8$\",\n", - " r\"$S_8$\",\n", - " r\"$\\log_{10}{T_{\\rm AGN}}$\",\n", - " r\"$\\mathcal{A}_rm IA}$\",\n", - " r\"$m_1$\",\n", - " r\"$\\Delta z$\",\n", - " r\"$\\alpha$\",\n", - " r\"$\\beta$\",\n", - " r\"$\\Omega_{\\rm c,0}$\",\n", - " r\"$\\Omega_{\\rm b,0}$\",\n", - "]\n", - "\n", - "values = {param: [] for param in params}\n", - "for i, chain in enumerate(chains):\n", - " margestats = chain.getMargeStats()\n", - " likestats = chain.getLikeStats()\n", - " p = chain.getParams()\n", - "\n", - " print(legend_labels[i])\n", - "\n", - " for param in params:\n", - " if hasattr(p, param):\n", - " param_stats = margestats.parWithName(param)\n", - " a = np.array(\n", - " [\n", - " param_stats.mean,\n", - " param_stats.mean - param_stats.limits[0].lower,\n", - " param_stats.limits[0].upper - param_stats.mean,\n", - " ]\n", - " )\n", - " print(f\"{param}: {a[0]:.3g}_-{a[1]:.2g}^+{a[1]:.2g}\")\n", - " values[param].append(a[0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "12", - "metadata": {}, - "outputs": [], - "source": [ - "bestfit_ix = np.argmax(chains[0].loglikes)\n", - "maxlike = chains[0].loglikes[bestfit_ix]\n", - "print(maxlike)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "13", - "metadata": {}, - "outputs": [], - "source": [ - "chains[0].loglikes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "14", - "metadata": {}, - "outputs": [], - "source": [ - "print(chains[0].likeStats)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15", - "metadata": {}, - "outputs": [], - "source": [ - "plt.figure(figsize=(15, 5))\n", - "\n", - "plt.subplot(131)\n", - "\n", - "plt.hist(values[\"OMEGA_M\"], bins=10, color=\"cornflowerblue\", alpha=0.5)\n", - "plt.axvline(0.301316, color=\"black\", linestyle=\"--\", label=\"True value\")\n", - "plt.xlabel(r\"$\\Omega_{\\rm m,0}$\")\n", - "plt.ylabel(\"Counts\")\n", - "plt.legend()\n", - "\n", - "plt.subplot(132)\n", - "\n", - "plt.hist(values[\"SIGMA_8\"], bins=10, color=\"cornflowerblue\", alpha=0.5)\n", - "plt.axvline(0.793897, color=\"black\", linestyle=\"--\", label=\"True value\")\n", - "plt.xlabel(r\"$\\sigma_8$\")\n", - "plt.ylabel(\"Counts\")\n", - "plt.legend()\n", - "\n", - "plt.subplot(133)\n", - "\n", - "plt.hist(values[\"S_8\"], bins=10, color=\"cornflowerblue\", alpha=0.5)\n", - "plt.axvline(0.79563645, color=\"black\", linestyle=\"--\", label=\"True value\")\n", - "plt.xlabel(r\"$S_8$\")\n", - "plt.ylabel(\"Counts\")\n", - "plt.legend()\n", - "plt.tight_layout()\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "16", - "metadata": {}, - "outputs": [], - "source": [ - "np.sum(np.abs(np.array(values[\"S_8\"]) - 0.79563645) < 0.03) / len(values[\"S_8\"])" - ] - }, - { - "cell_type": "markdown", - "id": "17", - "metadata": {}, - "source": [ - "## Looking at best fit" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "18", - "metadata": {}, - "outputs": [], - "source": [ - "from astropy.io import fits\n", - "\n", - "version = \"SP_v1.4.5_glass_mock_1\"\n", - "\n", - "data = fits.open(\n", - " f\"/home/guerrini/sp_validation/cosmo_inference/data/{version}/cosmosis_{version}.fits\"\n", - ")\n", - "xi_plus = data[\"XI_PLUS\"].data\n", - "xi_minus = data[\"XI_MINUS\"].data\n", - "cov_mat = data[\"COVMAT\"].data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "19", - "metadata": {}, - "outputs": [], - "source": [ - "plt.figure(figsize=(15, 15))\n", - "\n", - "plt.subplot(211)\n", - "\n", - "plt.errorbar(\n", - " xi_plus[\"ANG\"],\n", - " xi_plus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - ")\n", - "\n", - "plt.ylabel(r\"$\\xi_{+}$\")\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.axvline(3.0, color=\"grey\", linestyle=\"--\", label=\"3 arcmin\")\n", - "plt.axvline(100.0, color=\"grey\", linestyle=\"--\", label=\"100 arcmin\")\n", - "plt.legend()\n", - "\n", - "plt.subplot(212)\n", - "\n", - "plt.errorbar(\n", - " xi_minus[\"ANG\"],\n", - " xi_minus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - ")\n", - "\n", - "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", - "plt.ylabel(r\"$\\xi_{-}$\")\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.axvline(10.0, color=\"grey\", linestyle=\"--\", label=\"10 arcmin\")\n", - "plt.axvline(200.0, color=\"grey\", linestyle=\"--\", label=\"200 arcmin\")\n", - "plt.legend()\n", - "\n", - "plt.savefig(\"xi_data.png\")\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "20", - "metadata": {}, - "outputs": [], - "source": [ - "import pyccl as ccl\n", - "\n", - "# Get theory correlation function from CCL\n", - "# Define the cosmology\n", - "theta_arcmin = np.logspace(np.log10(0.1), np.log10(250), 1000)\n", - "h = 0.7\n", - "Oc = 0.25\n", - "Ob = 0.05\n", - "sigma8 = 0.793897\n", - "n_s = 0.96\n", - "cosmo = ccl.Cosmology(\n", - " h=h,\n", - " Omega_c=Oc,\n", - " Omega_b=Ob,\n", - " sigma8=sigma8,\n", - " n_s=n_s,\n", - " transfer_function=\"boltzmann_camb\",\n", - ")\n", - "\n", - "# Define the redshift distribution\n", - "z, dndz = np.loadtxt(\n", - " \"/home/guerrini/sp_validation/cosmo_inference/cosmocov_config/dndz_test.txt\",\n", - " unpack=True,\n", - ")\n", - "\n", - "tracer = ccl.WeakLensingTracer(cosmo, dndz=(z, dndz), ia_bias=None)\n", - "\n", - "# COmpute the angular power spectrum C_ell\n", - "ell = np.logspace(0, np.log10(10000), 2000)\n", - "cl_gg = ccl.angular_cl(cosmo, tracer, tracer, ell)\n", - "\n", - "# Compute the 2PCF\n", - "theta_deg = theta_arcmin / 60\n", - "# xi+ fit\n", - "xi_p_theta_true = ccl.correlation(\n", - " cosmo, ell=ell, C_ell=cl_gg, theta=theta_deg, type=\"GG+\"\n", - ")\n", - "# xi- fit\n", - "xi_m_theta_true = ccl.correlation(\n", - " cosmo, ell=ell, C_ell=cl_gg, theta=theta_deg, type=\"GG-\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "21", - "metadata": {}, - "outputs": [], - "source": [ - "# Get theory correlation function from CCL\n", - "# Define the cosmology\n", - "theta_arcmin = np.logspace(np.log10(0.1), np.log10(250), 1000)\n", - "h = 0.6982064176424748\n", - "Oc = 0.21522128974860827 / h**2\n", - "print(\"Omega_c:\", Oc)\n", - "Ob = 0.024410205304489712 / h**2\n", - "print(\"Omega_b:\", Ob)\n", - "sigma8 = 0.5330533925822226\n", - "print(\"S8:\", sigma8 * np.sqrt((Oc + Ob) / 0.3))\n", - "n_s = 0.9867762122563981\n", - "a_ia = 0.0\n", - "cosmo = ccl.Cosmology(\n", - " h=h,\n", - " Omega_c=Oc,\n", - " Omega_b=Ob,\n", - " sigma8=sigma8,\n", - " n_s=n_s,\n", - " transfer_function=\"boltzmann_camb\",\n", - ")\n", - "\n", - "# Define the redshift distribution\n", - "z, dndz = np.loadtxt(\n", - " \"/home/guerrini/sp_validation/cosmo_inference/cosmocov_config/dndz_test.txt\",\n", - " unpack=True,\n", - ")\n", - "\n", - "tracer = ccl.WeakLensingTracer(\n", - " cosmo, dndz=(z, dndz), ia_bias=(z, np.ones_like(z) * a_ia)\n", - ")\n", - "\n", - "# COmpute the angular power spectrum C_ell\n", - "ell = np.logspace(0, np.log10(10000), 2000)\n", - "cl_gg = ccl.angular_cl(cosmo, tracer, tracer, ell)\n", - "\n", - "# Compute the 2PCF\n", - "theta_deg = theta_arcmin / 60\n", - "# xi+ fit\n", - "xi_p_theta_fit = ccl.correlation(\n", - " cosmo, ell=ell, C_ell=cl_gg, theta=theta_deg, type=\"GG+\"\n", - ")\n", - "# xi- fit\n", - "xi_m_theta_fit = ccl.correlation(\n", - " cosmo, ell=ell, C_ell=cl_gg, theta=theta_deg, type=\"GG-\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "22", - "metadata": {}, - "outputs": [], - "source": [ - "# Get theory correlation function from CCL\n", - "# Define the cosmology\n", - "theta_arcmin = np.logspace(np.log10(0.1), np.log10(250), 1000)\n", - "h = 0.6982064176424748\n", - "Oc = 0.21522128974860827 / h**2\n", - "print(\"Omega_c:\", Oc)\n", - "Ob = 0.024410205304489712 / h**2\n", - "print(\"Omega_b:\", Ob)\n", - "sigma8 = 0.5330533925822226\n", - "print(\"S8:\", sigma8 * np.sqrt((Oc + Ob) / 0.3))\n", - "n_s = 0.9867762122563981\n", - "a_ia = -0.4225120529551343\n", - "cosmo = ccl.Cosmology(\n", - " h=h,\n", - " Omega_c=Oc,\n", - " Omega_b=Ob,\n", - " sigma8=sigma8,\n", - " n_s=n_s,\n", - " transfer_function=\"boltzmann_camb\",\n", - ")\n", - "\n", - "# Define the redshift distribution\n", - "z, dndz = np.loadtxt(\n", - " \"/home/guerrini/sp_validation/cosmo_inference/cosmocov_config/dndz_test.txt\",\n", - " unpack=True,\n", - ")\n", - "\n", - "tracer = ccl.WeakLensingTracer(\n", - " cosmo, dndz=(z, dndz), ia_bias=(z, np.ones_like(z) * a_ia)\n", - ")\n", - "\n", - "# COmpute the angular power spectrum C_ell\n", - "ell = np.logspace(0, np.log10(10000), 2000)\n", - "cl_gg = ccl.angular_cl(cosmo, tracer, tracer, ell)\n", - "\n", - "# Compute the 2PCF\n", - "theta_deg = theta_arcmin / 60\n", - "# xi+ fit\n", - "xi_p_theta_fit_IA = ccl.correlation(\n", - " cosmo, ell=ell, C_ell=cl_gg, theta=theta_deg, type=\"GG+\"\n", - ")\n", - "# xi- fit\n", - "xi_m_theta_fit_IA = ccl.correlation(\n", - " cosmo, ell=ell, C_ell=cl_gg, theta=theta_deg, type=\"GG-\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "23", - "metadata": {}, - "outputs": [], - "source": [ - "plt.figure(figsize=(15, 15))\n", - "\n", - "plt.subplot(211)\n", - "\n", - "plt.errorbar(\n", - " xi_plus[\"ANG\"],\n", - " xi_plus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - ")\n", - "plt.plot(theta_arcmin, xi_p_theta_fit, label=\"SP_v1.4.5 fit\", color=\"red\")\n", - "plt.plot(theta_arcmin, xi_p_theta_true, label=\"SP_v1.4.5 true\", color=\"blue\")\n", - "plt.plot(theta_arcmin, xi_p_theta_fit_IA, label=\"SP_v1.4.5 fit IA\", color=\"green\")\n", - "\n", - "plt.ylabel(r\"$\\xi_{+}$\")\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.axvline(3.0, color=\"grey\", linestyle=\"--\", label=\"3 arcmin\")\n", - "plt.axvline(100.0, color=\"grey\", linestyle=\"--\", label=\"100 arcmin\")\n", - "plt.legend()\n", - "\n", - "plt.subplot(212)\n", - "\n", - "plt.errorbar(\n", - " xi_minus[\"ANG\"],\n", - " xi_minus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - ")\n", - "plt.plot(theta_arcmin, xi_m_theta_fit, label=\"SP_v1.4.5 fit\", color=\"red\")\n", - "plt.plot(theta_arcmin, xi_m_theta_true, label=\"SP_v1.4.5 true\", color=\"blue\")\n", - "plt.plot(theta_arcmin, xi_m_theta_fit_IA, label=\"SP_v1.4.5 fit IA\", color=\"green\")\n", - "\n", - "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", - "plt.ylabel(r\"$\\xi_{-}$\")\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.axvline(10.0, color=\"grey\", linestyle=\"--\", label=\"10 arcmin\")\n", - "plt.axvline(200.0, color=\"grey\", linestyle=\"--\", label=\"200 arcmin\")\n", - "plt.legend()\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24", - "metadata": {}, - "outputs": [], - "source": [ - "# Add best-fit model\n", - "root_dir = \"/n09data/guerrini/output_chains/output_result/glass_mock_1/\"\n", - "xi_plus_bf_no_psf = np.loadtxt(root_dir + \"best_fit/shear_xi_plus/bin_1_1.txt\")\n", - "xi_minus_bf_no_psf = np.loadtxt(root_dir + \"best_fit/shear_xi_minus/bin_1_1.txt\")\n", - "xi_sys_p = np.loadtxt(root_dir + \"best_fit/xi_sys/shear_xi_plus.txt\")\n", - "xi_sys_m = np.loadtxt(root_dir + \"best_fit/xi_sys/shear_xi_minus.txt\")\n", - "theta_xi_sys = np.loadtxt(root_dir + \"best_fit/xi_sys/theta.txt\")\n", - "theta_xi_sys = theta_xi_sys * 180 * 60 / np.pi\n", - "angle = np.loadtxt(root_dir + \"best_fit/shear_xi_plus/theta.txt\")\n", - "angle = angle * 180 * 60 / np.pi\n", - "\n", - "mask = (angle < 250) & (angle > 0.1)\n", - "\n", - "from scipy.interpolate import interp1d\n", - "\n", - "xi_sys_p_interp = interp1d(\n", - " theta_xi_sys, xi_sys_p, kind=\"linear\", fill_value=\"extrapolate\"\n", - ")\n", - "xi_sys_m_interp = interp1d(\n", - " theta_xi_sys, xi_sys_m, kind=\"linear\", fill_value=\"extrapolate\"\n", - ")\n", - "xi_plus_bf = xi_plus_bf_no_psf[mask] + xi_sys_p_interp(angle[mask])\n", - "xi_minus_bf = xi_minus_bf_no_psf[mask] + xi_sys_m_interp(angle[mask])\n", - "\n", - "xi_plus_bf_no_baryons = np.loadtxt(\n", - " root_dir + \"best_fit_no_feedback/shear_xi_plus/bin_1_1.txt\"\n", - ")\n", - "xi_minus_bf_no_baryons = np.loadtxt(\n", - " root_dir + \"best_fit_no_feedback/shear_xi_minus/bin_1_1.txt\"\n", - ")\n", - "\n", - "xi_plus_bf_no_IA = np.loadtxt(root_dir + \"best_fit_no_IA/shear_xi_plus/bin_1_1.txt\")\n", - "xi_minus_bf_no_IA = np.loadtxt(root_dir + \"best_fit_no_IA/shear_xi_minus/bin_1_1.txt\")\n", - "\n", - "xi_plus_truth_cosmosis = np.loadtxt(root_dir + \"/truth/shear_xi_plus/bin_1_1.txt\")\n", - "xi_minus_truth_cosmosis = np.loadtxt(root_dir + \"/truth/shear_xi_minus/bin_1_1.txt\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "25", - "metadata": {}, - "outputs": [], - "source": [ - "plt.figure(figsize=(15, 15))\n", - "\n", - "plt.subplot(211)\n", - "\n", - "plt.errorbar(\n", - " xi_plus[\"ANG\"],\n", - " xi_plus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - ")\n", - "plt.plot(theta_arcmin, xi_p_theta_true, label=\"SP_v1.4.5 true\", color=\"blue\")\n", - "plt.plot(angle[mask], xi_plus_bf, label=\"SP_v1.4.5 COSMOSIS\", color=\"orange\")\n", - "plt.plot(\n", - " angle[mask],\n", - " xi_plus_bf_no_psf[mask],\n", - " label=\"SP_v1.4.5 COSMOSIS no PSF\",\n", - " color=\"green\",\n", - ")\n", - "plt.plot(\n", - " angle[mask],\n", - " xi_plus_bf_no_baryons[mask],\n", - " label=\"SP_v1.4.5 COSMOSIS no baryons\",\n", - " color=\"red\",\n", - ")\n", - "plt.plot(\n", - " angle[mask],\n", - " xi_plus_bf_no_IA[mask],\n", - " label=\"SP_v1.4.5 COSMOSIS no IA\",\n", - " color=\"purple\",\n", - ")\n", - "plt.plot(\n", - " angle[mask],\n", - " xi_plus_truth_cosmosis[mask],\n", - " label=\"SP_v1.4.5 COSMOSIS truth\",\n", - " color=\"black\",\n", - " linestyle=\"--\",\n", - ")\n", - "\n", - "plt.ylabel(r\"$\\xi_{+}$\")\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.axvline(3.0, color=\"grey\", linestyle=\"--\", label=\"3 arcmin\")\n", - "plt.axvline(150.0, color=\"grey\", linestyle=\"--\", label=\"150 arcmin\")\n", - "plt.legend()\n", - "\n", - "plt.subplot(212)\n", - "\n", - "plt.errorbar(\n", - " xi_minus[\"ANG\"],\n", - " xi_minus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - " markersize=2,\n", - ")\n", - "plt.plot(theta_arcmin, xi_m_theta_true, label=\"SP_v1.4.5 true\", color=\"blue\")\n", - "plt.plot(angle[mask], xi_minus_bf, label=\"SP_v1.4.5 COSMOSIS\", color=\"orange\")\n", - "plt.plot(\n", - " angle[mask],\n", - " xi_minus_bf_no_psf[mask],\n", - " label=\"SP_v1.4.5 COSMOSIS no PSF\",\n", - " color=\"green\",\n", - ")\n", - "plt.plot(\n", - " angle[mask],\n", - " xi_minus_bf_no_baryons[mask],\n", - " label=\"SP_v1.4.5 COSMOSIS no baryons\",\n", - " color=\"red\",\n", - ")\n", - "plt.plot(\n", - " angle[mask],\n", - " xi_minus_bf_no_IA[mask],\n", - " label=\"SP_v1.4.5 COSMOSIS no IA\",\n", - " color=\"purple\",\n", - ")\n", - "plt.plot(\n", - " angle[mask],\n", - " xi_minus_truth_cosmosis[mask],\n", - " label=\"SP_v1.4.5 COSMOSIS truth\",\n", - " color=\"black\",\n", - " linestyle=\"--\",\n", - ")\n", - "\n", - "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", - "plt.ylabel(r\"$\\xi_{-}$\")\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.axvline(10.0, color=\"grey\", linestyle=\"--\", label=\"10 arcmin\")\n", - "plt.axvline(200.0, color=\"grey\", linestyle=\"--\", label=\"200 arcmin\")\n", - "plt.legend()\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "26", - "metadata": {}, - "outputs": [], - "source": [ - "import treecorr\n", - "\n", - "theta_min = 0.1\n", - "theta_max = 250.0\n", - "nbins = 20\n", - "var_method = \"jackknife\"\n", - "\n", - "treecorr_config = {\n", - " \"ra_units\": \"degrees\",\n", - " \"dec_units\": \"degrees\",\n", - " \"min_sep\": theta_min,\n", - " \"max_sep\": theta_max,\n", - " \"sep_units\": \"arcmin\",\n", - " \"nbins\": nbins,\n", - " \"var_method\": var_method,\n", - "}\n", - "\n", - "gg = treecorr.GGCorrelation(treecorr_config)\n", - "\n", - "# Load the measurement\n", - "cat = fits.getdata(\n", - " \"/n09data/guerrini/glass_mock/results/unions_glass_sim_00001_4096.fits\"\n", - ")\n", - "\n", - "e1 = cat[\"e1\"]\n", - "e2 = cat[\"e2\"]\n", - "ra = cat[\"ra\"]\n", - "dec = cat[\"dec\"]\n", - "\n", - "# Create the catalog\n", - "cat = treecorr.Catalog(\n", - " ra=ra, dec=dec, ra_units=\"degrees\", dec_units=\"degrees\", g1=e1, g2=e2, npatch=200\n", - ")\n", - "\n", - "# Process the catalog\n", - "gg.process(cat)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "27", - "metadata": {}, - "outputs": [], - "source": [ - "cov = treecorr.estimate_multi_cov([gg], method=\"jackknife\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "28", - "metadata": {}, - "outputs": [], - "source": [ - "data_vector_sim = []\n", - "\n", - "for ver in [f\"SP_v1.4.5_glass_mock_{i}\" for i in range(1, 17)]:\n", - " data = fits.open(\n", - " f\"/home/guerrini/sp_validation/cosmo_inference/data/{ver}/cosmosis_{ver}.fits\"\n", - " )\n", - " xi_plus = data[\"XI_PLUS\"].data\n", - " xi_minus = data[\"XI_MINUS\"].data\n", - " data_vector_sim.append(np.concatenate((xi_plus[\"VALUE\"], xi_minus[\"VALUE\"])))\n", - "\n", - "data_vector_sim = np.array(data_vector_sim)\n", - "\n", - "data_vector_sim.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "29", - "metadata": {}, - "outputs": [], - "source": [ - "cov_sim = np.cov(data_vector_sim.T)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "30", - "metadata": {}, - "outputs": [], - "source": [ - "ver_sacha = \"SP_v1.4.5\"\n", - "\n", - "cov_th_sacha = np.loadtxt(\n", - " \"/home/guerrini/sp_validation/cosmo_inference/data/{}/covs/cov_{}.txt\".format(\n", - " ver_sacha, ver_sacha\n", - " )\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "31", - "metadata": {}, - "outputs": [], - "source": [ - "plt.figure()\n", - "\n", - "plt.plot(xi_plus[\"ANG\"], np.diag(cov)[:20])\n", - "plt.plot(xi_plus[\"ANG\"], np.diag(cov_sim)[:20], label=\"SP_v1.4.5 data\", color=\"red\")\n", - "plt.plot(\n", - " xi_plus[\"ANG\"], np.diag(cov_th_sacha)[:20], label=\"SP_v1.4.5 data\", color=\"green\"\n", - ")\n", - "plt.plot(xi_plus[\"ANG\"], np.diag(cov_mat)[:20], label=\"SP_v1.4.5 data\", color=\"black\")\n", - "\n", - "plt.ylabel(\"Diagonal of the covariance\")\n", - "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", - "\n", - "plt.yscale(\"log\")\n", - "plt.xscale(\"log\")\n", - "plt.savefig(\"check_cov.png\")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "32", - "metadata": {}, - "outputs": [], - "source": [ - "gg.varxip" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "33", - "metadata": {}, - "outputs": [], - "source": [ - "np.sqrt(gg.varxip)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "34", - "metadata": {}, - "outputs": [], - "source": [ - "np.sqrt(np.diag(cov_mat)[0:20])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "35", - "metadata": {}, - "outputs": [], - "source": [ - "plt.figure(figsize=(15, 15))\n", - "\n", - "plt.subplot(211)\n", - "\n", - "plt.errorbar(\n", - " xi_plus[\"ANG\"],\n", - " xi_plus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - ")\n", - "plt.plot(angle[mask], xi_plus_bf[mask], label=\"Best-fit model\", color=\"red\")\n", - "\n", - "plt.ylabel(r\"$\\xi_{+}$\")\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.axvline(3.0, color=\"grey\", linestyle=\"--\", label=\"3 arcmin\")\n", - "plt.axvline(100.0, color=\"grey\", linestyle=\"--\", label=\"100 arcmin\")\n", - "plt.legend()\n", - "\n", - "plt.subplot(212)\n", - "\n", - "plt.errorbar(\n", - " xi_minus[\"ANG\"],\n", - " xi_minus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - ")\n", - "plt.plot(angle[mask], xi_minus_bf[mask], label=\"Best-fit model\", color=\"red\")\n", - "\n", - "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", - "plt.ylabel(r\"$\\xi_{-}$\")\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.axvline(10.0, color=\"grey\", linestyle=\"--\", label=\"10 arcmin\")\n", - "plt.axvline(200.0, color=\"grey\", linestyle=\"--\", label=\"200 arcmin\")\n", - "plt.legend()\n", - "\n", - "plt.savefig(\"xi_data_bf.png\")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "36", - "metadata": {}, - "outputs": [], - "source": [ - "# Add PSF systematic\n", - "xi_sys_plus = np.loadtxt(root_dir + \"/xi_sys/shear_xi_plus.txt\")\n", - "xi_sys_minus = np.loadtxt(root_dir + \"/xi_sys/shear_xi_minus.txt\")\n", - "theta_sys = np.loadtxt(root_dir + \"/xi_sys/theta.txt\")\n", - "theta_sys = theta_sys * 180 * 60 / np.pi" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "37", - "metadata": {}, - "outputs": [], - "source": [ - "plt.figure(figsize=(15, 15))\n", - "\n", - "plt.subplot(211)\n", - "\n", - "plt.errorbar(\n", - " xi_plus[\"ANG\"],\n", - " xi_plus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - ")\n", - "plt.plot(angle[mask], xi_plus_bf[mask], label=\"Best-fit model wo SYS\", color=\"red\")\n", - "plt.plot(theta_sys, xi_sys_plus, label=r\"$\\xi_{\\rm sys}$\", color=\"green\")\n", - "\n", - "plt.ylabel(r\"$\\xi_{+}$\")\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.axvline(3.0, color=\"grey\", linestyle=\"--\", label=\"3 arcmin\")\n", - "plt.axvline(100.0, color=\"grey\", linestyle=\"--\", label=\"100 arcmin\")\n", - "plt.legend()\n", - "\n", - "plt.subplot(212)\n", - "\n", - "plt.errorbar(\n", - " xi_minus[\"ANG\"],\n", - " xi_minus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - ")\n", - "plt.plot(angle[mask], xi_minus_bf[mask], label=\"Best-fit model wo SYS\", color=\"red\")\n", - "plt.plot(theta_sys, xi_sys_minus, label=r\"$\\xi_{\\rm sys}$\", color=\"green\")\n", - "\n", - "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", - "plt.ylabel(r\"$\\xi_{-}$\")\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.axvline(10.0, color=\"grey\", linestyle=\"--\", label=\"10 arcmin\")\n", - "plt.axvline(200.0, color=\"grey\", linestyle=\"--\", label=\"200 arcmin\")\n", - "plt.legend()\n", - "\n", - "plt.savefig(\"xi_data_bf_sys.png\")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "38", - "metadata": {}, - "outputs": [], - "source": [ - "shear_cl = np.loadtxt(root_dir + \"/shear_cl/bin_1_1.txt\")\n", - "shear_cl_gg = np.loadtxt(root_dir + \"/shear_cl_gg/bin_1_1.txt\")\n", - "shear_cl_gi = np.loadtxt(root_dir + \"/shear_cl_gi/bin_1_1.txt\")\n", - "shear_cl_ii = np.loadtxt(root_dir + \"/shear_cl_ii/bin_1_1.txt\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "39", - "metadata": {}, - "outputs": [], - "source": [ - "A = 3.355083374185272\n", - "np.isclose(shear_cl, shear_cl_gg + 2 * shear_cl_gi + shear_cl_ii)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "40", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "41", - "metadata": {}, - "outputs": [], - "source": [ - "# Add the lensing part without intrinsic alignment\n", - "root_dir = \"/n09data/guerrini/output_chains/test_pipeline/\"\n", - "xi_plus_wo_ia = np.loadtxt(root_dir + \"/shear_xi_plus_wo_IA/bin_1_1.txt\")\n", - "xi_minus_wo_ia = np.loadtxt(root_dir + \"/shear_xi_minus_wo_IA/bin_1_1.txt\")\n", - "angle = np.loadtxt(root_dir + \"/shear_xi_plus_wo_IA/theta.txt\")\n", - "angle = angle * 180 * 60 / np.pi\n", - "\n", - "mask = (angle < 250) & (angle > 0.1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "42", - "metadata": {}, - "outputs": [], - "source": [ - "plt.figure(figsize=(15, 15))\n", - "\n", - "plt.subplot(211)\n", - "\n", - "plt.errorbar(\n", - " xi_plus[\"ANG\"],\n", - " xi_plus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - ")\n", - "plt.plot(angle[mask], xi_plus_bf[mask], label=\"Best-fit model wo SYS\", color=\"red\")\n", - "plt.plot(theta_sys, xi_sys_plus, label=r\"$\\xi_{\\rm sys}$\", color=\"green\")\n", - "plt.plot(\n", - " angle[mask], xi_plus_wo_ia[mask], label=\"Best-fit model wo IA and SYS\", color=\"blue\"\n", - ")\n", - "\n", - "plt.ylabel(r\"$\\xi_{+}$\")\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.axvline(3.0, color=\"grey\", linestyle=\"--\", label=\"3 arcmin\")\n", - "plt.axvline(100.0, color=\"grey\", linestyle=\"--\", label=\"100 arcmin\")\n", - "plt.legend()\n", - "\n", - "plt.subplot(212)\n", - "\n", - "plt.errorbar(\n", - " xi_minus[\"ANG\"],\n", - " xi_minus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - ")\n", - "plt.plot(angle[mask], xi_minus_bf[mask], label=\"Best-fit model wo SYS\", color=\"red\")\n", - "plt.plot(theta_sys, xi_sys_minus, label=r\"$\\xi_{\\rm sys}$\", color=\"green\")\n", - "plt.plot(\n", - " angle[mask],\n", - " xi_minus_wo_ia[mask],\n", - " label=\"Best-fit model wo IA and SYS\",\n", - " color=\"blue\",\n", - ")\n", - "\n", - "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", - "plt.ylabel(r\"$\\xi_{-}$\")\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.axvline(10.0, color=\"grey\", linestyle=\"--\", label=\"10 arcmin\")\n", - "plt.axvline(200.0, color=\"grey\", linestyle=\"--\", label=\"200 arcmin\")\n", - "plt.legend()\n", - "\n", - "plt.savefig(\"xi_data_bf_sys_ia.png\")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "43", - "metadata": {}, - "outputs": [], - "source": [ - "# Add the lensing part without intrinsic alignment\n", - "root_dir = \"/n09data/guerrini/output_chains/test_pipeline/\"\n", - "xi_plus_reas = np.loadtxt(root_dir + \"/shear_xi_plus/bin_1_1.txt\")\n", - "xi_minus_reas = np.loadtxt(root_dir + \"/shear_xi_minus/bin_1_1.txt\")\n", - "angle_reas = np.loadtxt(root_dir + \"/shear_xi_plus/theta.txt\")\n", - "angle_reas = angle_reas * 180 * 60 / np.pi\n", - "\n", - "mask = (angle < 250) & (angle > 0.1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "44", - "metadata": {}, - "outputs": [], - "source": [ - "plt.figure(figsize=(15, 15))\n", - "\n", - "plt.subplot(211)\n", - "\n", - "plt.errorbar(\n", - " xi_plus[\"ANG\"],\n", - " xi_plus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[:20],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - ")\n", - "plt.plot(angle[mask], xi_plus_bf[mask], label=\"Best-fit model wo SYS\", color=\"red\")\n", - "plt.plot(theta_sys, xi_sys_plus, label=r\"$\\xi_{\\rm sys}$\", color=\"green\")\n", - "plt.plot(\n", - " angle[mask], xi_plus_wo_ia[mask], label=\"Best-fit model wo IA and SYS\", color=\"blue\"\n", - ")\n", - "plt.plot(\n", - " angle_reas[mask],\n", - " xi_plus_reas[mask],\n", - " label=\"Lower IA\",\n", - " color=\"orange\",\n", - " linestyle=\"--\",\n", - ")\n", - "\n", - "plt.ylabel(r\"$\\xi_{+}$\")\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.axvline(3.0, color=\"grey\", linestyle=\"--\", label=\"3 arcmin\")\n", - "plt.axvline(100.0, color=\"grey\", linestyle=\"--\", label=\"100 arcmin\")\n", - "plt.legend()\n", - "\n", - "plt.subplot(212)\n", - "\n", - "plt.errorbar(\n", - " xi_minus[\"ANG\"],\n", - " xi_minus[\"VALUE\"],\n", - " yerr=np.sqrt(np.diag(cov_mat))[20:40],\n", - " fmt=\"o\",\n", - " label=\"SP_v1.4.5 data\",\n", - " color=\"black\",\n", - ")\n", - "plt.plot(angle[mask], xi_minus_bf[mask], label=\"Best-fit model wo SYS\", color=\"red\")\n", - "plt.plot(theta_sys, xi_sys_minus, label=r\"$\\xi_{\\rm sys}$\", color=\"green\")\n", - "plt.plot(\n", - " angle[mask],\n", - " xi_minus_wo_ia[mask],\n", - " label=\"Best-fit model wo IA and SYS\",\n", - " color=\"blue\",\n", - ")\n", - "plt.plot(\n", - " angle_reas[mask],\n", - " xi_minus_reas[mask],\n", - " label=\"Lower IA\",\n", - " color=\"orange\",\n", - " linestyle=\"--\",\n", - ")\n", - "\n", - "plt.xlabel(r\"$\\theta$ [arcmin]\")\n", - "plt.ylabel(r\"$\\xi_{-}$\")\n", - "plt.xscale(\"log\")\n", - "plt.yscale(\"log\")\n", - "plt.axvline(10.0, color=\"grey\", linestyle=\"--\", label=\"10 arcmin\")\n", - "plt.axvline(200.0, color=\"grey\", linestyle=\"--\", label=\"200 arcmin\")\n", - "plt.legend()\n", - "\n", - "plt.savefig(\"xi_data_bf_sys_ia_reas.png\")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "45", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "base", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.0" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cosmo_inference/notebooks/get_prior_psf_leakage.ipynb b/cosmo_inference/notebooks/get_prior_psf_leakage.ipynb deleted file mode 100644 index 1dae9a36..00000000 --- a/cosmo_inference/notebooks/get_prior_psf_leakage.ipynb +++ /dev/null @@ -1,269 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "0", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "if not os.path.exists(\"./Plots\"):\n", - " os.makedirs(\"./Plots\")\n", - "\n", - "# Trick to plot with tex\n", - "os.environ[\"LD_LIBRARY_PATH\"] = \"\"\n", - "os.environ[\"CONDA_PREFIX\"] = \"/home/guerrini/.conda/envs/sp_validation_3.11\"\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import seaborn as sns\n", - "from astropy.io import fits\n", - "from getdist import MCSamples, plots\n", - "from shear_psf_leakage.rho_tau_stat import PSFErrorFit, RhoStat, TauStat\n", - "\n", - "# Use paper style and seaborn with husl palette\n", - "plt.style.use(\"/home/guerrini/matplotlib_config/paper.mplstyle\")\n", - "# Set default palette - will be updated per plot as needed\n", - "sns.set_palette(\"husl\")\n", - "%matplotlib inline\n", - "\n", - "g = plots.get_subplot_plotter(width_inch=30)\n", - "g.settings.axes_fontsize = 30\n", - "g.settings.axes_labelsize = 30\n", - "g.settings.alpha_filled_add = 0.7\n", - "g.settings.legend_fontsize = 25" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1", - "metadata": {}, - "outputs": [], - "source": [ - "data_path = \"/home/guerrini/sp_validation/cosmo_inference/data/\"\n", - "\n", - "path_cosmo_val = \"/home/guerrini/sp_validation/cosmo_val/output/\"\n", - "\n", - "roots_cosmo_val = [\"SP_v1.4.6\", \"SP_v1.4.6_leak_corr\"]\n", - "\n", - "roots = [\"SP_v1.4.6_no_leak_corr_A_masked\", \"SP_v1.4.6_leak_corr_A_masked\"]\n", - "\n", - "labels = [\"SP_v1.4.6_A\", \"SP_v1.4.6_A leakage corrected\"]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ - "data_vectors = []\n", - "\n", - "for root in roots:\n", - " data_vectors.append(fits.open(data_path + root + f\"/cosmosis_{root}.fits\"))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "def cov_to_corr(cov):\n", - " \"\"\"Convert a covariance matrix to a correlation matrix.\"\"\"\n", - " d = np.sqrt(np.diag(cov))\n", - " corr = cov / np.outer(d, d)\n", - " corr[cov == 0] = 0\n", - " return corr" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "# Print the covariance matrix for each root\n", - "for i, root in enumerate(roots):\n", - " print(f\"Covariance matrix for {labels[i]}:\")\n", - " cov = data_vectors[i][\"COVMAT\"].data\n", - "\n", - " n_bins = cov.shape[0] // 4\n", - "\n", - " fig, ax = plt.subplots(figsize=(10, 8))\n", - "\n", - " im = ax.imshow(cov_to_corr(cov), vmin=-1, vmax=1, cmap=\"seismic\")\n", - " ax.set_aspect(\"equal\")\n", - " ax.set_yticks(np.array([10, 30, 50, 70]))\n", - " ax.set_yticklabels(\n", - " [\n", - " r\"$\\xi_+(\\vartheta)$\",\n", - " r\"$\\xi_-(\\vartheta)$\",\n", - " r\"$\\tau_0(\\vartheta)$\",\n", - " r\"$\\tau_2(\\vartheta)$\",\n", - " ]\n", - " )\n", - " ax.set_xticks(np.array([10, 30, 50, 70]))\n", - " ax.set_xticklabels(\n", - " [\n", - " r\"$\\xi_+(\\vartheta)$\",\n", - " r\"$\\xi_-(\\vartheta)$\",\n", - " r\"$\\tau_0(\\vartheta)$\",\n", - " r\"$\\tau_2(\\vartheta)$\",\n", - " ],\n", - " rotation=45,\n", - " )\n", - " fig.colorbar(im, ax=ax)\n", - "\n", - " plt.savefig(f\"./Plots/cov_matrix_{root}.png\", bbox_inches=\"tight\", dpi=300)\n", - " plt.show()\n", - " print(\"\\n\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5", - "metadata": {}, - "outputs": [], - "source": [ - "# Create dummy rho and tau stat handler.\n", - "\n", - "# Inference of the xi_sys parameters\n", - "sep_units = \"arcmin\"\n", - "coord_units = \"degrees\"\n", - "theta_min = 1.0\n", - "theta_max = 250\n", - "nbins = 20\n", - "\n", - "\n", - "TreeCorrConfig_xi = {\n", - " \"ra_units\": coord_units,\n", - " \"dec_units\": coord_units,\n", - " \"min_sep\": theta_min,\n", - " \"max_sep\": theta_max,\n", - " \"sep_units\": sep_units,\n", - " \"nbins\": nbins,\n", - " \"var_method\": \"jackknife\",\n", - "}\n", - "\n", - "rho_stats_handler = RhoStat(output=\".\", treecorr_config=TreeCorrConfig_xi, verbose=True)\n", - "\n", - "tau_stats_handler = TauStat(\n", - " catalogs=rho_stats_handler.catalogs,\n", - " output=\".\",\n", - " treecorr_config=TreeCorrConfig_xi,\n", - " verbose=True,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "# Create a PSFErrorFit instance\n", - "psf_fitter = PSFErrorFit(\n", - " rho_stats_handler,\n", - " tau_stats_handler,\n", - " path_cosmo_val + \"rho_tau_stats/\",\n", - " use_eta=False,\n", - ")\n", - "\n", - "\n", - "def load_matrix_and_cut(root, root_cosmo_val, type=\"rho\"):\n", - " if type == \"rho\":\n", - " cov = np.load(f\"{root}/cov_rho_{root_cosmo_val}.npy\")\n", - " nbins = cov.shape[0] // 6\n", - " cov = cov[: nbins * 3, : nbins * 3]\n", - " np.save(f\"{root}/cov_rho_{root_cosmo_val}_cut.npy\", cov)\n", - " elif type == \"tau\":\n", - " cov = np.load(f\"{root}/cov_tau_{root_cosmo_val}_th.npy\")\n", - " nbins = cov.shape[0] // 3\n", - " cov = cov[: nbins * 2, : nbins * 2]\n", - " np.save(f\"{root}/cov_tau_{root_cosmo_val}_th_cut.npy\", cov)\n", - " else:\n", - " raise ValueError(\"type must be 'rho' or 'tau'\")\n", - "\n", - "\n", - "g = plots.get_subplot_plotter(width_inch=30)\n", - "\n", - "g.settings.axes_fontsize = 30\n", - "g.settings.axes_labelsize = 30\n", - "g.settings.alpha_filled_add = 0.7\n", - "g.settings.legend_fontsize = 40\n", - "\n", - "chains = []\n", - "\n", - "# Load rho-, tau-statistics, and cov_tau from the data_vector\n", - "for i, root_cosmo_val in enumerate(roots_cosmo_val):\n", - " print(\"Sampling PSF parameters for \", labels[i])\n", - " path_rho = f\"rho_stats_{root_cosmo_val}.fits\"\n", - " path_tau = f\"tau_stats_{root_cosmo_val}.fits\"\n", - " path_cov_rho = f\"cov_rho_{root_cosmo_val}.npy\"\n", - " path_cov_tau = f\"cov_tau_{root_cosmo_val}_th.npy\"\n", - "\n", - " load_matrix_and_cut(path_cosmo_val + \"/rho_tau_stats/\", root_cosmo_val, type=\"rho\")\n", - " load_matrix_and_cut(path_cosmo_val + \"/rho_tau_stats/\", root_cosmo_val, type=\"tau\")\n", - " path_cov_rho = f\"cov_rho_{root_cosmo_val}_cut.npy\"\n", - " path_cov_tau = f\"cov_tau_{root_cosmo_val}_th_cut.npy\"\n", - "\n", - " psf_fitter.load_rho_stat(path_rho)\n", - " psf_fitter.load_tau_stat(path_tau)\n", - " psf_fitter.load_covariance(path_cov_rho, cov_type=\"rho\")\n", - " psf_fitter.load_covariance(path_cov_tau, cov_type=\"tau\")\n", - " samples_lq, _, _ = psf_fitter.get_least_squares_params_samples(\n", - " npatch=None, apply_debias=False\n", - " )\n", - "\n", - " samples_gd = MCSamples(\n", - " samples=samples_lq, names=[r\"\\alpha\", r\"\\beta\"], labels=[r\"\\alpha\", r\"\\beta\"]\n", - " )\n", - "\n", - " chains.append(samples_gd)\n", - "\n", - "g.triangle_plot(chains, filled=True, legend_labels=labels, legend_loc=\"upper right\")\n", - "\n", - "plt.savefig(\"./Plots/psf_leakage_params.png\", bbox_inches=\"tight\", dpi=300)\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "sp_validation_3.11", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.0" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cosmo_inference/pipeline.sh b/cosmo_inference/pipeline.sh deleted file mode 100755 index f853d511..00000000 --- a/cosmo_inference/pipeline.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash - -# Transform long options to short ones -for arg in "$@"; do - shift - case "$arg" in - '--help') set -- "$@" '-h' ;; - '--pcf') set -- "$@" '-p' ;; - '--covmat') set -- "$@" '-c' ;; - '--inference') set -- "$@" '-i' ;; - '--mcmc_process') set -- "$@" '-m' ;; - *) set -- "$@" "$arg" ;; - esac -done - -# Parse short options -OPTIND=1 -while getopts "hpcim" opt -do - case "$opt" in - 'h') - echo "Please input a flag: --help, --pcf, --covmat, --inference or --mcmc_process "; - exit 0 - ;; - 'p') - echo "Running cosmo_val.py to calculate 2 point correlation functions"; - python cosmo_val/cosmo_val.py - ;; - 'c') - read -p 'COVARIANCE FILE: ' covmat_file; - read -p 'OUTPUT STUB (without extension): ' output_stub; - echo "Processing covariance matrix"; - python scripts/cosmocov_process.py $covmat_file $output_stub - ;; - 'i') - read -p 'XI ROOT: ' xi_root; - read -p 'TAU ROOT: ' tau_root; - read -p 'COSMOSIS ROOT: ' cosmosis_root; - read -p 'COSMO_VAL OUTPUT FOLDER: ' output_folder; - read -p 'NZ FILE:' nz_file; - read -p 'OUTPUT MCMC CHAIN FOLDER: ' data; - read -p 'USE PSEUDO_CELL? (y/n): ' pseudo_cell; - - if [ "${pseudo_cell}" == "y" ]; then - echo "Using pseudo cell" - - out_file="data/${root}/cosmosis_${root}_cell.fits" - - # Create the folder if it does not exist - if [ ! -d "data/$root" ]; then - mkdir -p "data/$root" - echo "Directory 'data/$root' created." - else - echo "Directory 'data/$root' already exists." - fi - - python scripts/cosmosis_fitting.py $root $output_folder $nz_file $pseudo_cell $out_file - else - - read -p 'USE RHO/TAU_STATS? (y/n): ' rhotau_stats; - echo $rhotau_stats - read -p 'COV_XI MAT TXT FILE:' covmat; - - out_file="data/${root}/cosmosis_${root}.fits"; - - # Create the folder if it does not exist - if [ ! -d "data/$root" ]; then - mkdir -p "data/$root" - echo "Directory 'data/$root' created." - else - echo "Directory 'data/$root' already exists." - fi - - #LG: add check if xi_plus/xi_minus fits file exists - python scripts/cosmosis_fitting.py $root $output_folder $nz_file $pseudo_cell $out_file $covmat $rhotau_stats; - - fi - - if [ "${pseudo_cell}" == "y" ]; then - output_ini_file="cosmosis_config/cosmosis_pipeline_${root}_cell.ini" - cp cosmosis_config/cosmosis_pipeline_A_ia_cell.ini $output_ini_file - else - output_ini_file="cosmosis_config/cosmosis_pipeline_${root}.ini" - if [ "${rhotau_stats}" == "y" ]; then - cp cosmosis_config/cosmosis_pipeline_A_psf.ini $output_ini_file; - else - cp cosmosis_config/cosmosis_pipeline_A_ia.ini $output_ini_file; - fi - fi - - sed -i "/^\[DEFAULT\]/a\SCRATCH = ${data}" $output_ini_file; - sed -i "/^\[DEFAULT\]/a\FITS_FILE = ${out_file}" $output_ini_file; - if [ "${pseudo_cell}" == "y" ]; then - sed -i "/^\[output\]/a\filename = %(SCRATCH)s/${root}_cell/samples_${root}_cell.txt" $output_ini_file; - sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_ia.ini" $output_ini_file; - sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors.ini" $output_ini_file; - sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" $output_ini_file; - sed -i "/^\[2pt_like]/a\data_sets=CELL_EE" $output_ini_file; - sed -i "/^\[polychord\]/a\polychord_outfile_root = ${root}_cell" $output_ini_file; - sed -i "/^\[test\]/a\save_dir = %(SCRATCH)s/best_fit/${root}_cell" $output_ini_file; - else - sed -i "/^\[output\]/a\filename = %(SCRATCH)s/${root}/samples_${root}.txt" $output_ini_file; - if [ "${rhotau_stats}" == "y" ]; then - sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_psf.ini" $output_ini_file; - sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors_psf.ini" $output_ini_file; - sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py" $output_ini_file; - sed -i "/^\[2pt_like]/a\data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS" $output_ini_file; - sed -i "/^\[2pt_like]/a\add_xi_sys=T" $output_ini_file; - else - sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_ia.ini" $output_ini_file; - sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors.ini" $output_ini_file; - sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" $output_ini_file; - sed -i "/^\[2pt_like]/a\data_sets=XI_PLUS XI_MINUS" $output_ini_file; - fi - sed -i "/^\[polychord\]/a\polychord_outfile_root = ${root}" $output_ini_file; - sed -i "/^\[test\]/a\save_dir = %(SCRATCH)s/best_fit/${root}" $output_ini_file; - fi - echo "Prepared CosmoSIS configuration file in $output_ini_file"; - echo "You can now run the inference with the command: cosmosis $output_ini_file" - ;; - 'm') - # LG: also convert this into a script to directly output contour plots - echo "Run the cosmo_inference/notebooks/MCMC.ipynb notebook to analyse your chains" - ;; - '?') - print_usage >&2; - exit 1 - ;; - esac -done -shift $(expr $OPTIND - 1) # remove options from positional parameters \ No newline at end of file diff --git a/cosmo_inference/scripts/2pt_like_xi_sys.py b/cosmo_inference/scripts/2pt_like_xi_sys.py deleted file mode 100644 index 93f11bd3..00000000 --- a/cosmo_inference/scripts/2pt_like_xi_sys.py +++ /dev/null @@ -1,614 +0,0 @@ -import gaussian_covariance -import numpy as np -import twopoint -from astropy.io import fits -from cosmosis.datablock import SectionOptions, names -from cosmosis.gaussian_likelihood import GaussianLikelihood -from scipy.interpolate import interp1d -from spec_tools import TheorySpectrum -from twopoint_cosmosis import theory_names, type_table - -default_array = np.repeat(-1.0, 99) - - -# To copy in cosmosis-standard-library/likelihood -def is_default(x): - return len(x) == len(default_array) and (x == default_array).all() - - -def convert_nz_steradian(n): - return n * (41253.0 * 60.0 * 60.0) / (4 * np.pi) - - -class TwoPointLikelihood(GaussianLikelihood): - # This is a sub-class of the class GaussianLikelihood - # which can be found in the file ${COSMOSIS_SRC_DIR}/cosmosis/gaussian_likelihood.py - # That super-class implements the generic behaviour that all Gaussian likelihoods - # follow - the basic form of the likelihoods, inverting covariance matrices, saving - # results, etc. This sub-clas does the parts that are specific to this 2-pt - # likelihood - loading data from a file, getting the specific theory prediction - # to which to compare it, etc. - like_name = "2pt" - - def __init__(self, options): - # We may decide to use an analytic gaussian covariance - # in that case we won't load the covmat. - self.gaussian_covariance = options.get_bool("gaussian_covariance", False) - if self.gaussian_covariance: - self.constant_covariance = False - - self.moped = options.get_string("moped", default="") - - super(TwoPointLikelihood, self).__init__(options) - - self.raw_data_x, self.raw_data_y = self.build_data() - - if self.moped: - print( - "Using compressed data from MOPED algorithm: {} data points".format( - len(self.moped_data) - ) - ) - if self.sellentin: - raise ValueError( - "Sellentin mode is incompatible with Moped mode in 2pt like" - ) - - def build_data(self): - filename = self.options.get_string("data_file") - - # Suffixes to added on to two point data from e.g. different experiments - suffix_string = self.options.get_string("suffixes", default="") - if suffix_string == "": - # If there are no suffixes provided, then we create a list of a single empty suffix - suffixes = [""] - else: - suffixes_temp = suffix_string.split() - suffixes = [] - for suffix in suffixes_temp: - if suffix.lower() == "none": - suffixes.append("") - else: - suffixes.append("_" + suffix) - self.suffixes = suffixes - - if self.gaussian_covariance: - covmat_name = None - area = self.options.get_double("survey_area") # in square degrees - self.sky_area = area * (np.pi * np.pi) / (180 * 180) - - def get_arr(x): - if self.options.has_value(x): - a = self.options[x] - if not isinstance(a, np.ndarray): - a = [a] - else: - a = default_array - return a - - self.number_density_shear_bin = get_arr("number_density_shear_bin") - self.number_density_lss_bin = get_arr("number_density_lss_bin") - self.sigma_e_bin = get_arr("sigma_e_bin") - - else: - covmat_name = self.options.get_string("covmat_name", "COVMAT") - - # This is the main work - read data in from the file - self.two_point_data = twopoint.TwoPointFile.from_fits(filename, covmat_name) - - # Potentially cut out lines. For some reason one version of - # this file used zeros to mark masked values. - if self.options.get_bool("cut_zeros", default=False): - print("Removing 2-point values with value=0.0") - self.two_point_data.mask_bad(0.0) - - if self.options.get_bool("cut_cross", default=False): - print("Removing 2-point values from cross-bins") - self.two_point_data.mask_cross() - - # All the names of two-points measurements that were found in the data - # file - all_names = [spectrum.name for spectrum in self.two_point_data.spectra] - - # We may not want to use all the likelihoods in the file. - # We can set an option to only use some of them - data_sets = self.options.get_string("data_sets", default="all") - if data_sets != "all": - data_sets = data_sets.split() - self.two_point_data.choose_data_sets(data_sets) - - # The ones we actually used. - self.used_names = [spectrum.name for spectrum in self.two_point_data.spectra] - - # Check for scale cuts. In general, this is a minimum and maximum angle for - # each spectrum, for each redshift bin combination. Which is clearly a massive pain... - # but what can you do? - - scale_cuts = {} - for name in self.used_names: - s = self.two_point_data.get_spectrum(name) - for b1, b2 in s.bin_pairs: - option_name = "angle_range_{}_{}_{}".format(name, b1, b2) - if self.options.has_value(option_name): - r = self.options.get_double_array_1d(option_name) - scale_cuts[(name, b1, b2)] = r - - # Now check for completely cut bins - # example: - # cut_wtheta = 1,2 1,3 2,3 - bin_cuts = [] - for name in self.used_names: - s = self.two_point_data.get_spectrum(name) - option_name = "cut_{}".format(name) - if self.options.has_value(option_name): - cuts = self.options[option_name].split() - cuts = [eval(cut) for cut in cuts] - for b1, b2 in cuts: - bin_cuts.append((name, b1, b2)) - - if scale_cuts or bin_cuts: - self.two_point_data.mask_scales(scale_cuts, bin_cuts) - else: - print("No scale cuts mentioned in ini file.") - - # Info on which likelihoods we do and do not use - print("Found these data sets in the file:") - total_data_points = 0 - final_names = [spectrum.name for spectrum in self.two_point_data.spectra] - for name in all_names: - if name in final_names: - data_points = len(self.two_point_data.get_spectrum(name)) - else: - data_points = 0 - if name in self.used_names: - print( - " - {} {} data points after cuts {}".format( - name, data_points, " [using in likelihood]" - ) - ) - total_data_points += data_points - else: - print( - " - {} {} data points after cuts {}".format( - name, data_points, " [not using in likelihood]" - ) - ) - print("Total data points used = {}".format(total_data_points)) - - # Convert all units to radians. The units in cosmosis are all - # in radians, so this is the easiest way to compare them. - for spectrum in self.two_point_data.spectra: - if spectrum.is_real_space(): - spectrum.convert_angular_units("rad") - # if self.options.get_bool("print physical scale",False): - # section,_,_=theory_names(spectrum) - # chi_peak = - # for ang in spectrum.angle: - - # build up the data vector from all the separate vectors. - # Just concatenation - data_vector = np.concatenate( - [spectrum.value for spectrum in self.two_point_data.spectra] - ) - - # Make sure - if len(data_vector) == 0: - raise ValueError( - "No data was chosen to be used from 2-point data file {0}. It was either not selectedin data_sets or cut out".format( - filename - ) - ) - - if self.moped: - data_file = fits.open(filename) - self.moped_data = data_file["MOPED-DATA-{}".format(self.moped)].data[ - "moped" - ] - self.moped_transform = data_file[ - "MOPED-TRANSFORM-{}".format(self.moped) - ].data - data_file.close() - - return None, self.moped_data - - # The x data is not especially useful here, so return None. - # We will access the self.two_point_data directly later to - # determine ell/theta values - return None, data_vector - - def build_covariance(self): - - C = np.array(self.two_point_data.covmat) - r = self.options.get_int("covariance_realizations", default=-1) - self.sellentin = self.options.get_bool("sellentin", default=False) - - if self.moped: - return np.identity(len(self.moped_data)) - - if self.sellentin: - if not self.constant_covariance: - print() - print("You asked for the Sellentin-Heavens correction to be applied") - print("But also asked for a non-constant (maybe Gaussian?) covariance") - print("matrix. I think that probably suggests you have made a mistake") - print("somewhere unless you have thought about this quite carefully.") - print() - if r < 0: - print() - print("ERROR: You asked for the Sellentin-Heavens corrections") - print( - "by setting sellentin=T, but you did not set covariance_realizations" - ) - print("If you want covariance_realizations=infinity you can use 0") - print( - "(unlikely, but it's also possible you were super-perverse and set it negative?)" - ) - print() - raise ValueError( - "Please set covariance_realizations for 2pt like. See message above." - ) - elif r == 0: - print() - print("NOTE: You asked for the Sellentin-Heavens corrections") - print("but set covariance_realizations=0. I am assuming you want") - print( - "the limit of an infinite number of realizations, so we will just go back" - ) - print("to the original Gaussian model") - print() - self.sellentin = False - else: - # use proper correction - self.covariance_realizations = r - print() - print( - "You set sellentin=T so I will apply the Sellentin-Heavens correction" - ) - print("for a covariance matrix estimated from Monte-Carlo simulations") - print("(you told us it was {} simulations in the ini file)".format(r)) - print( - "This analytic marginalization converts the Gaussian distribution" - ) - print("to a multivariate student's t distribution instead.") - print() - - elif r > 0: - # Just regular increase in covariance size, no Sellentin change. - p = C.shape[0] - # This x is the inverse of the alpha used in the old code - # because that applied to the weight matrix not the covariance - x = (r - 1.0) / (r - p - 2.0) - C = C * x - print() - print( - "You set covariance_realizations={} in the 2pt likelihood parameter file".format( - r - ) - ) - print( - "So I will apply the Anderson-Hartlap correction to the covariance matrix" - ) - print("The covariance matrix is nxn = {}x{}".format(p, p)) - print( - "So the correction scales the covariance matrix by (r - 1) / (r - n - 2) = {}".format( - x - ) - ) - print() - return C - - def extract_theory_points(self, block): - theory = [] - # We may want to save these splines for the covariance matrix later - self.theory_splines = {} - - # We have a collection of data vectors, one for each spectrum - # that we include. We concatenate them all into one long vector, - # so we do the same for our theory data so that they match - - # We will also save angles and bin indices for plotting convenience, - # although these are not actually used in the likelihood - angle = [] - bin1 = [] - bin2 = [] - - # Get appropriate suffixes - # If only a single suffix is provided, assume this applies to all data sets - if len(self.suffixes) == 1: - suffixes = np.tile(self.suffixes[0], len(self.two_point_data.spectra)) - elif len(self.suffixes) > 1 and len(self.suffixes) == len( - self.two_point_data.spectra - ): - suffixes = self.suffixes - else: - raise ValueError( - "The number of suffixes supplied does not match the number of two point spectra." - ) - - # Now we actually loop through our data sets - for ii, spectrum in enumerate(self.two_point_data.spectra): - theory_vector, angle_vector, bin1_vector, bin2_vector = ( - self.extract_spectrum_prediction(block, spectrum, suffixes[ii]) - ) - theory.append(theory_vector) - angle.append(angle_vector) - bin1.append(bin1_vector) - bin2.append(bin2_vector) - # dataset_name.append(np.repeat(spectrum.name, len(bin1_vector))) - - # We also collect the ell or theta values. - # The gaussian likelihood code itself is not expecting these, - # so we just save them here for convenience. - angle = np.concatenate(angle) - bin1 = np.concatenate(bin1) - bin2 = np.concatenate(bin2) - # dataset_name = np.concatenate(dataset_name) - block[names.data_vector, self.like_name + "_angle"] = angle - block[names.data_vector, self.like_name + "_bin1"] = bin1 - block[names.data_vector, self.like_name + "_bin2"] = bin2 - # block[names.data_vector, self.like_name+"_name"] = dataset_name - - # the thing it does want is the theory vector, for comparison with - # the data vector - theory = np.concatenate(theory) - - if self.moped: - return np.dot(self.moped_transform, theory) - - return theory - - def do_likelihood(self, block): - # Run the - super(TwoPointLikelihood, self).do_likelihood(block) - - if self.sellentin: - # The Sellentin-Heavens correction from arxiv 1511.05969 - # accounts for a finite number of Monte-Carlo realizations - # being used to estimate the covariance matrix. - - # Note that this invalidates the saved simulation used for - # the ABC sampler. I can't think of a better way of doing this - # than overwriting the whole things with NaNs - that will at - # least make clear there is a problem somewhere and not - # yield misleading results. - block[names.data_vector, self.like_name + "_simulation"] = ( - np.nan * block[names.data_vector, self.like_name + "_simulation"] - ) - - # It changes the Likelihood from Gaussian to a multivariate - # student's t distribution. Here we will have to do a little - # hack and overwrite the stuff that the original Gaussian - # method did above - N = self.covariance_realizations - chi2 = block[names.data_vector, self.like_name + "_CHI2"] - - # We might be using a cosmologically varying - # covariance matrix, though I'm not sure what that would mean. - # There is a warning about this above. - if self.constant_covariance: - log_det = 0.0 - else: - log_det = block[names.data_vector, self.like_name + "_LOG_DET"] - - like = -0.5 * log_det - 0.5 * N * np.log(1 + chi2 / (N - 1.0)) - - # overwrite the log-likelihood - block[names.likelihoods, self.like_name + "_LIKE"] = like - - # Should suffix be made into a keyword? - def extract_spectrum_prediction(self, block, spectrum, suffix): - - # We may need theory predictions for multiple different - # types of spectra: e.g. shear-shear, pos-pos, shear-pos. - # So first we find out from the spectrum where in the data - # block we expect to find these - mapping spectrum types - # to block names - section, x_name, y_name = theory_names(spectrum) - - # To handle multiple different data sets we allow a suffix - # to be applied to the section names, so that we can look up - # e.g. "shear_cl_des" instead of just "shear_cl". - section += suffix - - # Initialize TheorySpectrum class from block - bin_pairs = spectrum.get_bin_pairs() - theory_spec = TheorySpectrum.from_block(block, section, bin_pairs=bin_pairs) - - # If the theory spectrum has been bin-averaged then we expect the - # data to be so also. We check this by ensuring that angle_min is specified - # Based on this, we also generate the angle argument passed to the spectrum - # differently. The bin-averaged version expects a tuple angle_min and angle_max, - # whereas the interpolated version just wants a single angle. - if theory_spec.is_bin_averaged: - if spectrum.angle_min is None: - raise ValueError( - "Your theory pipeline produced angle-binnned values, but your data it not binned." - ) - angles = list(zip(spectrum.angle_min, spectrum.angle_max)) - else: - angles = spectrum.angle - - # We store the nominal mid-points for plotting later on, etc. - angle_mids = spectrum.angle - - # This is a bit of a hack, but later on if we are making a covariance - # we need all the splines, so pull them out here. - bin_splines = {} - - # We build up these vectors from all the data points. - # Only the theory vector is needed for the likelihood - the others - # are for convenience, debugging, etc. - theory_vector = [] - angle_vector = [] - bin1_vector = [] - bin2_vector = [] - - for b1, b2, angle, angle_mid in zip( - spectrum.bin1, spectrum.bin2, angles, angle_mids - ): - # The extra object will either be a spline (for interpolated spectra) - # or theta mid-point values (for bin-averaged ones, e.g. for plotting) - theory, extra = theory_spec.get_spectrum_value(b1, b2, angle) - - # We can only record the splines for non-bin-averaged values - if not theory_spec.is_bin_averaged: - bin_splines[y_name.format(b1, b2)] = extra - - # Build up the vector - we make this into an array later - theory_vector.append(theory) - angle_vector.append(angle_mid) - bin1_vector.append(b1) - bin2_vector.append(b2) - - self.theory_splines[section] = bin_splines - - # Return the whole collection as an array - theory_vector = np.array(theory_vector) - - # For convenience we also save the angle vector (ell or theta) - # and bin indices - angle_vector = np.array(angle_vector) - bin1_vector = np.array(bin1_vector, dtype=int) - bin2_vector = np.array(bin2_vector, dtype=int) - - return theory_vector, angle_vector, bin1_vector, bin2_vector - - def extract_covariance(self, block): - assert self.gaussian_covariance, ( - "Set constant_covariance=F but somehow not with Gaussian covariance. Internal error - please open an issue on the cosmosis site." - ) - - C = [] - # s and t index the spectra that we have. e.g. s or t=1 might be the full set of - # shear-shear measuremnts - for s, AB in enumerate(self.two_point_data.spectra[:]): - M = [] - for t, CD in enumerate(self.two_point_data.spectra[:]): - print( - "Looking at covariance between {} and {} (s={}, t={})".format( - AB.name, CD.name, s, t - ) - ) - # We only calculate the upper triangular. - # Get the lower triangular here. We have to - # transpose it compared to the upper one. - if s > t: - MI = C[t][s].T - else: - MI = gaussian_covariance.compute_gaussian_covariance( - self.sky_area, self._lookup_theory_cl, block, AB, CD - ) - M.append(MI) - C.append(M) - - # C is now a list of lists of 2D arrays. - # Now turn C into a big 2D array by stacking - # the arrays - C = np.vstack([np.hstack(CI) for CI in C]) - - return C - - def _lookup_theory_cl(self, block, A, B, i, j, ell): - """ - This is a helper function for the compute_gaussian_covariance code. - It looks up the theory value of C^{ij}_{AB}(ell) in the - """ - # We have already saved splines into the theory space earlier - # when constructing the theory vector. - # So now we just need to look those up again, using the same - # code we use in the twopoint library. - section, ell_name, value_name = type_table[A, B] - assert ell_name == "ell", ( - "Gaussian covariances are currently only written for C_ell, not other 2pt functions" - ) - d = self.theory_splines[section] - - # We save the splines with these names when we extract the theory vector - name_ij = value_name.format(i, j) - name_ji = value_name.format(j, i) - - # Hopefully we already have the theory spline extracted - if name_ij in d: - spline = d[name_ij] - # For symmetric spectra (not just auto-correlations, but any thing like C_EE or C_NN where - # we cross-correlate something with itself) we can use ji for ij as it is the same. This is - # not true for cross spectra - elif name_ji in d and (A == B): - spline = d[name_ji] - else: - # It's possible too that we need something for the covariance that we didn't need for the - # data vector - for example to got the covariance between C^EE and C^NN we need C^NE even - # if we don't have any actual measurements of NE. In that case we have to g - angle_theory = block[section, ell_name] - if block.has_value(section, name_ij): - theory = block[section, name_ij] - # The same symmetry argument as above applies - elif block.has_value(section, name_ji) and A == B: - theory = block[section, name_ji] - else: - raise ValueError( - "Could not find theory prediction {} in section {}".format( - value_name.format(i, j), section - ) - ) - - spline = interp1d(angle_theory, theory) - # Finally cache this so we don't have to do this again. - d[name_ij] = spline - - obs_cl = spline(ell) - - # For shear-shear the noise component is sigma^2 / number_density_bin - # and for position-position it is just 1/number_density_bin - if ( - (A == B) - and (A == twopoint.Types.galaxy_shear_emode_fourier.name) - and (i == j) - ): - if ( - i > len(self.number_density_shear_bin) - or i > len(self.sigma_e_bin) - or is_default(self.sigma_e_bin) - or is_default(self.number_density_shear_bin) - ): - raise ValueError("Not enough number density bins for shear specified") - noise = self.sigma_e_bin[i - 1] ** 2 / convert_nz_steradian( - self.number_density_shear_bin[i - 1] - ) - obs_cl += noise - if (A == B) and (A == twopoint.Types.galaxy_position_fourier.name) and (i == j): - if i > len(self.number_density_lss_bin) or is_default( - self.number_density_lss_bin - ): - raise ValueError("Not enough number density bins for lss specified") - noise = 1.0 / convert_nz_steradian(self.number_density_lss_bin[i - 1]) - obs_cl += noise - - return obs_cl - - def update_xi_w_sys(self, block): - self.data_y = self.raw_data_y + block["xi_sys", "xi_sys_vec"] - - @classmethod - def build_module(cls): - - def setup(options): - options = SectionOptions(options) - likelihoodCalculator = cls(options) - return likelihoodCalculator - - def execute(block, config): - likelihoodCalculator = config - likelihoodCalculator.update_xi_w_sys(block) - # print(likelihoodCalculator.data_y) - likelihoodCalculator.do_likelihood(block) - return 0 - - def cleanup(config): - likelihoodCalculator = config - likelihoodCalculator.cleanup() - - return setup, execute, cleanup - - -setup, execute, cleanup = TwoPointLikelihood.build_module() diff --git a/cosmo_inference/scripts/cosmocov_process.py b/cosmo_inference/scripts/cosmocov_process.py deleted file mode 100644 index c2c00996..00000000 --- a/cosmo_inference/scripts/cosmocov_process.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -import sys - -import matplotlib.pyplot as plt -import numpy as np - - -def get_cov(filename): - - data = np.loadtxt(filename) - ndata = int(np.max(data[:, 0])) + 1 - - print("Dimension of cov: %dx%d" % (ndata, ndata)) - - cov_g = np.zeros((ndata, ndata)) - cov_ng = np.zeros((ndata, ndata)) - for i in range(0, data.shape[0]): - cov_g[int(data[i, 0]), int(data[i, 1])] = data[i, 8] - cov_g[int(data[i, 1]), int(data[i, 0])] = data[i, 8] - cov_ng[int(data[i, 0]), int(data[i, 1])] = data[i, 9] - cov_ng[int(data[i, 1]), int(data[i, 0])] = data[i, 9] - - return cov_g, cov_ng, ndata - - -if __name__ == "__main__": - if len(sys.argv) != 3: - print("Usage: python cosmocov_process.py ") - sys.exit(1) - - covfile = sys.argv[1] - output_base = sys.argv[2] - - c_g, c_ng, ndata = get_cov(covfile) - - cov = c_ng + c_g - cov_g = c_g - - b = np.sort(np.linalg.eigvals(cov)) - print("min+max eigenvalues cov: %e, %e" % (np.min(b), np.max(b))) - if np.min(b) <= 0.0: - print("non-positive eigenvalue encountered! Covariance Invalid!") - exit() - - print("Covariance is positive definite!") - - np.savetxt(str(output_base) + ".txt", cov) - print("covmat saved as %s" % (str(output_base) + ".txt")) - - np.savetxt(str(output_base) + "_g.txt", cov_g) - print("Gaussian covmat saved as %s" % (str(output_base) + "_g.txt")) - - cmap = "seismic" - - pp_norm = np.zeros((ndata, ndata)) - for i in range(ndata): - for j in range(ndata): - pp_norm[i][j] = cov[i][j] / np.sqrt(cov[i][i] * cov[j][j]) - - print("Plotting correlation matrix ...") - - plot_path = str(output_base) + "_plot.pdf" - fig = plt.figure() - ax = fig.add_subplot(1, 1, 1) - extent = (0, ndata, ndata, 0) - im3 = ax.imshow(pp_norm, cmap=cmap, vmin=-1, vmax=1, extent=extent) - - plt.axvline(x=int(ndata / 2), color="black", linewidth=1.0) - plt.axhline(y=int(ndata / 2), color="black", linewidth=1.0) - - fig.colorbar(im3, orientation="vertical") - - ax.text(int(ndata / 4), ndata + 5, r"$\xi_+^{ij}(\theta)$", fontsize=12) - ax.text(3 * int(ndata / 4), ndata + 5, r"$\xi_-^{ij}(\theta)$", fontsize=12) - ax.text(-9, int(ndata / 4), r"$\xi_+^{ij}(\theta)$", fontsize=12) - ax.text(-9, 3 * int(ndata / 4), r"$\xi_-^{ij}(\theta)$", fontsize=12) - - plt.savefig(plot_path, dpi=2000) - plt.close() - print("Plot saved as %s" % (plot_path)) diff --git a/cosmo_inference/scripts/masking.py b/cosmo_inference/scripts/masking.py deleted file mode 100644 index ba2f3c4c..00000000 --- a/cosmo_inference/scripts/masking.py +++ /dev/null @@ -1,319 +0,0 @@ -import argparse -import os -from multiprocessing import Pool, cpu_count -from pathlib import Path - -import h5py -import healpy as hp -import numpy as np -import yaml - -# ------------------------- -# Spatially-structured cuts: these define the survey footprint. -# All other cuts (FLAGS, mag, SNR, shape measurement, PSF ellipticity, -# relative size) are per-galaxy quality cuts that should NOT affect -# the footprint definition. -SPATIAL_CUTS = { - "overlap", - "IMAFLAGS_ISO", - "N_EPOCH", - "4_Stars", - "8_Manual", - "64_r", - "1024_Maximask", - "npoint3", - "1_Faint_star_halos", - "2_Bright_star_halos", -} - -# ------------------------- -# Masking logic - - -def apply_condition(array, kind, value): - """ - Apply a logical condition to a NumPy array and return a boolean mask, based - on the "kind" key in the mask config YAML file. - """ - if kind == "equal": - return array == value - elif kind == "not_equal": - return array != value - elif kind == "greater_equal": - return array >= value - elif kind == "greater": - return array > value - elif kind == "less_equal": - return array <= value - elif kind == "less": - return array < value - elif kind == "range": - return (array >= value[0]) & (array <= value[1]) - else: - raise ValueError(f"Unknown kind: {kind}") - - -def apply_masks(data, data_ext, mask_config, footprint_only=False): - """ - Construct a boolean mask selecting galaxies that satisfy all - masking criteria defined in the YAML configuration file. - - Parameters - ---------- - data : numpy.ndarray or structured array - Slice of the HDF5 "data" group containing per-object - measurements (e.g. FLAGS, mag, NGMIX quantities). - - data_ext : numpy.ndarray or structured array - Slice of the HDF5 "data_ext" group containing external or - post-processing flags (e.g. star masks, footprint flags). - - mask_config : dict - Dictionary parsed from the YAML mask configuration file. - Expected structure: - - mask_config["dat"] : list of cuts applied to `data` - - mask_config["dat_ext"] : list of cuts applied to `data_ext` - - mask_config["metacal"] : derived-quantity parameters - (e.g. relative size limits) - - footprint_only : bool, optional - If True, only apply spatially-structured cuts (those in - SPATIAL_CUTS). Skips per-galaxy quality cuts (FLAGS, mag, - SNR, shape measurement, PSF ellipticity, relative size). - Used to define a consistent footprint from the comprehensive - catalog. Default is False. - - Returns - ------- - numpy.ndarray (bool) - Boolean array of length equal to the input data slice. - True indicates the object passes all cuts (kept), - False indicates the object is masked (removed). - """ - - # Initialize mask - mask = np.ones(len(data), dtype=bool) - - # --- dat group --- - for cut in mask_config.get("dat", []): - col = cut["col_name"] - if footprint_only and col not in SPATIAL_CUTS: - continue - kind = cut["kind"] - value = cut["value"] - - mask &= apply_condition(data[col], kind, value) - - # --- dat_ext group --- - for cut in mask_config.get("dat_ext", []): - col = cut["col_name"] - if footprint_only and col not in SPATIAL_CUTS: - continue - kind = cut["kind"] - value = cut["value"] - - mask &= apply_condition(data_ext[col], kind, value) - - # --- metacal relative size (skip for footprint-only) --- - if not footprint_only: - rel_size = np.divide( - data["NGMIX_T_NOSHEAR"], - data["NGMIX_T_PSF_RECONV_NOSHEAR"], - out=np.zeros_like(data["NGMIX_T_NOSHEAR"]), - where=(data["NGMIX_T_PSF_RECONV_NOSHEAR"] > 0), - ) - - rel_min = mask_config["metacal"]["gal_rel_size_min"] - rel_max = mask_config["metacal"]["gal_rel_size_max"] - - mask &= (rel_size >= rel_min) & (rel_size <= rel_max) - - return mask - - -# ------------------------- -# Process one chunk -def process_chunk(args): - """ - Process a chunk of the HDF5 catalogue and return the unique - HEALPix pixels containing unmasked galaxies,to be executed in - parallel. It reads a slice of the catalogue, applies - the defined masking criteria, converts the sky positions - (RA, Dec) of retained galaxies into HEALPix pixel indices, - and returns the unique pixel indices for that chunk. - - Parameters - ---------- - args : tuple - Tuple containing: - - start : int - Starting row index of the chunk (inclusive). - - stop : int - Ending row index of the chunk (exclusive). - - filename : str - Path to the input HDF5 catalogue. - - nside : int - HEALPix NSIDE parameter defining map resolution. - - mask_config : dict - Parsed YAML mask configuration. - - Returns - ------- - numpy.ndarray - Array of unique HEALPix pixel indices (int) corresponding - to sky locations of galaxies that pass all mask cuts in - this chunk. - """ - - start, stop, filename, nside, mask_config, footprint_only = args - with h5py.File(filename, "r") as f: - data = f["data"][start:stop] - data_ext = f["data_ext"][start:stop] - - mask = apply_masks(data, data_ext, mask_config, footprint_only=footprint_only) - - ra = data["RA"][mask] - dec = data["Dec"][mask] - - theta = np.radians(90.0 - dec) # colatitude - phi = np.radians(ra) # longitude - - pix = hp.ang2pix(nside, theta, phi) - - return np.unique(pix) - - -# ------------------------- -# Build mask map in parallel -def build_mask_map_hdf5( - filename, mask_config, nside, chunk_size=1_000_000, footprint_only=False -): - """ - Build a binary HEALPix mask map from an HDF5 galaxy catalogue. - - The catalogue is processed in chunks to limit memory usage. - - Parameters - ---------- - filename : str - Path to the input HDF5 catalogue containing "data" and - "data_ext" groups - mask_config : dict - Dictionary parsed from the YAML mask configuration file - nside : int - HEALPix NSIDE parameter defining the resolution of the - output map. - chunk_size : int, optional - Number of catalogue rows to process per chunk. - Default is 1,000,000. - footprint_only : bool, optional - If True, only apply spatially-structured cuts. - - Returns - ------- - numpy.ndarray - One-dimensional HEALPix map (dtype uint8) of length - hp.nside2npix(nside), where: - - 1 indicates at least one unmasked galaxy falls - in that pixel, - - 0 indicates no retained galaxies. - """ - with h5py.File(filename, "r") as f: - nrows = f["data"].shape[0] - - chunks = [ - (i, min(i + chunk_size, nrows), filename, nside, mask_config, footprint_only) - for i in range(0, nrows, chunk_size) - ] - - mask_map = np.zeros(hp.nside2npix(nside), dtype=np.uint8) - - with Pool(cpu_count()) as pool: - for pix_indices in pool.imap_unordered(process_chunk, chunks): - mask_map[pix_indices] = 1 - - return mask_map - - -############################################################################################################ -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Build HEALPix mask from HDF5 catalog") - parser.add_argument("nside", type=int, help="HEALPix NSIDE parameter") - parser.add_argument("--config", required=True, help="Path to mask config YAML") - parser.add_argument( - "--output-prefix", - required=True, - help="Output file prefix (e.g. 'footprint' or 'footprint_starhalo')", - ) - parser.add_argument( - "--footprint-only", - action="store_true", - help="Only apply spatially-structured cuts (for footprint definition)", - ) - parser.add_argument( - "--output-dir", - default=None, - help="Output directory (default: data/mask/ relative to script)", - ) - args = parser.parse_args() - - nside = args.nside - curr_dir = Path(os.path.dirname(os.path.abspath(__file__))) - - if args.output_dir: - out_dir = Path(args.output_dir) - else: - out_dir = curr_dir.parent / "data" / "mask" - out_dir.mkdir(parents=True, exist_ok=True) - - with open(args.config, "r") as f: - mask_config = yaml.safe_load(f) - - filename = f"/n17data/UNIONS/WL/v1.4.x/v1.4.5/{mask_config['params']['input_path']}" - prefix = args.output_prefix - - if args.footprint_only: - print(f"Footprint-only mode: applying only spatial cuts {SPATIAL_CUTS}") - - # Build mask map from comprehensive catalogue - mask_map = build_mask_map_hdf5( - filename, - mask_config, - nside, - chunk_size=500_000, - footprint_only=args.footprint_only, - ) - - # Get survey area after masking - npix = hp.nside2npix(nside) - pix_area_sr = 4 * np.pi / npix - pix_area_deg2 = (180 / np.pi) ** 2 * pix_area_sr - n_obs = mask_map.sum() - f_sky_obs = n_obs / npix - area_obs_deg2 = n_obs * pix_area_deg2 - print(f"Kept area = {area_obs_deg2:.2f} deg^2\n") - - # Compute Cls of the mask map - cl_mask = hp.anafast(mask_map, lmax=3 * nside - 1) - ells = np.arange(len(cl_mask)) - - # Save mask map and Cls - map_path = out_dir / f"mask_map_{prefix}_nside_{nside}.fits" - cls_path = out_dir / f"mask_cls_{prefix}_nside_{nside}.npz" - hp.write_map(map_path, mask_map, overwrite=True) - np.savez(cls_path, ells=ells, cl_mask=cl_mask) - - print(f"Mask map saved to {map_path}") - print(f"Mask Cls saved to {cls_path}\n") - - # Compute normalising factor for the mask Cls - integral_w = np.sum((2 * ells + 1) / (4 * np.pi) * cl_mask) / (np.pi / 180) ** 2 - norm_factor = area_obs_deg2 / integral_w - norm_cls = cl_mask * norm_factor - - # Save normalised Cls to text file - norm_path = out_dir / f"mask_cls_{prefix}_nside_{nside}_norm.txt" - idx = np.arange(len(cl_mask)) - data_to_save = np.column_stack((idx, norm_cls)) - np.savetxt(norm_path, data_to_save, fmt=["%d", "%.10e"]) - print(f"Normalised mask Cls saved to {norm_path}") diff --git a/cosmo_inference/scripts/matching.py b/cosmo_inference/scripts/matching.py deleted file mode 100644 index a465e449..00000000 --- a/cosmo_inference/scripts/matching.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Wed Mar 1 17:37:27 2023 -@author: fh272693 -""" - -import astropy.units as u -from astropy.coordinates import SkyCoord, match_coordinates_sky -from astropy.io import fits - -Cat1 = fits.open( - "/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/unions_shapepipe_2022_v1.0.fits" -) -Cat2 = fits.open( - "/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/lensfit_goldshape_2022v1.fits" -) - -coord_units = u.degree -Cat1_coord = SkyCoord( - ra=Cat1[1].data["ra"] * coord_units, dec=Cat1[1].data["dec"] * coord_units -) -Cat2_coord = SkyCoord( - ra=Cat2[1].data["ra"] * coord_units, dec=Cat2[1].data["dec"] * coord_units -) -idx, d2d, d3d = match_coordinates_sky(Cat1_coord, Cat2_coord) -max_sep = 1.0 * u.arcsec -sep_constraint = d2d < max_sep - -# Important here is that the first catalogue of match_coordinates_sky has -# indices [sep_constraint] and the second[idx[sep_constraint]] -Cat1_matches = Cat1[1].data[sep_constraint] - -# np.save('/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/shapepipe_unmatches_ra.npy',Cat1[1].data['ra'][sep_constraint]) -# np.save('/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/shapepipe_unmatches_dec.npy',Cat1[1].data['dec'][sep_constraint]) -# np.save('/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/shapepipe_unmatches_e1.npy',Cat1[1].data['e1'][sep_constraint]) -# np.save('/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/shapepipe_unmatches_e2.npy',Cat1[1].data['e2'][sep_constraint]) -# np.save('/feynman/work/dap/lcs/lg268561/UNIONS/Catalogues/shapepipe_unmatches_w.npy',Cat1[1].data['w'][sep_constraint]) - -print("there are ", len(Cat1_matches), " matching galaxies in catalogue", Cat1) -# print('there are ',len(Cat2_matches),' matching galaxies in catalogue', Cat2) diff --git a/cosmo_inference/scripts/nz_writeout.py b/cosmo_inference/scripts/nz_writeout.py deleted file mode 100644 index 81994335..00000000 --- a/cosmo_inference/scripts/nz_writeout.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -# In[ ]: - -import sys - -import matplotlib.pylab as plt -import numpy as np -from astropy.io import fits - -nz_hdu = sys.argv[1] -root = sys.argv[2] -blind = sys.argv[3] - -hdu = fits.open(nz_hdu) -z = hdu[1].data["Z_%s" % blind] - -zmax = 5.0 - -(n, bins, _) = plt.hist(z, bins=200, range=(0, zmax), density=True, weights=None) - -print("zmin = ", min(z)) -print("zmax = ", max(z)) - -np.savetxt("data/" + root + "/nz_" + root + ".txt", np.column_stack((bins[:-1], n))) diff --git a/cosmo_inference/scripts/slurm.sh b/cosmo_inference/scripts/slurm.sh deleted file mode 100644 index 793aa7cf..00000000 --- a/cosmo_inference/scripts/slurm.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=unions_V1.4 -#SBATCH --mail-user=lgoh@roe.ac.uk -#SBATCH --mail-type=END,FAIL -#SBATCH --partition=compl -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=48 -#SBATCH --time=4-00:00:00 -#SBATCH --output=/n23data1/n06data/lgoh/scratch/CFIS-UNIONS/chains/SP_v1.4_A/inference_A.log - -module load gcc -module load intelpython/3-2024.1.0 -module load openmpi -source cosmosis-configure -source activate my_env - -cosmosis --mpi /n23data1/n06data/lgoh/scratch/CFIS-UNIONS/CFIS-UNIONS_dev/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_1.ini - -# -# Return exit code -exit 0 \ No newline at end of file diff --git a/cosmo_inference/scripts/treecorr_calc.py b/cosmo_inference/scripts/treecorr_calc.py deleted file mode 100644 index 38eb855e..00000000 --- a/cosmo_inference/scripts/treecorr_calc.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - - -import os -import sys - -import numpy as np -import treecorr -from astropy.io import fits - -script_dir = os.path.dirname(os.path.abspath(sys.argv[0])) - -cat_name = sys.argv[1] -root = sys.argv[2] - -hdu = fits.open(cat_name) -data = hdu[1].data - -# Create TreeCorr catalogue -n_thread = 8 -treecorr.set_omp_threads(n_thread) - -sep_units = "arcmin" -nbins = 20 - -TreeCorrConfig = { - "ra_units": "degrees", - "dec_units": "degrees", - "max_sep": "200", - "min_sep": "1", - "sep_units": sep_units, - "nbins": nbins, - "var_method": "jackknife", -} - - -cat_gal = treecorr.Catalog( - ra=data["RA"], - dec=data["Dec"], - g1=data["e1_noleakage"], # for v1.4.1 - g2=data["e2_noleakage"], # for v1.4.1 - w=data["w"], - ra_units="degrees", - dec_units="degrees", - npatch=50, -) - -gg = treecorr.GGCorrelation(TreeCorrConfig) - -print("Running TreeCorr...") -gg.process(cat_gal) - - -lst = np.arange(1, nbins + 1) - -# create fits HDU with xi_p and xi_m data -col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) -col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) -col3 = fits.Column(name="ANGBIN", format="K", array=lst) -col4 = fits.Column(name="VALUE", format="D", array=gg.xip) -col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=gg.meanr) -coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) -xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") - - -col4 = fits.Column(name="VALUE", format="D", array=gg.xim) -coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) -ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") - -# append xi_p/xi_m header info -xip_dict = { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", -} -for key in xip_dict: - xiplus_hdu.header[key] = xip_dict[key] - - -xim_dict = { - "2PTDATA": "T", - "QUANT1": "G-R", - "QUANT2": "G-R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", -} - -for key in xim_dict: - ximinus_hdu.header[key] = xim_dict[key] - -ximinus_hdu.writeto( - "%s/../data/" % script_dir + root + "/ximinus_" + root + ".fits", overwrite=True -) -xiplus_hdu.writeto( - "%s/../data/" % script_dir + root + "/xiplus_" + root + ".fits", overwrite=True -) - -print( - "Correlation functions written to {}".format( - "%s/../data/" % script_dir + root + "/xiplus_minus_" + root + ".fits" - ) -) diff --git a/cosmo_inference/scripts/xi_sys_psf.py b/cosmo_inference/scripts/xi_sys_psf.py deleted file mode 100644 index 631b9464..00000000 --- a/cosmo_inference/scripts/xi_sys_psf.py +++ /dev/null @@ -1,53 +0,0 @@ -import numpy as np -from astropy.io import fits -from cosmosis.datablock import option_section - - -# This file should be added to your cosmosis_standard_library following the path shear/xi_sys/xi_sys_psf.py -def setup(options): - filename = options.get_string(option_section, "data_file") - data = fits.open(filename) - rho_stats_name = options.get_string(option_section, "rho_stats_name") - samples_path = options.get_string(option_section, "samples") - - samples = np.load(samples_path) - mean = np.mean(samples, axis=0) - cov = np.cov(samples.T) - - rho_stats = data[rho_stats_name].data - - return mean, cov, rho_stats - - -def execute(block, config): - - mean, cov, rho_stats = config - - alpha, beta, eta = np.random.multivariate_normal(mean, cov) - block["xi_sys", "alpha"], block["xi_sys", "beta"], block["xi_sys", "eta"] = ( - alpha, - beta, - eta, - ) - - xi_sys_p = ( - alpha**2 * rho_stats["rho_0_p"] - + beta**2 * rho_stats["rho_1_p"] - + eta**2 * rho_stats["rho_3_p"] - + 2 * alpha * beta * rho_stats["rho_2_p"] - + 2 * beta * eta * rho_stats["rho_4_p"] - + 2 * alpha * eta * rho_stats["rho_5_p"] - ) - - xi_sys_m = ( - alpha**2 * rho_stats["rho_0_m"] - + beta**2 * rho_stats["rho_1_m"] - + eta**2 * rho_stats["rho_3_m"] - + 2 * alpha * beta * rho_stats["rho_2_m"] - + 2 * beta * eta * rho_stats["rho_4_m"] - + 2 * alpha * eta * rho_stats["rho_5_m"] - ) - - block["xi_sys", "xi_sys_vec"] = np.concatenate([xi_sys_p, xi_sys_m]) - - return 0 From 36ff7b1a2964bc62760bc09708e13ebf937e5ec1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 16:55:21 +0200 Subject: [PATCH 028/160] chore(2pcf): default integration grid to 1000 bins The B-modes paper found no substantial 1k-vs-10k difference on the integration grid, and develop already unified COSEBIs/pure-EB to nbins_int=1000. Drop the run_2pcf_highres.py 10000-bin default to 1000 and reword the docstring/comments (the Asgari 10k figure becomes context, not the operative number). The grid is config-driven (nbins_int); the MPI path stays available but single-process is the default at 1000 bins. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD --- workflow/scripts/run_2pcf_highres.py | 30 +++++++++++++++------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py index 9fcab0d0..6049e105 100644 --- a/workflow/scripts/run_2pcf_highres.py +++ b/workflow/scripts/run_2pcf_highres.py @@ -1,13 +1,15 @@ #!/usr/bin/env python """ -High-resolution ξ± measurement for COSEBIS integration. +High-resolution ξ± measurement for COSEBIS/pure-EB integration. -Computes TreeCorr GGCorrelation with integration angular binning (10,000+ bins) -required for accurate COSEBIS mode integration. Uses MPI for patch-pair -distribution across nodes when available; falls back to multi-threaded -single-process otherwise. +Computes TreeCorr GGCorrelation on the fine integration angular grid (default +1000 log bins, config-driven) required for accurate COSEBIS/pure-EB mode +integration. Uses MPI for patch-pair distribution across nodes when available; +falls back to multi-threaded single-process otherwise (the default at 1000 bins). -Reference: Asgari et al. 2017 — minimum 10,000 bins for E_7 at 0.5% accuracy. +Reference: Asgari et al. 2017 motivates a fine integration grid; the B-modes +paper found no substantial 1k-vs-10k difference, so the operational default is +1000 bins (see config nbins_int). Usage: # MPI (via Slurm submission script): @@ -39,7 +41,7 @@ _HAVE_COSMO_VAL = True except ImportError: - # Bare-host path (host OpenMPI + host python for the 10k-bin MPI run): the + # Bare-host path (host OpenMPI + host python for an optional MPI run): the # full sp_validation stack (cs_util.plots -> healpy/healsparse) is not # installed. This measurement only needs the shear catalog path + column # names, which are a pure cat_config.yaml lookup — resolve them standalone. @@ -110,7 +112,7 @@ def parse_args(argv=None): default="SP_v1.4.6.3_leak_corr", help="Catalog version key in cat_config", ) - ap.add_argument("--nbins", type=int, default=10000, help="Number of log bins") + ap.add_argument("--nbins", type=int, default=1000, help="Number of log bins") ap.add_argument("--npatch", type=int, default=50, help="TreeCorr patch count") ap.add_argument( "--min-sep", type=float, default=0.5, help="Min separation [arcmin]" @@ -307,8 +309,8 @@ def main(): # Resolve catalog path + ellipticity/weight columns from cat_config + version # exactly as run_2pcf.py does (applies the _leak_corr column swap and the - # subdir path resolution). In-container this uses CosmologyValidation; bare- - # host (10k-bin MPI run) it uses the standalone cat_config resolver, which is + # subdir path resolution). In-container this uses CosmologyValidation; on the + # bare-host MPI fallback it uses the standalone cat_config resolver, which is # byte-identical for the shear-config fields this measurement reads. if _HAVE_COSMO_VAL: cv = CosmologyValidation( @@ -395,10 +397,10 @@ def main(): ) # Write only the main per-bin correlation. The convergence consumer # (cosebis_binning_comparison.py) reads just the per-bin columns - # (np.loadtxt max_rows=nbins) and the 1000-bin covariance — the 10k - # jackknife cov is used nowhere. write_patch_results/write_cov=True - # serialised a 20000x20000 cov + 180 patch blocks (~10 GB) that nothing - # reads and also cost the estimate_cov compute; drop both. The patches + # (np.loadtxt max_rows=nbins); the fine-grid jackknife cov is used + # nowhere. write_patch_results/write_cov=True serialised a full + # (2*nbins)^2 cov + patch blocks that nothing reads and also cost the + # estimate_cov compute; drop both. The patches # still parallelise gg.process; gg.xip/gg.xim (values, FITS) are # unaffected. gg.write(out_txt, write_patch_results=False, write_cov=False) From 6221c18cc195ccdfbb7f407ab70edca439883e57 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 17:22:52 +0200 Subject: [PATCH 029/160] =?UTF-8?q?feat(cosmo=5Fval):=20cv=5Fcosebis/cv=5F?= =?UTF-8?q?pure=5Feb=20consume=20the=20=CE=BE=C2=B1=20SACC=20parts;=20per-?= =?UTF-8?q?version=20xi=5Fhighres?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit COSEBIs and pure-E/B now derive their E-mode outputs from the born-as-SACC ξ± parts on disk instead of only the raw catalogue recompute. This makes born blinding automatic downstream: whatever the part carries (blinded on data runs in PR #253) flows into the E-modes with no blind-aware consumer code. b_modes: add the values-only seams cosebis_from_xi and pure_eb_from_xi (copied from the sacc-6b spur, de-blinded) — same cosmo_numba kernels as the raw path. cosebis_to_sacc_part / pure_eb_to_sacc_part: add en_override / eb_override so the E-mode En (COSEBIs) and the six pure-mode arrays (pure-E/B) written to the SACC part come from the consumed part; Bn and the covariance stay blind-invariant from the raw estimator run. No concealment machinery (that is PR #253's). cv_cosebis.py / cv_pure_eb.py: keep the raw plot_* run (Bn + jackknife covariance need the catalogue and are blind-invariant), then load the integration part (COSEBIs) / reporting + integration parts (pure-E/B), re-derive the E-modes through the seams, and override both the SACC part and the diagnostic npz. pure-E/B reads the reporting-grid bin edges from the raw reporting gg (SACC stores centers only). Unconditional and version-agnostic. xi_highres: per version (was fiducial-hardcoded); in-container single-process at the config-driven 1000-bin grid (the 10k-bin bare-host MPI path is unnecessary). Grid comes from a dedicated cosmo_val.integration block ([0.08, 300] @ 1000) so the one part serves both consumers (pure-E/B full range, COSEBIs scale-cuts to 0.9); decoupled from covariance.smk's FIDUCIAL grid. Shared twopoint.smk falls back to the fiducial integration grid for configs without a cosmo_val section (e.g. papers/bmodes). The raw .txt byproduct is left undeclared to avoid an AmbiguousRuleException with rule xi; nothing in the DAG consumes it. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD --- papers/cosmo_val/config/config.yaml | 14 +++-- src/sp_validation/b_modes.py | 73 ++++++++++++++++++++++++++ src/sp_validation/cosmo_val/cosebis.py | 11 +++- src/sp_validation/cosmo_val/pure_eb.py | 11 +++- workflow/rules/cosmo_val.smk | 31 +++++++---- workflow/rules/twopoint.smk | 63 ++++++++++++++++------ workflow/scripts/cv_cosebis.py | 30 +++++++++-- workflow/scripts/cv_pure_eb.py | 34 +++++++++++- 8 files changed, 228 insertions(+), 39 deletions(-) diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 98409371..232513c7 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -58,11 +58,15 @@ cosmo_val: kmax: 20 kmax_extrapolate: 500 - # Pure E/B-mode decomposition (config space) - pure_eb: - min_sep_int: 0.08 - max_sep_int: 300 - nbins_int: 1000 + # Integration-grid ξ± (the shared fine grid measured once per version by rule + # xi_highres as {version}_xi_integration.sacc). Both estimators consume this one + # part: pure-E/B uses the full range (it must strictly contain the reporting + # grid [1, 250]); COSEBIs scale-cuts it up to 0.9. Owned here, not inside either + # consumer's block. Decoupled from covariance.smk's own FIDUCIAL grid. + integration: + min_sep: 0.08 + max_sep: 300 + nbins: 1000 # COSEBIs decomposition (config space, fine integration binning) cosebis: diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index 13f4c2e9..2b7a4e96 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -255,6 +255,79 @@ def pure_EB(corrs): return results +def cosebis_from_xi(theta, xip, xim, nmodes, scale_cut=None): + """COSEBIs (Eₙ, Bₙ) from ξ± arrays through the pipeline kernel (values only). + + The values-only seam of :func:`calculate_cosebis`, for callers holding + ξ± arrays rather than a TreeCorr ``GGCorrelation`` — e.g. deriving COSEBIs + from an integration-ξ± SACC part. Calls the same ``cosmo_numba`` kernel + (``COSEBIS.cosebis_from_xipm``) directly on the values; the covariance/χ² + machinery stays with :func:`calculate_cosebis`. + + ``scale_cut`` follows the :func:`sacc_io.add_cosebis` writer contract: + ``(theta_min, theta_max)`` are min/max of the *retained* bin centres + after the pipeline's ``scale_cut_to_bins``. The cut is contiguous in an + ascending grid, so selecting ``theta_min ≤ θ ≤ theta_max`` inclusively + reproduces exactly the retained set, and the kernel is built on that + set's min/max support and fed only the retained ξ± — bit-matching + :func:`calculate_cosebis`'s ``theta_cut``/``xip_cut``/``xim_cut`` path. + Identical inputs ⇒ identical numbers. + """ + from cosmo_numba.B_modes.cosebis import COSEBIS + + theta, xip, xim = (np.asarray(a) for a in (theta, xip, xim)) + tmin, tmax = scale_cut if scale_cut is not None else (theta.min(), theta.max()) + cut = (theta >= tmin) & (theta <= tmax) + theta_cut, xip_cut, xim_cut = theta[cut], xip[cut], xim[cut] + cosebis = COSEBIS( + theta_min=np.min(theta_cut), + theta_max=np.max(theta_cut), + N_max=nmodes, + precision=120, + ) + En, Bn = cosebis.cosebis_from_xipm(theta_cut, xip_cut, xim_cut, parallel=True) + return np.asarray(En), np.asarray(Bn) + + +def pure_eb_from_xi( + theta_report, xip_report, xim_report, theta_int, xip_int, xim_int, tmin, tmax +): + """Pure-E/B correlation functions from ξ± arrays through the pipeline kernel. + + The values-only seam of :func:`calculate_pure_eb_correlation`, for + callers holding ξ± arrays rather than TreeCorr correlations — e.g. + deriving pure-E/B from SACC parts. Calls the same ``cosmo_numba`` kernel + (``get_pure_EB_modes``) directly on the values. + The reporting grid must be a strict sub-range of the integration grid; + ``tmin``/``tmax`` are the reporting correlation's TreeCorr *bin edges* + (``gg.left_edges[0]`` / ``gg.right_edges[-1]``) — the pipeline's + convention, carried on SACC files by ``sacc_io.add_pure_eb``. A + reporting point coinciding with the integration boundary is degenerate + (no interior support) and comes back NaN, exactly as + :func:`calculate_pure_eb_correlation` returns it — never a spurious + finite value. + + Returns + ------- + dict + Keyed by ``_EB_KEYS`` (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb). + """ + from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes + + modes = get_pure_EB_modes( + theta=np.asarray(theta_report), + xip=np.asarray(xip_report), + xim=np.asarray(xim_report), + theta_int=np.asarray(theta_int), + xip_int=np.asarray(xip_int), + xim_int=np.asarray(xim_int), + tmin=tmin, + tmax=tmax, + parallel=True, + ) + return dict(zip(_EB_KEYS, (np.asarray(m) for m in modes))) + + def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): """ Calculate COSEBIs modes from a correlation function for multiple scale cuts. diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index 9ca048c3..406f0a11 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -164,7 +164,9 @@ def _fiducial_cosebis_result(results, fiducial_scale_cut): ) return results[key], tuple(key) - def cosebis_to_sacc_part(self, version, out_path, results, fiducial_scale_cut=None): + def cosebis_to_sacc_part( + self, version, out_path, results, fiducial_scale_cut=None, en_override=None + ): """Write the COSEBIs SACC part at the fiducial scale cut. ``results`` is the object ``calculate_cosebis`` returned (single dict or @@ -172,8 +174,15 @@ def cosebis_to_sacc_part(self, version, out_path, results, fiducial_scale_cut=No part — a ``FullCovariance`` must cover every stored point and the cuts overlap in mode space, so the non-fiducial cuts stay in the diagnostic ``.npz`` sidecar. The nz/metadata are the version's. + + ``en_override`` is the consume-the-part plumbing: the E-mode ``En`` written + to the part in place of ``result["En"]`` — re-derived from the integration + ξ± SACC part at the fiducial scale cut (Bn and the covariance stay from the + raw estimator ``result``). With ``None`` the behaviour is unchanged. """ result, scale_cut = self._fiducial_cosebis_result(results, fiducial_scale_cut) + if en_override is not None: + result = {**result, "En": np.asarray(en_override)} s = cosebis_to_sacc( self.sacc_nz(version), self.sacc_metadata(version), diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 8375d5e0..29e1f888 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -134,16 +134,23 @@ def calculate_pure_eb( return results - def pure_eb_to_sacc_part(self, version, out_path, results): + def pure_eb_to_sacc_part(self, version, out_path, results, eb_override=None): """Write the pure-E/B SACC part (six ``PURE_KEYS`` blocks + covariance). ``results`` is the dict ``calculate_pure_eb`` returned: the six pure-mode arrays under ``sacc_io.PURE_KEYS``, the ``"cov"`` block (in ``PURE_KEYS`` order), and the reporting-grid TreeCorr object ``"gg"`` whose ``meanr`` is the shared ``theta``. + + ``eb_override`` is the consume-the-part plumbing: the six pure-mode arrays + (a mapping keyed by ``sacc_io.PURE_KEYS``) written in place of ``results``' + — re-derived from the reporting + integration ξ± SACC parts (the covariance + stays blind-invariant from the raw estimator ``results``). With ``None`` the + behaviour is unchanged. """ theta = results["gg"].meanr - eb = {key: results[key] for key in sacc_io.PURE_KEYS} + source = eb_override if eb_override is not None else results + eb = {key: source[key] for key in sacc_io.PURE_KEYS} s = pure_eb_to_sacc( self.sacc_nz(version), self.sacc_metadata(version), diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 3536ed11..0cac5126 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -74,13 +74,13 @@ def cv_tau_stats(version): def cv_pure_eb_npz(version): - eb = CV["pure_eb"] + eb = CV["integration"] return str( COSMO_VAL / ( f"{version}_eb_minsep={CV['theta_min']}_maxsep={CV['theta_max']}" - f"_nbins={CV['nbins']}_minsepint={eb['min_sep_int']}" - f"_maxsepint={eb['max_sep_int']}_nbinsint={eb['nbins_int']}" + f"_nbins={CV['nbins']}_minsepint={eb['min_sep']}" + f"_maxsepint={eb['max_sep']}_nbinsint={eb['nbins']}" f"_npatch={CV['npatch']}_varmethod=jackknife_data.npz" ) ) @@ -169,6 +169,16 @@ def cv_xi_reporting_sacc(version): ) +def cv_xi_integration_sacc(version): + """Integration-grid ξ± SACC part the xi_highres rule writes, per version. + + Intermediate per-statistic part (grid='integration', its own DiagonalCovariance + from TreeCorr varxip/varxim). NOT folded into the terminal {version}.sacc (see + #247 ruling) — COSEBIs and pure-E/B consume it directly. + """ + return str(COSMO_VAL / f"{version}_xi_integration.sacc") + + def cv_analysis_sacc(version): """Terminal assembled analysis file {version}.sacc.""" return str(COSMO_VAL / f"{version}.sacc") @@ -362,14 +372,16 @@ rule cv_pure_eb: """Pure E/B-mode decomposition for one version (config-space).""" input: xi=lambda w: cv_xi_txt(w.version), + xi_reporting=lambda w: cv_xi_reporting_sacc(w.version), + xi_integration=lambda w: cv_xi_integration_sacc(w.version), output: npz=cv_pure_eb_npz("{version}"), sacc=cv_pure_eb_sacc("{version}"), params: version="{version}", - min_sep_int=CV["pure_eb"]["min_sep_int"], - max_sep_int=CV["pure_eb"]["max_sep_int"], - nbins_int=CV["pure_eb"]["nbins_int"], + min_sep_int=CV["integration"]["min_sep"], + max_sep_int=CV["integration"]["max_sep"], + nbins_int=CV["integration"]["nbins"], fiducial_scale_cut=CV["fiducial_scale_cut"], cv_init=lambda w: cv_init_params(config, version_list=[w.version]), rundir=CV_RUNDIR, @@ -385,6 +397,7 @@ rule cv_cosebis: """COSEBIs E/B decomposition for one version (config-space, fine binning).""" input: xi=lambda w: cv_xi_txt(w.version), + xi_integration=lambda w: cv_xi_integration_sacc(w.version), output: npz=cv_cosebis_npz("{version}"), sacc=cv_cosebis_sacc("{version}"), @@ -420,9 +433,9 @@ rule cv_summarize_bmodes: summary_json=str(COSMO_VAL / "bmode_summary.json"), params: fiducial_scale_cut=CV["fiducial_scale_cut"], - pure_eb_min_sep_int=CV["pure_eb"]["min_sep_int"], - pure_eb_max_sep_int=CV["pure_eb"]["max_sep_int"], - pure_eb_nbins_int=CV["pure_eb"]["nbins_int"], + pure_eb_min_sep_int=CV["integration"]["min_sep"], + pure_eb_max_sep_int=CV["integration"]["max_sep"], + pure_eb_nbins_int=CV["integration"]["nbins"], cosebis_min_sep_int=CV["cosebis"]["min_sep_int"], cosebis_max_sep_int=CV["cosebis"]["max_sep_int"], cosebis_nbins_int=CV["cosebis"]["nbins_int"], diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 07e36372..69c516e9 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -30,34 +30,63 @@ rule xi: "../scripts/run_2pcf.py" +# Integration-grid ξ± measured by xi_highres. The cosmo_val paper owns a dedicated +# cosmo_val.integration block ([0.08, 300] @ 1000 bins); other workflows sharing +# this file (e.g. papers/bmodes, whose config carries no cosmo_val section) fall +# back to their fiducial integration grid. Evaluated at parse time, so the lookup +# must not assume the cosmo_val key exists. +_INTEGRATION = config.get("cosmo_val", {}).get("integration") or { + "min_sep": FIDUCIAL["min_sep_int"], + "max_sep": FIDUCIAL["max_sep_int"], + "nbins": FIDUCIAL["nbins_int"], +} + + rule xi_highres: - """High-resolution xi for COSEBIS integration. + """High-resolution integration-grid xi for COSEBIs + pure-E/B, per version. Intermediate born-as-SACC part: {version}_xi_integration.sacc (a DiagonalCovariance from TreeCorr varxip/varxim). COSEBIs and pure-E/B consume it; it stays a standalone per-part file and does not join the terminal {version}.sacc (see #247 ruling). The raw .txt dump is kept as a convergence byproduct. + + In-container single-process TreeCorr: at the config-driven nbins_int=1000 grid + this is a normal single-node job (the global container: in the Snakefile makes + a plain shell: run in-container). run_2pcf_highres.py runs its single-process + path when not launched under mpiexec. The historical 10k-bin bare-host MPI path + is removed as unnecessary. """ - container: None + input: + catalog=get_shear_catalog, output: - txt=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.txt"), - xi_integration=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_integration.sacc"), + # Only the uniquely-named SACC part is tracked. The raw TreeCorr .txt dump + # run_2pcf_highres.py writes ({version}_xi_minsep=..._nbins=..._npatch=1.txt) + # is left UNDECLARED: it is a convergence byproduct nothing in the DAG + # consumes (cv_xi_txt is the reporting grid), and declaring it would collide + # with rule xi's wildcard txt output (same filename pattern) — an + # AmbiguousRuleException. Shared integration grid (cosmo_val.integration: + # [0.08, 300] at 1000 bins) so the single part serves both consumers: + # pure-E/B needs it to strictly contain its reporting grid down to 0.08; + # COSEBIs scale-cuts on the same part. Decoupled from covariance.smk. + xi_integration=str(COSMO_VAL / "{version}_xi_integration.sacc"), + params: + version="{version}", + cat_config=CAT_CONFIG, + min_sep=_INTEGRATION["min_sep"], + max_sep=_INTEGRATION["max_sep"], + nbins=_INTEGRATION["nbins"], + out=str(COSMO_VAL), + scripts=WORKFLOW_SCRIPTS, + threads: 24 resources: - tasks=30, - cpus_per_task=12, - nodes=6, - mem_mb_per_cpu=2000, - runtime=2880, - slurm_extra="'--exclude=n17,n09,n36 --partition=pscomp'", - mpi="/softs/openmpi/5.0.5-slurm-CentOS8/bin/mpiexec", + mem_mb=40000, + runtime=600, shell: - "{resources.mpi} -n {resources.tasks} " - "apptainer exec " - "--bind /home,/n09data,/n17data,/n23data1,/softs " - "--env LD_LIBRARY_PATH=/softs/openmpi/5.0.5-slurm-CentOS8/lib " - "/n17data/cdaley/containers/containers " - f"python {WORKFLOW_SCRIPTS}/run_2pcf_highres.py" + "python {params.scripts}/run_2pcf_highres.py " + "--version {params.version} --cat-config {params.cat_config} " + "--min-sep {params.min_sep} --max-sep {params.max_sep} " + "--nbins {params.nbins} --npatch 1 --out {params.out}" rule run_cosmo_val: diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index b90ebc5b..84991d7b 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -10,6 +10,7 @@ PTE scan. """ +import numpy as np from cv_runner import _unbuffer_streams, make_cv, verify_outputs from snakemake.script import snakemake @@ -17,6 +18,7 @@ cv = make_cv(snakemake) p = snakemake.params version = p["version"] +fiducial_scale_cut = tuple(p["fiducial_scale_cut"]) cv.plot_cosebis( version=version, min_sep_int=p["min_sep_int"], @@ -25,14 +27,36 @@ npatch=p["npatch"], nmodes=p["nmodes"], scale_cuts=[tuple(sc) for sc in p["scale_cuts"]], - fiducial_scale_cut=tuple(p["fiducial_scale_cut"]), + fiducial_scale_cut=fiducial_scale_cut, ) + +# Consume the integration-grid ξ± SACC part: re-derive the fiducial-cut E-mode En +# from it through the same cosmo_numba kernel plot_cosebis' raw path uses +# (b_modes.cosebis_from_xi). Bn and the covariance stay blind-invariant from the +# raw plot_cosebis result. Version-agnostic — every version binds the part. +from sp_validation import sacc_io +from sp_validation.b_modes import cosebis_from_xi + +integ = sacc_io.load(snakemake.input["xi_integration"]) +theta, xip, xim = sacc_io.get_xi(integ, (0, 0), grid="integration") +en_part, _ = cosebis_from_xi(theta, xip, xim, p["nmodes"], scale_cut=fiducial_scale_cut) + # Born-as-SACC COSEBIs part at the fiducial scale cut (plot_cosebis stored the -# multi-cut results on the instance). +# multi-cut results on the instance); En comes from the consumed part. cv.cosebis_to_sacc_part( version, snakemake.output["sacc"], cv._cosebis_results[version], - fiducial_scale_cut=tuple(p["fiducial_scale_cut"]), + fiducial_scale_cut=fiducial_scale_cut, + en_override=en_part, ) + +# Overwrite the raw fiducial-cut En plot_cosebis wrote into the diagnostic npz with +# the part-derived En (identical to the SACC part's). Bn / cov / PTE fields are +# untouched, so the B-mode summary reader is unaffected. +npz_path = snakemake.output["npz"] +data = dict(np.load(npz_path, allow_pickle=True)) +data["En"] = np.asarray(en_part) +np.savez(npz_path, **data) + verify_outputs(snakemake) diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index 7a453b16..d4797922 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -9,6 +9,7 @@ + covariance) that the assemble_sacc rule consumes. """ +import numpy as np from cv_runner import _unbuffer_streams, make_cv, verify_outputs from snakemake.script import snakemake @@ -24,6 +25,35 @@ fiducial_xip_scale_cut=tuple(p["fiducial_scale_cut"]), fiducial_xim_scale_cut=tuple(p["fiducial_scale_cut"]), ) -# Born-as-SACC pure-E/B part (plot_pure_eb stored the results on the instance). -cv.pure_eb_to_sacc_part(version, snakemake.output["sacc"], cv._pure_eb_results[version]) +results = cv._pure_eb_results[version] + +# Consume the reporting + integration ξ± SACC parts: re-derive the six pure-mode +# arrays through the same cosmo_numba kernel plot_pure_eb' raw path uses +# (b_modes.pure_eb_from_xi). The covariance stays blind-invariant from the raw +# result. tmin/tmax are the reporting grid's TreeCorr bin edges (from the raw +# reporting gg — add_xi stores no edges). Version-agnostic — every version binds +# both parts. +from sp_validation import sacc_io +from sp_validation.b_modes import pure_eb_from_xi + +gg = results["gg"] +tmin, tmax = float(gg.left_edges[0]), float(gg.right_edges[-1]) +rep = sacc_io.load(snakemake.input["xi_reporting"]) +integ = sacc_io.load(snakemake.input["xi_integration"]) +tr, xpr, xmr = sacc_io.get_xi(rep, (0, 0), grid="reporting") +ti, xpi, xmi = sacc_io.get_xi(integ, (0, 0), grid="integration") +modes = pure_eb_from_xi(tr, xpr, xmr, ti, xpi, xmi, tmin, tmax) + +# Born-as-SACC pure-E/B part; the six pure-mode blocks come from the consumed parts. +cv.pure_eb_to_sacc_part(version, snakemake.output["sacc"], results, eb_override=modes) + +# Overwrite the raw pure-mode arrays plot_pure_eb wrote into the diagnostic npz with +# the part-derived ones (identical to the SACC part's). theta / cov / PTE fields are +# untouched, so the B-mode summary reader is unaffected. +npz_path = snakemake.output["npz"] +data = dict(np.load(npz_path, allow_pickle=True)) +for key, arr in modes.items(): + data[key] = np.asarray(arr) +np.savez(npz_path, **data) + verify_outputs(snakemake) From 251f4f51c3f8b9edaa7b2b0a854b1b08bc894b75 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 17 Aug 2026 20:21:43 +0200 Subject: [PATCH 030/160] Blind through the fork's vector core, and bind the draw scheme into custody MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes to blinding.py, both about the seam with Smokescreen. **The concealing factor is a vector, so ask for a vector.** `_concealing_factor` carved a sub-SACC per block (`s.copy()` + `keep_indices`, covariance and all), handed it to `ConcealDataVector`, applied the blind, and subtracted the result back off to recover the shift it wanted in the first place. The fork now exposes `concealing_factor(fiducial_params, shifts_dict, *, seed, theory_fn)` — the pure theory difference, no SACC, no data vector — so call that. The theory factories already read their layout off whatever SACC they are handed, so they run on the part directly and return a full-length vector, NaN outside their own block; the factor is that vector at the block's rows. `unblind_sacc` differenced the two theory vectors itself. It now goes through the same `_concealing_factor` call, so the shift added and the shift subtracted cannot drift apart. `_extract_block` is gone with its last caller. A backend that leaves a row of its *own* block unfilled would have shifted that point by NaN, silently. The slice is now checked finite and refuses instead. **A blind is (seed, config, draw scheme) — not (seed, config).** The fork versions its shift-draw semantics as `smokescreen.DRAW_SCHEME`: upstream's one global RNG stream over sorted keys is scheme 1, the fork's per-key `(seed, key)` RNG is scheme 2. Blind under one and unblind under the other and the same seed draws a *different* hidden cosmology — so the unblind subtracts a shift that was never added, leaving a smooth residual in the "revealed" vector while the seed hash, the config digest and the escrow equality check all still pass. It is the one blinding failure with no symptom. The scheme is now custody state. `blind_init` writes it into commitment.json; `_stamp_provenance` stamps `blind_draw_scheme` on every blinded file; and it is re-checked against the installed fork wherever a shift is drawn or subtracted — `_read_seed` (so a scheme change between blinding part 1 and part 2 is caught), `unblind_sacc` before any subtraction, `assert_consistent_blind` at the terminal, `stamp_concealed_passthrough`, and the CLI's seedless `verify`. A missing record fails closed: a blind whose scheme is unknown cannot be shown reproducible. Docstrings that promised same-(seed, config) reproducibility "forever", or per-key draw independence, now say what guarantees it. Closes review findings 1, 2 and 4 on PR #253. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BRYg9fjMevgcsvgjh3KN6x --- scripts/blind_data_vector.py | 24 ++ src/sp_validation/blinding.py | 336 ++++++++++++++++------- src/sp_validation/tests/test_blinding.py | 186 ++++++++++++- 3 files changed, 433 insertions(+), 113 deletions(-) diff --git a/scripts/blind_data_vector.py b/scripts/blind_data_vector.py index 9b41109d..d9aeb502 100644 --- a/scripts/blind_data_vector.py +++ b/scripts/blind_data_vector.py @@ -113,6 +113,30 @@ def _verify(args): problems.append("blind_commitment does not match the committed sha256(seed)") if s.metadata.get("blind_config_digest") != commitment["config_digest"]: problems.append("blind_config_digest does not match the committed digest") + # The draw scheme is checked three ways: file ↔ commitment, and both against + # the installed fork. A scheme mismatch is the one blinding failure with no + # numerical symptom — the unblind would subtract a different shift than was + # added and every other check here would still pass. + installed = blinding.draw_scheme() + file_scheme = s.metadata.get("blind_draw_scheme") + committed = commitment.get("draw_scheme") + if file_scheme is None or committed is None: + problems.append( + "no draw-scheme record on the file and/or in the commitment — " + "this blind predates draw-scheme binding and cannot be verified " + "reproducible" + ) + elif int(file_scheme) != int(committed): + problems.append( + f"blind_draw_scheme {int(file_scheme)} does not match the committed " + f"draw_scheme {int(committed)}" + ) + elif int(file_scheme) != installed: + problems.append( + f"blind was drawn under Smokescreen DRAW_SCHEME={int(file_scheme)} " + f"but the installed fork implements DRAW_SCHEME={installed} — this " + "install cannot reproduce the shift" + ) if "seed_smokescreen" in s.metadata: problems.append("PLAINTEXT SEED LEAKED into file metadata (seed_smokescreen)") if problems: diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 97a220f1..c2ff148f 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -12,20 +12,25 @@ **The fork is the concealment engine.** The hidden cosmology is drawn by the ``UNIONS-WL/Smokescreen`` fork's own CCL-native, per-key-independent - draw; each part goes through its ``ConcealDataVector(fiducial_params, - shifts_dict, sacc_data, *, seed, theory_fn)`` entry point. This module - supplies the two sp_validation-specific pieces: the three ``theory_fn`` - backends matching each part's row layout (reporting ξ±, integration ξ±, pseudo-Cℓ) - and the SACC handling around the concealed vectors. + draw. Blinding is a vector operation, so this module calls the fork's + vector core directly — ``smokescreen.concealing_factor(fiducial_params, + shifts_dict, *, seed, theory_fn)``, which returns + ``t(hidden) − t(fiducial)`` and never sees a SACC. This module supplies + the two sp_validation-specific pieces: the three ``theory_fn`` backends + matching each part's row layout (reporting ξ±, integration ξ±, pseudo-Cℓ) + and the SACC handling around the returned factors. **Envelope calibration.** The blinding intent is an amplitude smear of a chosen (S8, Ωm) box. The fork draws in CCL-native primitives, so :meth:`BlindingConfig.shifts_dict` maps the intended box to ``{sigma8, Omega_c}`` half-widths evaluated at the fiducial — - ``σ8 = S8/√(Ωm/0.3)``, ``Ω_c = Ωm − Ω_b − Ω_ν``. The same seed yields - the same hidden cosmology across all part passes (the fork's draw depends - only on ``(key, seed)``), so the blinded parts are mutually consistent by - construction. + ``σ8 = S8/√(Ωm/0.3)``, ``Ω_c = Ωm − Ω_b − Ω_ν``. Under the installed draw + scheme the deltas depend only on ``(key, seed, shift_distr)``, so one seed + yields one hidden cosmology across all part passes and the blinded parts + are mutually consistent by construction. That "only" is a property of the + *draw scheme*, not of the seed: :func:`draw_scheme` records which scheme a + blind was drawn under and every custody gate refuses an install that + disagrees (see **Custody** below). **Derived statistics are born blinded.** COSEBIs and pure-E/B are never touched by this module: the pipeline's own estimators @@ -37,17 +42,23 @@ **Custody: hash commitment, no keyholder.** :func:`blind_init` runs once per catalogue version: it draws an OS-entropy seed, publishes - ``sha256(seed)`` plus a canonical config digest as a repo-committable - ``commitment.json``, and encrypts the seed into a Fernet bundle - (``smokescreen.encryption``) — the plaintext seed is never written. + ``sha256(seed)``, a canonical config digest, and the installed fork's + ``DRAW_SCHEME`` as a repo-committable ``commitment.json``, and encrypts + the seed into a Fernet bundle (``smokescreen.encryption``) — the plaintext + seed is never written. The three together, not the seed alone, are what + reproduces a blind: seed and config fix *which* shift, the draw scheme + fixes *how* the seed becomes that shift. + Each :func:`blind_part` call reads that fixed state, conceals one part, escrows the part's true vector into its own encrypted bundle beside the - blinded output, and deletes the plaintext part. Terminal assembly - (:func:`sp_validation.sacc_io.gather`) calls - :func:`assert_consistent_blind` to fail closed unless every blindable - part carries the identical ``blind_commitment``. :func:`unblind_part` - verifies both hashes against the commitment *before* subtracting - anything, then restores the true part. + blinded output, and deletes the plaintext part. Every path that assembles + parts into the one-file product goes through + :func:`sp_validation.sacc_io.gather` — the production assembler is passed + *into* it rather than wrapped around it — and gather calls + :func:`assert_consistent_blind` to fail closed unless every blindable part + carries the identical ``blind_commitment``, ``blind_config_digest`` and + ``blind_draw_scheme``. :func:`unblind_part` verifies all three against the + commitment *before* subtracting anything, then restores the true part. """ import dataclasses @@ -89,8 +100,11 @@ def shifts_dict(self): ``ΔS8/√(Ωm_fid/0.3)`` in σ8 (at fixed Ωm), and a ΔΩm half-width maps one-to-one to Ω_c (Ω_b and Ω_ν are fixed). Exact enough for a blinding smear — the target is a characteristic amplitude, not a - precise (S8, Ωm) posterior. The fork draws each key independently as - ``U(fid − h, fid + h)``. + precise (S8, Ωm) posterior. Under the installed draw scheme the fork + draws each key independently as ``U(fid − h, fid + h)``, so adding or + resizing one key never moves another; :func:`draw_scheme` is what + makes that independence a guarantee rather than a hope, by refusing an + install whose draw semantics differ from the blind's. """ return { "sigma8": self.s8_half_width / np.sqrt(self.theory.Omega_m / 0.3), @@ -166,15 +180,68 @@ def seed_commitment(seed): return hashlib.sha256(seed.encode("utf-8")).hexdigest() +def draw_scheme(): + """The installed Smokescreen fork's shift-draw semantics version. + + ``smokescreen.DRAW_SCHEME`` is the fork's own version number for *how* a + seed becomes a set of parameter deltas — scheme 1 is upstream DESC's one + global RNG stream consumed over the sorted keys, scheme 2 (this fork) a + per-key RNG derived from ``(seed, key)``. Two installs that agree on + ``(seed, config)`` but disagree on this number draw *different* hidden + cosmologies from the same inputs. + + That is the one blinding failure with no loud symptom: a blind added under + one scheme and subtracted under another leaves a residual smooth + cosmological shift in the "unblinded" vector, which every hash, digest and + escrow check would still pass. So the scheme is bound into the blind's + custody state — ``commitment.json`` and every blinded file's + ``blind_draw_scheme`` — and re-checked against this function wherever a + shift is drawn or subtracted (:func:`_assert_draw_scheme`). + """ + from smokescreen import DRAW_SCHEME + + return int(DRAW_SCHEME) + + +def _assert_draw_scheme(recorded, what): + """Fail closed unless ``recorded`` is the installed fork's draw scheme. + + ``what`` names the surface the scheme was read from, for the message. + A missing record (``None``) is a failure, not a pass: a blind whose scheme + is unknown cannot be shown to be reproducible by this install. + """ + installed = draw_scheme() + if recorded is None: + raise ValueError( + f"{what} carries no draw-scheme record — refusing to proceed. " + f"It predates draw-scheme binding, so there is no way to tell " + f"whether the installed Smokescreen (DRAW_SCHEME={installed}) " + f"reproduces the shift it was blinded with." + ) + if int(recorded) != installed: + raise ValueError( + f"{what} was drawn under Smokescreen DRAW_SCHEME={int(recorded)} " + f"but the installed fork implements DRAW_SCHEME={installed} — " + f"refusing to proceed. The same seed draws a different hidden " + f"cosmology under a different scheme, so this install would " + f"subtract the wrong shift and pass every other check. Install " + f"the Smokescreen the blind was made with." + ) + + def hidden_params(seed, config): """The hidden CCL parameter point the fork realizes for ``(seed, config)``. Re-runs the fork's own draw (``smokescreen.param_shifts.draw_param_shifts`` — per-key-independent, local RNG) on the calibrated envelope and overlays - the deltas on the fiducial, exactly as ``ConcealDataVector`` does - internally. Deterministic: same ``(seed, config)`` ⇒ same hidden point, - forever — the reproducibility contract unblinding relies on, and what - makes every part share one hidden cosmology under one seed. + the deltas on the fiducial, exactly as + ``smokescreen.concealing_factor`` does internally. Deterministic under a + fixed draw scheme: same ``(seed, config)`` ⇒ same hidden point on any + install whose :func:`draw_scheme` matches — the reproducibility contract + unblinding relies on, and what makes every part share one hidden cosmology + under one seed. This is an introspection helper (what *was* the hidden + cosmology, once revealed); the blinding path itself never calls it, and so + does not check the scheme here — every gate that acts on a shift does. """ from smokescreen.param_shifts import draw_param_shifts @@ -315,7 +382,7 @@ def theory_fn(params): # --------------------------------------------------------------------------- # -# Extract → conceal → merge, per block +# The concealing factor, per block # --------------------------------------------------------------------------- # def _blindable_blocks(s): """The blindable blocks of a SACC as ``(name, indices, factory)``. @@ -323,10 +390,9 @@ def _blindable_blocks(s): Works identically on a standalone part (which carries exactly one block) and on the assembled one-file product (whose integration rows are selected by the ``grid`` tag — the layout contract's per-block tag selection). - ``indices`` are each block's recorded row indices (ascending, so the - extracted sub-SACC preserves row order); ``factory`` builds the matching - ``theory_fn`` from the extracted sub-SACC. Blocks absent from the file - are simply not listed. + ``indices`` are each block's recorded row indices (ascending); ``factory`` + builds the matching ``theory_fn`` from the SACC those indices point into. + Blocks absent from the file are simply not listed. """ blocks = [] for grid in ("reporting", "integration"): @@ -345,43 +411,54 @@ def _blindable_blocks(s): return blocks -def _extract_block(s, indices): - """Extract rows ``indices`` (ascending) into a sub-SACC, order preserved. - - Deliberately index-based rather than ``sacc_io.extract`` / - ``update_statistic`` (which select and merge by ``(data_type, tracers, - tags)``): Smokescreen's ``ConcealDataVector`` aligns its ``theory_fn`` - output to the sub-SACC's ``mean`` element-for-element by row position, so - the block must be carved and written back by contiguous integer index, not - by tag-matching. This is the one place the row-index path is load-bearing. - """ - sub = s.copy() - sub.keep_indices(np.asarray(indices, dtype=int)) - return sub - - def _concealing_factor(s, indices, factory, config, seed): - """The fork-computed additive concealing factor for one block. + """The fork-computed additive concealing factor for one block of ``s``. + + Blinding is a vector operation, so this goes straight to the fork's vector + core: ``smokescreen.concealing_factor`` draws the hidden deltas from + ``seed``, overlays them on the fiducial, evaluates the block's + ``theory_fn`` at both points and differences them. No data vector and no + SACC reach the fork — nothing is carved out of ``s``, and ``s`` itself is + not modified. + + The factory reads its layout off ``s`` directly, so the returned vector is + full-length: a block's ``theory_fn`` fills only its own rows and leaves + every other row NaN. Slicing to ``indices`` drops those NaNs by + construction; the finite check then proves the converse — that the + factory filled *all* of this block's rows. A row the block claims but the + factory cannot cover (a pair carrying ξ− without ξ+, say) would otherwise + write NaN into the data vector, silently. + + Both :func:`blind_sacc` and :func:`unblind_sacc` reach the fork through + this one function, so the added and the subtracted shift cannot drift + apart. - Extracts the block into a sub-SACC containing exactly the rows the - ``theory_fn`` spans (the fork's length guard enforces the agreement), - drives ``ConcealDataVector`` with the fiducial point, the calibrated - envelope, and the block's ``theory_fn``, and returns the factor - ``t(hidden) − t(fiducial)`` aligned to ``indices``. + Returns + ------- + np.ndarray + ``t(hidden) − t(fiducial)``, aligned to ``indices``. """ - from smokescreen import ConcealDataVector - - sub = _extract_block(s, indices) - smoke = ConcealDataVector( - config.theory.ccl_params(), - config.shifts_dict(), - sub, - seed=seed, - theory_fn=factory(sub, config.theory), + from smokescreen import concealing_factor + + full = np.asarray( + concealing_factor( + config.theory.ccl_params(), + config.shifts_dict(), + seed=seed, + theory_fn=factory(s, config.theory), + factor_type="add", + ), + dtype=float, ) - smoke.calculate_concealing_factor(factor_type="add") - concealed = smoke.apply_concealing_to_likelihood_datavec() - return np.asarray(concealed, dtype=float) - np.asarray(sub.mean, dtype=float) + factor = full[indices] + if not np.all(np.isfinite(factor)): + raise ValueError( + f"the theory backend left {int(np.sum(~np.isfinite(factor)))} of " + f"{len(indices)} blindable rows unfilled — refusing to blind " + "(these rows would be shifted by NaN). The block's row layout is " + "not fully covered by its theory_fn." + ) + return factor def _set_values(s, indices, values): @@ -399,11 +476,13 @@ def blind_sacc(part, seed, config=None, label="A", log=print): """Return a blinded copy of a part SACC (covariance and tags untouched). Per blindable block present (a standalone part carries exactly one — - reporting ξ±, integration ξ±, or pseudo-Cℓ_EE): extract the block into a sub-SACC, - conceal through the fork, write the shifted values back at their - recorded indices (row order preserved). Provenance is stamped and any - leaked seed key stripped. A file with no blindable block (e.g. a ρ/τ - diagnostic part) is refused loudly — it should never see a blind call. + reporting ξ±, integration ξ±, or pseudo-Cℓ_EE): ask the fork for the + block's concealing factor (:func:`_concealing_factor`) and add it at the + block's recorded indices. Row order, tags, n(z) and covariance are + untouched — only ``value`` changes, and only on blindable rows. + Provenance is stamped and any leaked seed key stripped. A file with no + blindable block (e.g. a ρ/τ diagnostic part) is refused loudly — it should + never see a blind call. """ config = config or BlindingConfig() if _concealed(part): @@ -428,17 +507,20 @@ def blind_sacc(part, seed, config=None, label="A", log=print): def unblind_sacc(blinded, seed, config=None, log=print): """Recover the true part SACC from a blinded one + the revealed ``seed``. - Verifies ``sha256(seed)`` and the config digest against the stamped - metadata (loud failure on either mismatch — verification precedes - subtraction), recomputes each block's shift from the seed through the - same backends, and subtracts it. Works on a standalone part or on the - assembled file (integration rows selected by the ``grid`` tag). Derived - statistics, if present (assembled file), are *not* recomputed here — the - pipeline's own estimators re-derive them from the unblinded integration ξ±. + Verifies the stamped draw scheme against the installed fork, then + ``sha256(seed)`` and the config digest against the stamped metadata (loud + failure on any of the three — verification precedes subtraction), then + recomputes each block's shift through :func:`_concealing_factor`, the same + call :func:`blind_sacc` added it with, and subtracts it. Works on a + standalone part or on the assembled file (integration rows selected by the + ``grid`` tag). Derived statistics, if present (assembled file), are *not* + recomputed here — the pipeline's own estimators re-derive them from the + unblinded integration ξ±. """ config = config or BlindingConfig() if not _concealed(blinded): raise ValueError("file is not concealed — nothing to unblind") + _assert_draw_scheme(blinded.metadata.get("blind_draw_scheme"), "this blinded file") if seed_commitment(seed) != blinded.metadata["blind_commitment"]: raise ValueError( "seed does not match blind_commitment — refusing to unblind " @@ -451,34 +533,46 @@ def unblind_sacc(blinded, seed, config=None, log=print): "added)" ) - hidden = hidden_params(seed, config) - fiducial = config.theory.ccl_params() part = blinded.copy() for name, indices, factory in _blindable_blocks(blinded): - theory = factory(_extract_block(blinded, indices), config.theory) - factor = theory(hidden) - theory(fiducial) + factor = _concealing_factor(blinded, indices, factory, config, seed) _set_values(part, indices, np.asarray(part.mean)[indices] - factor) log(f"[unblind] {name}: subtracted {len(indices)} shifts") - for key in ("concealed", "blind", "blind_commitment", "blind_config_digest"): + for key in ( + "concealed", + "blind", + "blind_commitment", + "blind_config_digest", + "blind_draw_scheme", + ): part.metadata.pop(key, None) return part -def _stamp_provenance(s, commitment, label, config_digest): +def _stamp_provenance(s, commitment, label, config_digest, scheme=None): """Stamp the blind's public provenance; strip any leaked seed. ``blind_commitment`` (sha256 of the seed) ties the file to its blind without revealing it; ``blind_config_digest`` pins the envelope + - fiducial the shift was drawn against; ``concealed``/``blind`` mark the - file. ``seed_smokescreen`` — the raw seed Smokescreen's own writer would - stamp — is popped defensively: the seed must never ride a kept file. + fiducial the shift was drawn against; ``blind_draw_scheme`` pins the + Smokescreen draw semantics that turned the seed into that shift (see + :func:`draw_scheme`); ``concealed``/``blind`` mark the file. The three + together are what an unblind must reproduce. ``seed_smokescreen`` — the + raw seed upstream Smokescreen's writer would stamp — is popped + defensively: the seed must never ride a kept file. + + ``scheme`` defaults to the installed fork's, which is the right answer + whenever this call is stamping a blind that was just computed. Callers + propagating an existing blind's provenance pass that blind's recorded + scheme instead. """ s.metadata.pop("seed_smokescreen", None) s.metadata["concealed"] = True s.metadata["blind"] = label s.metadata["blind_commitment"] = commitment s.metadata["blind_config_digest"] = config_digest + s.metadata["blind_draw_scheme"] = draw_scheme() if scheme is None else int(scheme) def stamp_concealed_passthrough(s, commitment_path): @@ -489,12 +583,16 @@ def stamp_concealed_passthrough(s, commitment_path): values are blind by construction and only the provenance stamp is missing. ρ/τ carries no cosmological vector at all — the stamp merely clears it for assembly under the blind (values unchanged either way). Both cases need the - four custody keys the fail-closed load gate and :func:`assert_consistent_blind` + custody keys the fail-closed load gate and :func:`assert_consistent_blind` check: ``concealed``, ``blind`` (label), ``blind_commitment`` - (= ``seed_sha256``), ``blind_config_digest``. This reads those from the - version's ``commitment.json`` (written by :func:`blind_init`) and stamps them - via :func:`_stamp_provenance`, so a pass-through part shares the exact same - ``(commitment, digest)`` custody state as the blinded ξ±/pseudo-Cℓ parts. + (= ``seed_sha256``), ``blind_config_digest``, ``blind_draw_scheme``. This + reads those from the version's ``commitment.json`` (written by + :func:`blind_init`) and stamps them via :func:`_stamp_provenance`, so a + pass-through part shares the exact same custody state as the blinded + ξ±/pseudo-Cℓ parts. The committed draw scheme is checked against the + installed fork first: a pass-through part is blind because it was derived + from parts blinded under that scheme, so stamping it from an install that + draws differently would mint a custody claim this install cannot honour. Unlike :func:`blind_sacc`, this shifts nothing and does not require a blindable block — it is the seam for parts blinded (or made blind-irrelevant) @@ -502,8 +600,15 @@ def stamp_concealed_passthrough(s, commitment_path): """ with open(commitment_path, encoding="utf-8") as f: commitment = json.load(f) + _assert_draw_scheme( + commitment.get("draw_scheme"), f"the blind at {commitment_path}" + ) _stamp_provenance( - s, commitment["seed_sha256"], commitment["label"], commitment["config_digest"] + s, + commitment["seed_sha256"], + commitment["label"], + commitment["config_digest"], + scheme=commitment["draw_scheme"], ) return s @@ -518,9 +623,12 @@ def assert_consistent_blind(parts): part is *blindable* if it carries a blindable block (ξ± or pseudo-Cℓ_EE); ρ/τ diagnostic and covariance-only parts are exempt. Fails closed — ``ValueError`` — if blinded and plaintext blindable parts are mixed, or - if two parts carry different ``blind_commitment``/``blind_config_digest`` - (they were blinded under different seeds or configs and must never be - combined). The consistency key is ``(commitment, digest)`` only — the + if two parts carry different + ``blind_commitment``/``blind_config_digest``/``blind_draw_scheme`` (they + were blinded under different seeds, configs or draw semantics and must + never be combined), or if the shared draw scheme is not the installed + fork's (this install could not unblind what it is about to assemble). + The consistency key is ``(commitment, digest, scheme)`` — the ``blind`` *label* is informational provenance, not custody state, so parts blinded under one seed+config but tagged with different ``--label`` values assemble cleanly (a distinct warning is logged, not a failure). @@ -538,8 +646,9 @@ def assert_consistent_blind(parts): ------- dict or None The shared blind metadata (``concealed``, ``blind``, - ``blind_commitment``, ``blind_config_digest``) for the gather to - stamp on the assembled file, or ``None`` when nothing is blinded. + ``blind_commitment``, ``blind_config_digest``, ``blind_draw_scheme``) + for the gather to stamp on the assembled file, or ``None`` when + nothing is blinded. """ blindable = [p for p in parts if _blindable_blocks(p)] concealed = [p for p in blindable if _concealed(p)] @@ -562,18 +671,26 @@ def assert_consistent_blind(parts): f"({len(concealed)} of {len(blindable)} blinded) — refusing to " "combine (a plaintext part beside blinded ones leaks the shift)" ) - # Custody state is (commitment, digest) only — the label is provenance. + # Custody state is (commitment, digest, scheme) — the label is provenance. + # `.get` on the scheme so a part predating scheme binding reads as None and + # fails at _assert_draw_scheme with its explanation, not with a KeyError. stamps = { - (p.metadata["blind_commitment"], p.metadata["blind_config_digest"]) + ( + p.metadata["blind_commitment"], + p.metadata["blind_config_digest"], + p.metadata.get("blind_draw_scheme"), + ) for p in concealed } if len(stamps) != 1: raise ValueError( "parts carry different blind commitments — they were blinded " - "under different seeds or configs and must never be combined: " - + "; ".join(f"({c[:12]}…, {d[:12]}…)" for c, d in stamps) + "under different seeds, configs or draw schemes and must never be " + "combined: " + + "; ".join(f"({c[:12]}…, {d[:12]}…, scheme {v})" for c, d, v in stamps) ) - ((commitment, digest),) = stamps + ((commitment, digest, scheme),) = stamps + _assert_draw_scheme(scheme, "the blind these parts share") labels = sorted({p.metadata["blind"] for p in concealed}) if len(labels) != 1: warnings.warn( @@ -587,6 +704,7 @@ def assert_consistent_blind(parts): "blind": labels[0], "blind_commitment": commitment, "blind_config_digest": digest, + "blind_draw_scheme": int(scheme), } @@ -619,7 +737,9 @@ def blind_init(blind_dir, config=None, label="A", log=print): 1. Draw an OS-entropy seed (never written in plaintext, never returned). 2. Write ``commitment.json`` (repo-committable): ``sha256(seed)`` + the - canonical config digest + the blind label. + canonical config digest + the installed fork's draw scheme + the blind + label. Those first three are the full reproducibility statement — see + :func:`draw_scheme` for why the seed and config alone are not. 3. Encrypt the seed into a Fernet bundle (``smokescreen.encryption``); the temporary plaintext is deleted by the encryptor. @@ -650,6 +770,7 @@ def blind_init(blind_dir, config=None, label="A", log=print): "label": label, "seed_sha256": seed_commitment(seed), "config_digest": config.config_digest(), + "draw_scheme": draw_scheme(), } with open(paths["commitment"], "w", encoding="utf-8") as f: json.dump(commitment, f, indent=2, sort_keys=True) @@ -667,9 +788,12 @@ def blind_init(blind_dir, config=None, label="A", log=print): def _read_seed(blind_dir, config): """Decrypt the seed bundle and verify it against the commitment. - Both ``sha256(seed)`` and the config digest are checked before the seed - is handed to any caller — a tampered bundle or a drifted config fails - loud here, whether the caller is about to blind or to unblind. + ``sha256(seed)``, the config digest, and the committed draw scheme are all + checked before the seed is handed to any caller — a tampered bundle, a + drifted config or a Smokescreen that draws differently from the one that + fixed this blind all fail loud here, whether the caller is about to blind + or to unblind. The scheme check is what stops a re-blind of a later part + from landing a different hidden cosmology than the earlier parts got. Returns ------- @@ -680,6 +804,7 @@ def _read_seed(blind_dir, config): bundle = _read_encrypted_json(paths["bundle"], paths["key"]) with open(paths["commitment"], encoding="utf-8") as f: commitment = json.load(f) + _assert_draw_scheme(commitment.get("draw_scheme"), f"the blind in {blind_dir}") if seed_commitment(bundle["seed"]) != commitment["seed_sha256"]: raise ValueError( "bundle seed does not match the committed sha256(seed) — refusing " @@ -751,8 +876,9 @@ def blind_part(part_path, blind_dir, config=None, keep_input=False, log=print): def unblind_part(blinded_path, blind_dir, out_path, config=None, log=print): """Unblind one blinded part (or the assembled file), verifying first. - Decrypts the seed bundle and verifies both ``sha256(seed)`` and the - config digest against ``commitment.json`` (fail closed on either — + Decrypts the seed bundle and verifies ``sha256(seed)``, the config digest + and the draw scheme against ``commitment.json``, and the same three + against the blinded file's own stamps (fail closed on any — verification precedes subtraction), recomputes the part's shift from the seed and subtracts it (:func:`unblind_sacc`). **The seed-subtracted vector is the authority** — the seed plus its commitment is the custody diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 5cd0f0f7..a3cd9880 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -13,10 +13,10 @@ per-part-at-birth architecture; derived statistics (COSEBIs, pure-E/B) are never stored in parts — they are computed downstream from the (blinded) integration ξ± through the pipeline seams ``b_modes.cosebis_from_xi`` / -``b_modes.pure_eb_from_xi``, exactly as the pipeline does. The fork's -``ConcealDataVector`` carries no data-vector consistency check (only the -length guard), so fixture ξ± values are smooth synthetic templates — no -theory fill is needed to blind. +``b_modes.pure_eb_from_xi``, exactly as the pipeline does. The blinding path +hands the fork no data vector at all (``smokescreen.concealing_factor`` is a +pure theory difference), so fixture ξ± values are smooth synthetic templates — +no theory fill is needed to blind. """ import json @@ -418,6 +418,7 @@ def test_provenance_metadata_contract(monkeypatch): assert blinded.metadata["blind"] == "B" assert blinded.metadata["blind_commitment"] == bd.seed_commitment("seed") assert blinded.metadata["blind_config_digest"] == c.config_digest() + assert blinded.metadata["blind_draw_scheme"] == bd.draw_scheme() assert "seed_smokescreen" not in blinded.metadata assert blinded.metadata["catalogue_version"] == "vTEST" @@ -454,6 +455,170 @@ def test_unblind_fails_closed_on_wrong_seed_or_config(monkeypatch): bd.unblind_sacc(s, "right-seed", log=_NOLOG) +# --------------------------------------------------------------------------- # +# The vector core: full-length theory in, block slice out (fast — fake backend) +# --------------------------------------------------------------------------- # +def _fake_factory(indices, hole=None): + """A backend that fills ``indices`` with ``sigma8 * arange`` and nothing else. + + Mimics the real backends' contract — a block's ``theory_fn`` reads its + layout off the SACC it is given and returns a full-length vector, NaN on + every row outside its own block — without importing CCL. ``hole`` leaves + one row of the block itself unfilled. + """ + + def factory(s, theory): + def theory_fn(params): + out = np.full(len(s.mean), np.nan) + out[indices] = params["sigma8"] * np.arange(len(indices)) + if hole is not None: + out[hole] = np.nan + return out + + return theory_fn + + return factory + + +def test_concealing_factor_slices_its_own_block_from_a_full_length_vector(): + """The factor is the theory difference at the block's rows, and the rows + the backend does not fill are never read. + + This is the shape of the whole path after the sub-SACC carving came out: + the backend is driven off the assembled SACC directly, returns a + full-length vector that is NaN everywhere but its own block, and + ``_concealing_factor`` returns exactly that block's rows. Checked against + :func:`hidden_params`, which reaches the same hidden point by an + independent route. + """ + ordered = ["xi_reporting", "cl", "rho_tau", "xi_integration"] + s = sio.gather([make_parts(nbins=1)[k] for k in ordered]) + blocks = bd._blindable_blocks(s) + assert len(blocks) == 3, "assembled file carries all three blindable blocks" + + cfg = bd.BlindingConfig() + delta = bd.hidden_params("seed", cfg)["sigma8"] - cfg.theory.ccl_params()["sigma8"] + assert delta != 0.0 + for _, indices, _ in blocks: + factor = bd._concealing_factor(s, indices, _fake_factory(indices), cfg, "seed") + assert factor.shape == (len(indices),) + assert np.allclose(factor, delta * np.arange(len(indices))) + + +def test_concealing_factor_refuses_a_row_its_backend_cannot_fill(): + """A NaN on a row the block *claims* is a layout the backend cannot cover. + + Slicing to the block drops the NaNs outside it by construction; this is + the converse guard, and it has to be explicit — without it a row the + backend silently skipped would be shifted by NaN, destroying that point + with no error anywhere. + """ + part = make_xi_part("reporting") + ((_, indices, _),) = bd._blindable_blocks(part) + with pytest.raises(ValueError, match="unfilled"): + bd._concealing_factor( + part, + indices, + _fake_factory(indices, hole=indices[0]), + bd.BlindingConfig(), + "seed", + ) + + +# --------------------------------------------------------------------------- # +# Draw-scheme binding: the blind is (seed, config, draw semantics) +# --------------------------------------------------------------------------- # +def test_draw_scheme_is_the_installed_fork_constant(): + """The recorded scheme is read from Smokescreen, not hardcoded here.""" + import smokescreen + + assert bd.draw_scheme() == int(smokescreen.DRAW_SCHEME) + assert isinstance(bd.draw_scheme(), int) + + +def test_assert_draw_scheme_message_names_both_versions(monkeypatch): + """A scheme mismatch says which scheme made the blind and which is installed.""" + monkeypatch.setattr(bd, "draw_scheme", lambda: 2) + bd._assert_draw_scheme(2, "the blind") # matching scheme is silent + with pytest.raises(ValueError, match=r"DRAW_SCHEME=1.*DRAW_SCHEME=2"): + bd._assert_draw_scheme(1, "the blind") + with pytest.raises(ValueError, match="no draw-scheme record"): + bd._assert_draw_scheme(None, "the blind") + + +def test_unblind_refuses_a_blind_drawn_under_another_scheme(monkeypatch): + """The finding this closes: a blind added under one draw scheme and + subtracted under another silently produces a wrong data vector, because + the seed hash, the config digest and the escrow check all still pass. The + scheme must be part of the custody state, checked before any subtraction. + """ + _patch_constant_factor(monkeypatch) + blinded = bd.blind_sacc(make_xi_part("reporting"), "seed", log=_NOLOG) + assert blinded.metadata["blind_draw_scheme"] == bd.draw_scheme() + + # everything else about this file is valid — only the draw semantics moved + monkeypatch.setattr( + bd, "draw_scheme", lambda: blinded.metadata["blind_draw_scheme"] + 1 + ) + with pytest.raises(ValueError, match="DRAW_SCHEME"): + bd.unblind_sacc(blinded, "seed", log=_NOLOG) + + +def test_unblind_refuses_a_file_predating_scheme_binding(monkeypatch): + """A blinded file with no scheme record fails closed, not open.""" + _patch_constant_factor(monkeypatch) + blinded = bd.blind_sacc(make_xi_part("reporting"), "seed", log=_NOLOG) + del blinded.metadata["blind_draw_scheme"] + with pytest.raises(ValueError, match="no draw-scheme record"): + bd.unblind_sacc(blinded, "seed", log=_NOLOG) + + +def test_unblind_strips_the_scheme_stamp_with_the_rest(monkeypatch): + _patch_constant_factor(monkeypatch) + blinded = bd.blind_sacc(make_xi_part("reporting"), "seed", log=_NOLOG) + part = bd.unblind_sacc(blinded, "seed", log=_NOLOG) + assert "blind_draw_scheme" not in part.metadata + + +def test_read_seed_fails_closed_on_scheme_drift(tmp_path, monkeypatch): + """blind_part reads the seed through _read_seed, so a scheme change between + blinding part 1 and part 2 is caught before the second part is shifted.""" + bd.blind_init(str(tmp_path), log=_NOLOG) + monkeypatch.setattr(bd, "draw_scheme", lambda: 99) + with pytest.raises(ValueError, match="DRAW_SCHEME"): + bd._read_seed(str(tmp_path), bd.BlindingConfig()) + + +def test_stamp_passthrough_carries_the_committed_scheme(tmp_path, monkeypatch): + """A pass-through part inherits the blind's scheme, and cannot be stamped + from an install that draws differently.""" + paths = bd.blind_init(str(tmp_path), log=_NOLOG) + s = bd.stamp_concealed_passthrough(make_rho_part(), paths["commitment"]) + assert s.metadata["blind_draw_scheme"] == bd.draw_scheme() + + monkeypatch.setattr(bd, "draw_scheme", lambda: 99) + with pytest.raises(ValueError, match="DRAW_SCHEME"): + bd.stamp_concealed_passthrough(make_rho_part(), paths["commitment"]) + + +def test_assert_consistent_blind_refuses_divergent_or_foreign_schemes(monkeypatch): + """Parts blinded under different schemes never assemble; nor does a set + that agrees with itself but not with the installed fork.""" + parts = make_parts(nbins=1, with_rho=False) + for p in parts.values(): + _stamp(p) + parts["cl"].metadata["blind_draw_scheme"] = bd.draw_scheme() + 1 + with pytest.raises(ValueError, match="different blind commitments"): + bd.assert_consistent_blind(list(parts.values())) + + parts = make_parts(nbins=1, with_rho=False) + for p in parts.values(): + _stamp(p) + p.metadata["blind_draw_scheme"] = bd.draw_scheme() + 1 + with pytest.raises(ValueError, match="DRAW_SCHEME"): + bd.assert_consistent_blind(list(parts.values())) + + # --------------------------------------------------------------------------- # # blind-init custody + assembly hash assertion (fast — encryption only) # --------------------------------------------------------------------------- # @@ -462,7 +627,7 @@ def test_blind_init_writes_commitment_and_encrypted_bundle_only(tmp_path): paths = bd.blind_init(str(tmp_path), log=_NOLOG) with open(paths["commitment"], encoding="utf-8") as f: commitment = json.load(f) - assert set(commitment) == {"label", "seed_sha256", "config_digest"} + assert set(commitment) == {"label", "seed_sha256", "config_digest", "draw_scheme"} assert len(commitment["seed_sha256"]) == 64 assert commitment["config_digest"] == bd.BlindingConfig().config_digest() # exactly the three custody outputs, no plaintext bundle @@ -507,6 +672,7 @@ def test_assert_consistent_blind_shared_stamp(): "blind": "A", "blind_commitment": bd.seed_commitment("s"), "blind_config_digest": bd.BlindingConfig().config_digest(), + "blind_draw_scheme": bd.draw_scheme(), } @@ -681,8 +847,12 @@ def test_ac2_on_file_shift_equals_theory_difference_per_part(transfer_function): hiddens.append(hidden) fiducial = cfg.theory.ccl_params() ((block_name, indices, factory),) = bd._blindable_blocks(part) - theory = factory(bd._extract_block(part, indices), cfg.theory) - expected = theory(hidden) - theory(fiducial) + # Independent of the blinding path: the factory is driven directly off + # the part, at the hidden point recovered by hidden_params, and the two + # theory vectors are differenced here rather than by the fork. The + # factory fills only its own block, so slice to it. + theory = factory(part, cfg.theory) + expected = (theory(hidden) - theory(fiducial))[indices] actual = np.array(blinded.mean)[indices] - np.array(part.mean)[indices] gap = np.max(np.abs(actual - expected)) scale = np.max(np.abs(expected)) @@ -989,7 +1159,7 @@ def test_ac6_ac8_end_to_end_init_parts_gather_unblind(tmp_path): assert not np.array_equal(np.array(blinded.mean), np.array(parts[name].mean)) with open(init["commitment"], encoding="utf-8") as f: commitment = json.load(f) - assert set(commitment) == {"label", "seed_sha256", "config_digest"} + assert set(commitment) == {"label", "seed_sha256", "config_digest", "draw_scheme"} blinded_parts = {n: sio.load(p["blinded"]) for n, p in out_paths.items()} for b in blinded_parts.values(): assert b.metadata["blind_commitment"] == commitment["seed_sha256"] From 7cbfe7300c2b709a447d66fb1e045d09623c5011 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 17 Aug 2026 20:22:12 +0200 Subject: [PATCH 031/160] One terminal seam: route the production assembly through sacc_io.gather MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `sacc_io.gather` was the documented custody terminal — it asserts one blind across all blindable parts, then assembles — but nothing in production called it. `workflow/scripts/assemble_sacc.py` inlined its own copy of the same wrapper (assert, assemble, stamp) around a *different* assembler, and blinding.py's docstrings claimed gather guarded the terminal. Two custody implementations, one of them dead, and a docstring true of neither. The difference between them is only the assembly: `merge` unions parts that already carry covariance; `assemble_analysis_sacc` rebuilds from an n(z) and requires one covariance block per part, which is what the production terminal needs (its ξ± and pseudo-Cℓ parts are born cov-less and have blocks injected first). That is a real difference and both are worth keeping — so the assembler becomes an argument, `gather(parts, metadata=None, assemble=None)`, and assemble_sacc.py passes its own in rather than wrapping its own guard around it. The custody wrapper now exists once and cannot be routed around. Tests: the production path gets the case the fail-closed load gate cannot catch — every part concealed, so every part loads, and only `assert_consistent_blind` can see that the install's draw scheme is not the blind's. Plus the terminal file's draw-scheme stamp. Closes review finding 3 on PR #253. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BRYg9fjMevgcsvgjh3KN6x --- src/sp_validation/sacc_io.py | 43 ++++++++++++------- .../tests/test_blinding_wiring.py | 28 ++++++++++++ workflow/scripts/assemble_sacc.py | 35 +++++++-------- 3 files changed, 74 insertions(+), 32 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 0923326e..7dcb72a7 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -994,25 +994,35 @@ def load(path, *, allow_unblinded=False): # Everything above this banner is byte-identical to feat/sacc-2-sacc-io; only # gather() and its blind-custody call site live here. # --------------------------------------------------------------------------- # -def gather(parts, metadata=None): +def gather(parts, metadata=None, assemble=None): """Assemble standalone part SACCs into the one-file ``{version}.sacc``. Each part is an intermediate product as it came off its producing rule - (reporting ξ±, integration ξ±, pseudo-Cℓ, ρ/τ, …). Assembly itself — the - first-wins tracer union, in-order point concatenation with all tags - (bandpower windows included), and the block-diagonal covariance — is - exactly :func:`merge`, so gather delegates to it and adds only the one - thing merge cannot know about: **blind custody.** + (reporting ξ±, integration ξ±, pseudo-Cℓ, ρ/τ, …). Gather is **the** terminal + seam: every path that combines parts into the one-file product goes + through here, because this is where the one thing an assembler cannot know + about is enforced — **blind custody.** - **Blind custody (the module's one blind-aware call site):** + **Blind custody.** :func:`sp_validation.blinding.assert_consistent_blind` runs before the - merge — it fails closed unless every blindable part carries the identical - ``blind_commitment``/``blind_config_digest`` (or, when nothing is blinded, - every blindable part is declared ``type='mock'``). Its returned shared - stamp is written onto the assembled file so the one-file product carries - the blind it was built from; the blinded parts already carry those keys, - so merge preserves them and this stamp is a consistent (idempotent) - re-affirmation. + assembly — it fails closed unless every blindable part carries the + identical ``blind_commitment``/``blind_config_digest``/``blind_draw_scheme`` + (or, when nothing is blinded, every blindable part is declared + ``type='mock'``). Its returned shared stamp is written onto the assembled + file so the one-file product carries the blind it was built from; the + blinded parts already carry those keys, so the assembly preserves them and + this stamp is a consistent (idempotent) re-affirmation. + + **The assembly itself is the caller's.** Two exist and both are real: + :func:`merge` (the default) does the first-wins tracer union, in-order + point concatenation with all tags, and a covariance built from whatever the + parts carry — the right thing when parts are already covariance-bearing. + :func:`sp_validation.cosmo_val.sacc_writers.assemble_analysis_sacc` rebuilds + from a given n(z) + metadata and *requires* one covariance block per part — + the right thing for the production terminal, where the ξ± and pseudo-Cℓ + parts are born cov-less and have their blocks injected first. Passing the + assembler in, rather than duplicating the custody wrapper around each one, + is what keeps the guard un-bypassable. Parameters ---------- @@ -1020,6 +1030,9 @@ def gather(parts, metadata=None): The part SACCs, in the assembly (covariance) order. metadata : dict, optional Extra key/value pairs to store on the assembled file's metadata. + assemble : callable, optional + ``assemble(parts) -> sacc.Sacc``. Defaults to :func:`merge`. Bind any + further arguments (n(z), metadata) into the callable. Returns ------- @@ -1030,7 +1043,7 @@ def gather(parts, metadata=None): parts = list(parts) stamp = blinding.assert_consistent_blind(parts) - s = merge(parts) + s = (assemble or merge)(parts) for key, value in {**(metadata or {}), **(stamp or {})}.items(): s.metadata[key] = value return s diff --git a/src/sp_validation/tests/test_blinding_wiring.py b/src/sp_validation/tests/test_blinding_wiring.py index 2eac1988..706200cf 100644 --- a/src/sp_validation/tests/test_blinding_wiring.py +++ b/src/sp_validation/tests/test_blinding_wiring.py @@ -200,6 +200,34 @@ def test_data_assemble_refuses_blinded_plaintext_mix(tmp_path): ) +def test_data_assemble_runs_behind_the_custody_guard(tmp_path, monkeypatch): + """The custody guard is reached *through* the production assembly. + + Every part here is concealed, so every part clears the fail-closed load + gate — the earlier tests all stop there. Only + ``blinding.assert_consistent_blind`` can catch what is wrong with this + assembly: the install's Smokescreen draws shifts under a different scheme + than the one that made the blind, so it could never unblind what it is + about to write. That ``assemble_sacc`` raises is the check that it runs + through :func:`sacc_io.gather` like every other assembly path, rather than + reimplementing the custody wrapper around its own assembler. + """ + paths = _data_parts(tmp_path, conceal=True) + monkeypatch.setattr(blinding, "draw_scheme", lambda: 99) + with pytest.raises(ValueError, match="DRAW_SCHEME"): + asm.assemble_sacc( + "vSYNTH", paths, str(tmp_path / "vSYNTH.sacc"), placeholder_var=1.0 + ) + + +def test_data_assemble_stamps_the_draw_scheme_on_the_terminal_file(tmp_path): + """The assembled file carries the blind's draw scheme, like its parts.""" + paths = _data_parts(tmp_path, conceal=True) + out = tmp_path / "vSYNTH.sacc" + asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + assert sio.load(str(out)).metadata["blind_draw_scheme"] == blinding.draw_scheme() + + def test_assert_consistent_blind_rejects_divergent_commitments(tmp_path): """Two ξ± parts blinded under different commitments must never combine.""" nz = {0: _nz()} diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 118ec5b8..4c7d41a1 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -7,9 +7,13 @@ Each per-statistic ``*.sacc`` *part* (written born-as-SACC by the mixins and the run_2pcf / generate_pseudo_cl scripts) holds one statistic. The assembler loads them in canonical order — ξ± reporting, pseudo-Cℓ, COSEBIs, pure-E/B, ρ/τ — and -calls :func:`sacc_writers.assemble_analysis_sacc`, which rebuilds one Sacc with a -single ``BlockDiagonalCovariance`` (point-insertion order = block order, -validated by ``sacc_io.assemble_covariance``). +hands them to :func:`sacc_io.gather` along with +:func:`sacc_writers.assemble_analysis_sacc` as the assembly, which rebuilds one +Sacc with a single ``BlockDiagonalCovariance`` (point-insertion order = block +order, validated by ``sacc_io.assemble_covariance``). Going through ``gather`` +rather than calling the assembler directly is what puts this path behind the +blind-custody gate (``blinding.assert_consistent_blind``) — there is one +terminal seam, not one per assembler. Covariance sourcing (the part-by-part decision) ----------------------------------------------- @@ -176,20 +180,17 @@ def assemble_sacc( ) if not parts: raise ValueError(f"no parts found for {version}: {part_paths}") - # Assembly-time custody assertion (#252): every blindable part (ξ± / pseudo-Cℓ - # EE) must share one blind commitment + config digest, or assembly fails - # closed — mixed blinded/plaintext parts and divergent-seed parts both raise. - # ρ/τ and covariance-only parts are exempt. Returns the shared blind stamp to - # carry onto the assembled file (or None for a fully-mock plaintext assembly). - from sp_validation import blinding - - shared = blinding.assert_consistent_blind(parts) - s = assemble_analysis_sacc(nz, metadata, parts) - if shared is not None: - # Stamp the assembled file with the shared blind so it, too, reads as - # concealed (its parts already carried the stamp into `metadata` above; - # this makes the custody state explicit and authoritative on the union). - s.metadata.update(shared) + # Assembly-time custody assertion (#252) runs inside sacc_io.gather, the one + # terminal seam: every blindable part (ξ± / pseudo-Cℓ EE) must share one + # blind commitment + config digest + draw scheme, or assembly fails closed — + # mixed blinded/plaintext parts and divergent-seed parts both raise. ρ/τ and + # covariance-only parts are exempt. Gather also stamps the shared blind onto + # the assembled file, so the union reads as concealed in its own right. + # The production assembly (n(z) + per-part covariance blocks) is passed in; + # gather's own default merge() is for parts that already carry covariance. + s = sacc_io.gather( + parts, assemble=lambda ordered: assemble_analysis_sacc(nz, metadata, ordered) + ) # Assembly preserves its parts' provenance: every part was written by # sacc_io.save and therefore carries the type=data|mock stamp in its # metadata (copied into the assembled file above). From 646349c97da6e1905938603a07a961fc879410e8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 17 Aug 2026 21:42:14 +0200 Subject: [PATCH 032/160] Take the commitment from the fork, so it stops publishing the seed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The local sha256(seed) commitment leaked the blind. Smokescreen derives its per-key RNG base seed from the first 8 bytes of the *same* undomained digest, so the published commitment carried that base seed verbatim in its first 16 hex characters. With the commitment and the (public) fiducial config, anyone could redraw the hidden cosmology and subtract the shift — from the artifact introduced to protect it. seed_commitment now delegates to smokescreen.seed_commitment, which hashes COMMITMENT_DOMAIN + str(seed). One definition of the commitment across the stack, in a different hash domain from the seed normalizer. A regression test asserts the commitment never embeds _normalize_seed(seed). The commitment.json key is renamed seed_sha256 -> seed_commitment: it is no longer sha256 of the seed, and the old name claimed otherwise. No real blind exists, so nothing migrates. Also, four honesty fixes the same review found: - The NaN refusal fires on both blind and unblind (they share one call), so its message no longer says "refusing to blind". - The CLI module docstring describes the three-way commitment (seed, config, draw scheme), and _verify's docstring says that its draw-scheme check makes the result environment-dependent by design. - _verify loads with allow_unblinded=True, so its "file is not marked concealed" diagnostic is reachable instead of an uncaught load exception. - The module docstring states what a pre-draw_scheme blind costs: recoverable only by hand from escrow. None exists; the CLI offers no override. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BRYg9fjMevgcsvgjh3KN6x --- scripts/blind_data_vector.py | 44 ++++++++++----- src/sp_validation/blinding.py | 68 +++++++++++++++--------- src/sp_validation/tests/test_blinding.py | 61 +++++++++++++++++---- workflow/rules/blinding.smk | 6 +-- 4 files changed, 129 insertions(+), 50 deletions(-) diff --git a/scripts/blind_data_vector.py b/scripts/blind_data_vector.py index d9aeb502..692ebd94 100644 --- a/scripts/blind_data_vector.py +++ b/scripts/blind_data_vector.py @@ -7,15 +7,20 @@ ``blind-init`` runs once per catalogue version: it draws an OS-entropy seed (never printed, never written in plaintext), publishes a repo-committable -``commitment.json`` (``sha256(seed)`` + config digest), and encrypts the seed -into a Fernet bundle. ``blind-part`` blinds one intermediate part SACC -(reporting ξ±, integration ξ±, or pseudo-Cℓ) under that fixed state, escrows -the true vector into a per-part encrypted bundle beside the blinded output, -and deletes the plaintext part. ``unblind`` verifies both commitment hashes -and restores a true part (bit-for-bit when the part's escrow bundle is beside -it); it also works on the assembled ``{version}.sacc`` (integration rows -selected by the ``grid`` tag). ``verify`` is a cheap, seedless check that a -blinded file matches a commitment. +``commitment.json`` (the seed commitment + the config digest + the installed +Smokescreen fork's draw scheme), and encrypts the seed into a Fernet bundle. +Those three, not the seed alone, are what reproduces a blind: seed and config +fix *which* shift, the draw scheme fixes *how* the seed becomes that shift. +``blind-part`` blinds one intermediate part SACC (reporting ξ±, integration ξ±, +or pseudo-Cℓ) under that fixed state, escrows the true vector into a per-part +encrypted bundle beside the blinded output, and deletes the plaintext part. +``unblind`` verifies all three commitments and restores a true part +(bit-for-bit when the part's escrow bundle is beside it); it also works on the +assembled ``{version}.sacc`` (integration rows selected by the ``grid`` tag). +``verify`` is a cheap, seedless check that a blinded file matches a commitment +— seedless, but not environment-independent: it also compares both recorded +draw schemes against the installed fork, so it reports a problem on a machine +whose Smokescreen draws differently even when file and commitment agree. :Authors: Cail Daley @@ -102,15 +107,28 @@ def _unblind(args): def _verify(args): - """Seedless check: blinded-file metadata ↔ commitment JSON.""" - s = sacc_io.load(args.blinded) + """Seedless check: blinded-file metadata ↔ commitment JSON ↔ this install. + + No seed is read, so this cannot confirm that the blind is *subtractable* — + only that the file's three custody stamps agree with the commitment, and + that the recorded draw scheme is the one this install implements. That last + check makes the result environment-dependent by design: a machine carrying a + different Smokescreen reports a problem even when file and commitment agree + perfectly, because that machine could not unblind the file. + + Loads with ``allow_unblinded=True``: the whole job here is to report on a + file's custody state, including the state where the file is not concealed at + all, which the fail-closed loader would otherwise raise on before any + diagnostic could be assembled. + """ + s = sacc_io.load(args.blinded, allow_unblinded=True) with open(args.commitment, encoding="utf-8") as f: commitment = json.load(f) problems = [] if not s.metadata.get("concealed"): problems.append("file is not marked concealed") - if s.metadata.get("blind_commitment") != commitment["seed_sha256"]: - problems.append("blind_commitment does not match the committed sha256(seed)") + if s.metadata.get("blind_commitment") != commitment["seed_commitment"]: + problems.append("blind_commitment does not match the committed seed commitment") if s.metadata.get("blind_config_digest") != commitment["config_digest"]: problems.append("blind_config_digest does not match the committed digest") # The draw scheme is checked three ways: file ↔ commitment, and both against diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index c2ff148f..34b648d2 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -42,12 +42,13 @@ **Custody: hash commitment, no keyholder.** :func:`blind_init` runs once per catalogue version: it draws an OS-entropy seed, publishes - ``sha256(seed)``, a canonical config digest, and the installed fork's - ``DRAW_SCHEME`` as a repo-committable ``commitment.json``, and encrypts - the seed into a Fernet bundle (``smokescreen.encryption``) — the plaintext - seed is never written. The three together, not the seed alone, are what - reproduces a blind: seed and config fix *which* shift, the draw scheme - fixes *how* the seed becomes that shift. + the seed commitment (:func:`seed_commitment`), a canonical config digest, + and the installed fork's ``DRAW_SCHEME`` as a repo-committable + ``commitment.json``, and encrypts the seed into a Fernet bundle + (``smokescreen.encryption``) — the plaintext seed is never written. The + three together, not the seed alone, are what reproduces a blind: seed and + config fix *which* shift, the draw scheme fixes *how* the seed becomes that + shift. Each :func:`blind_part` call reads that fixed state, conceals one part, escrows the part's true vector into its own encrypted bundle beside the @@ -59,6 +60,15 @@ carries the identical ``blind_commitment``, ``blind_config_digest`` and ``blind_draw_scheme``. :func:`unblind_part` verifies all three against the commitment *before* subtracting anything, then restores the true part. + + Custody has no back door, including for its own history: a blind fixed + before the draw scheme was bound into ``commitment.json`` carries no + scheme record, and :func:`_assert_draw_scheme` refuses it ahead of every + seed read, so neither :func:`blind_part` nor :func:`unblind_part` will run + on it. Such a blind is recoverable only by hand, by decrypting its per-part + escrow bundles (:func:`_read_encrypted_json`) and reading ``true_mean`` + back out. No such blind exists — the scheme has been bound since before + the first real blind — and the CLI deliberately offers no override. """ import dataclasses @@ -125,7 +135,7 @@ def config_digest(self): digests and deny a legitimate unblind. Python's ``json`` then emits each float via its shortest round-trip ``repr``, so two runs of the same config produce byte-identical digests. Checked (with - ``sha256(seed)``) at unblind, so a wrong envelope or a mismatched + the seed commitment) at unblind, so a wrong envelope or a mismatched P(k) recipe cannot silently subtract a wrong shift. """ payload = { @@ -172,12 +182,22 @@ def from_overrides(cls, overrides): # Custody primitives # --------------------------------------------------------------------------- # def seed_commitment(seed): - """Public commitment for a seed: its sha256 hex digest. + """Public commitment for a seed: the fork's domain-separated sha256 digest. Safe to publish and commit to the repo — it ties a blinded file to its blind without revealing the seed, and lets unblind refuse a wrong seed. + + This delegates to :func:`smokescreen.seed_commitment` so the commitment has + exactly one definition across the blinding stack. That function hashes + ``smokescreen.COMMITMENT_DOMAIN + str(seed)``, *not* the bare seed, and the + domain prefix is load-bearing: the fork derives the RNG base seed from the + bare sha256 of the same string, so an undomained commitment would publish + that base seed verbatim in its first 16 hex characters and the blind would + be recoverable from the artifact meant to protect it. """ - return hashlib.sha256(seed.encode("utf-8")).hexdigest() + from smokescreen import seed_commitment as _fork_seed_commitment + + return _fork_seed_commitment(seed) def draw_scheme(): @@ -454,9 +474,9 @@ def _concealing_factor(s, indices, factory, config, seed): if not np.all(np.isfinite(factor)): raise ValueError( f"the theory backend left {int(np.sum(~np.isfinite(factor)))} of " - f"{len(indices)} blindable rows unfilled — refusing to blind " - "(these rows would be shifted by NaN). The block's row layout is " - "not fully covered by its theory_fn." + f"{len(indices)} blindable rows unfilled — refusing to apply the " + "concealing factor (these rows would be shifted by NaN). The " + "block's row layout is not fully covered by its theory_fn." ) return factor @@ -508,7 +528,7 @@ def unblind_sacc(blinded, seed, config=None, log=print): """Recover the true part SACC from a blinded one + the revealed ``seed``. Verifies the stamped draw scheme against the installed fork, then - ``sha256(seed)`` and the config digest against the stamped metadata (loud + the seed commitment and the config digest against the stamped metadata (loud failure on any of the three — verification precedes subtraction), then recomputes each block's shift through :func:`_concealing_factor`, the same call :func:`blind_sacc` added it with, and subtracts it. Works on a @@ -585,7 +605,7 @@ def stamp_concealed_passthrough(s, commitment_path): assembly under the blind (values unchanged either way). Both cases need the custody keys the fail-closed load gate and :func:`assert_consistent_blind` check: ``concealed``, ``blind`` (label), ``blind_commitment`` - (= ``seed_sha256``), ``blind_config_digest``, ``blind_draw_scheme``. This + (= ``seed_commitment``), ``blind_config_digest``, ``blind_draw_scheme``. This reads those from the version's ``commitment.json`` (written by :func:`blind_init`) and stamps them via :func:`_stamp_provenance`, so a pass-through part shares the exact same custody state as the blinded @@ -605,7 +625,7 @@ def stamp_concealed_passthrough(s, commitment_path): ) _stamp_provenance( s, - commitment["seed_sha256"], + commitment["seed_commitment"], commitment["label"], commitment["config_digest"], scheme=commitment["draw_scheme"], @@ -736,7 +756,7 @@ def blind_init(blind_dir, config=None, label="A", log=print): Runs once per catalogue version: 1. Draw an OS-entropy seed (never written in plaintext, never returned). - 2. Write ``commitment.json`` (repo-committable): ``sha256(seed)`` + the + 2. Write ``commitment.json`` (repo-committable): the seed commitment + the canonical config digest + the installed fork's draw scheme + the blind label. Those first three are the full reproducibility statement — see :func:`draw_scheme` for why the seed and config alone are not. @@ -768,7 +788,7 @@ def blind_init(blind_dir, config=None, label="A", log=print): seed = secrets.token_hex(16) commitment = { "label": label, - "seed_sha256": seed_commitment(seed), + "seed_commitment": seed_commitment(seed), "config_digest": config.config_digest(), "draw_scheme": draw_scheme(), } @@ -788,7 +808,7 @@ def blind_init(blind_dir, config=None, label="A", log=print): def _read_seed(blind_dir, config): """Decrypt the seed bundle and verify it against the commitment. - ``sha256(seed)``, the config digest, and the committed draw scheme are all + The seed commitment, the config digest, and the committed draw scheme are all checked before the seed is handed to any caller — a tampered bundle, a drifted config or a Smokescreen that draws differently from the one that fixed this blind all fail loud here, whether the caller is about to blind @@ -805,9 +825,9 @@ def _read_seed(blind_dir, config): with open(paths["commitment"], encoding="utf-8") as f: commitment = json.load(f) _assert_draw_scheme(commitment.get("draw_scheme"), f"the blind in {blind_dir}") - if seed_commitment(bundle["seed"]) != commitment["seed_sha256"]: + if seed_commitment(bundle["seed"]) != commitment["seed_commitment"]: raise ValueError( - "bundle seed does not match the committed sha256(seed) — refusing " + "bundle seed does not match the committed seed commitment — refusing " "to proceed" ) if config.config_digest() != commitment["config_digest"]: @@ -856,7 +876,7 @@ def blind_part(part_path, blind_dir, config=None, keep_input=False, log=print): paths["escrow"], { "label": commitment["label"], - "seed_sha256": commitment["seed_sha256"], + "seed_commitment": commitment["seed_commitment"], "true_mean": np.asarray(part.mean, dtype=float).tolist(), }, ) @@ -876,7 +896,7 @@ def blind_part(part_path, blind_dir, config=None, keep_input=False, log=print): def unblind_part(blinded_path, blind_dir, out_path, config=None, log=print): """Unblind one blinded part (or the assembled file), verifying first. - Decrypts the seed bundle and verifies ``sha256(seed)``, the config digest + Decrypts the seed bundle and verifies the seed commitment, the config digest and the draw scheme against ``commitment.json``, and the same three against the blinded file's own stamps (fail closed on any — verification precedes subtraction), recomputes the part's shift from the @@ -885,7 +905,7 @@ def unblind_part(blinded_path, blind_dir, out_path, config=None, log=print): root of trust, and the returned vector is what the seed math produced. When the part's escrow bundle exists beside the blinded file it serves - two subordinate roles, and only after its stored ``seed_sha256`` is + two subordinate roles, and only after its stored ``seed_commitment`` is verified to match the same commitment (so an escrow from a different blind can never be trusted): (1) a *tighter equality check* — the seed subtraction and the escrowed truth must agree to ``1e-6`` relative or the @@ -910,7 +930,7 @@ def unblind_part(blinded_path, blind_dir, out_path, config=None, log=print): escrow = part_paths(unblinded_stem + ext) if os.path.exists(escrow["escrow"]): bundle = _read_encrypted_json(escrow["escrow"], escrow["escrow_key"]) - if bundle.get("seed_sha256") != commitment["seed_sha256"]: + if bundle.get("seed_commitment") != commitment["seed_commitment"]: raise ValueError( "escrow bundle beside the blinded file was written under a " "different seed than the commitment — refusing to trust it " diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index a3cd9880..6d808895 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -334,14 +334,45 @@ def test_theory_config_ccl_params_exact_keyset(): # --------------------------------------------------------------------------- # # Commitment + the fork's draw (fast; smokescreen import is light) # --------------------------------------------------------------------------- # -def test_commitment_is_sha256_of_seed(): +def test_commitment_is_the_forks_domain_separated_digest(): + """One definition of the commitment, and it is the fork's.""" import hashlib + import smokescreen + seed = "the-secret" - assert bd.seed_commitment(seed) == hashlib.sha256(seed.encode()).hexdigest() + assert bd.seed_commitment(seed) == smokescreen.seed_commitment(seed) + assert ( + bd.seed_commitment(seed) + == hashlib.sha256( + smokescreen.COMMITMENT_DOMAIN + seed.encode("utf-8") + ).hexdigest() + ) assert bd.seed_commitment("right") != bd.seed_commitment("wrong") +def test_commitment_does_not_embed_the_rng_seed(): + """The published commitment must not carry the effective RNG seed. + + The fork derives the base seed for its per-key RNG from the *undomained* + sha256 of the seed string, taking the digest's first 8 bytes. A commitment + hashed over the bare seed would therefore publish that base seed verbatim in + its first 16 hex characters, and anyone holding the (public) commitment and + the (public) fiducial config could redraw the hidden cosmology and subtract + the blind. The domain prefix is what breaks that identity — this is the + regression guard for it. + """ + import secrets + + from smokescreen.param_shifts import _normalize_seed + + # The third seed is drawn exactly as blind_init draws a production one. + for seed in ("my_secret_seed", "the-secret", secrets.token_hex(16)): + commitment = bd.seed_commitment(seed) + assert int(commitment[:16], 16) != _normalize_seed(seed) + assert str(_normalize_seed(seed)) not in commitment + + def test_hidden_params_deterministic_and_in_envelope(): """Same (seed, config) ⇒ same hidden point; draws respect the envelope.""" import secrets @@ -627,8 +658,13 @@ def test_blind_init_writes_commitment_and_encrypted_bundle_only(tmp_path): paths = bd.blind_init(str(tmp_path), log=_NOLOG) with open(paths["commitment"], encoding="utf-8") as f: commitment = json.load(f) - assert set(commitment) == {"label", "seed_sha256", "config_digest", "draw_scheme"} - assert len(commitment["seed_sha256"]) == 64 + assert set(commitment) == { + "label", + "seed_commitment", + "config_digest", + "draw_scheme", + } + assert len(commitment["seed_commitment"]) == 64 assert commitment["config_digest"] == bd.BlindingConfig().config_digest() # exactly the three custody outputs, no plaintext bundle assert {p.name for p in tmp_path.iterdir()} == { @@ -638,7 +674,7 @@ def test_blind_init_writes_commitment_and_encrypted_bundle_only(tmp_path): } # the decrypted seed matches the public commitment bundle = bd._read_encrypted_json(paths["bundle"], paths["key"]) - assert bd.seed_commitment(bundle["seed"]) == commitment["seed_sha256"] + assert bd.seed_commitment(bundle["seed"]) == commitment["seed_commitment"] # one-shot custody: a second init in the same dir refuses with pytest.raises(FileExistsError, match="refusing to overwrite"): bd.blind_init(str(tmp_path), log=_NOLOG) @@ -1159,10 +1195,15 @@ def test_ac6_ac8_end_to_end_init_parts_gather_unblind(tmp_path): assert not np.array_equal(np.array(blinded.mean), np.array(parts[name].mean)) with open(init["commitment"], encoding="utf-8") as f: commitment = json.load(f) - assert set(commitment) == {"label", "seed_sha256", "config_digest", "draw_scheme"} + assert set(commitment) == { + "label", + "seed_commitment", + "config_digest", + "draw_scheme", + } blinded_parts = {n: sio.load(p["blinded"]) for n, p in out_paths.items()} for b in blinded_parts.values(): - assert b.metadata["blind_commitment"] == commitment["seed_sha256"] + assert b.metadata["blind_commitment"] == commitment["seed_commitment"] # no plaintext json anywhere beside the blind outputs assert not list(tmp_path.rglob("*escrow.json")) assert not (blind_dir / "blind_seed.json").exists() @@ -1177,7 +1218,7 @@ def test_ac6_ac8_end_to_end_init_parts_gather_unblind(tmp_path): ], metadata={"catalogue_version": "vTEST", "type": "mock"}, ) - assert assembled.metadata["blind_commitment"] == commitment["seed_sha256"] + assert assembled.metadata["blind_commitment"] == commitment["seed_commitment"] # born-blinded derived statistics from the blinded parts differ from truth En_b, Bn_b, _ = _derive_downstream( blinded_parts["xi_reporting"], blinded_parts["xi_integration"] @@ -1185,10 +1226,10 @@ def test_ac6_ac8_end_to_end_init_parts_gather_unblind(tmp_path): assert not np.allclose(En_b, En_true, atol=0) # -- fail-closed on tampered commitment (AC6) --------------------------- -- - tampered = dict(commitment, seed_sha256="0" * 64) + tampered = dict(commitment, seed_commitment="0" * 64) with open(init["commitment"], "w", encoding="utf-8") as f: json.dump(tampered, f) - with pytest.raises(ValueError, match="sha256"): + with pytest.raises(ValueError, match="committed seed commitment"): bd.unblind_part( out_paths["xi_integration"]["blinded"], str(blind_dir), diff --git a/workflow/rules/blinding.smk b/workflow/rules/blinding.smk index bf79433f..a5f0dedb 100644 --- a/workflow/rules/blinding.smk +++ b/workflow/rules/blinding.smk @@ -30,9 +30,9 @@ rule blind_init: """Fix the blind for one catalogue version (blind-init). Draws an OS-entropy seed, writes the repo-committable commitment.json - (sha256(seed) + config digest) and the Fernet-encrypted seed bundle. Runs - once per version and refuses to overwrite existing state — a blind is a - one-shot custody event. + (seed commitment + config digest + the installed fork's draw scheme) and the + Fernet-encrypted seed bundle. Runs once per version and refuses to overwrite + existing state — a blind is a one-shot custody event. """ output: commitment=str(COSMO_VAL / "blind" / "{version}" / "commitment.json"), From 81af22795d1b98778bfdb167a691a81c77204a55 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 17 Aug 2026 21:42:54 +0200 Subject: [PATCH 033/160] Make the run type real, and stamp rho/tau for the blind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two claims the code could not cash, both at the terminal assembly. The mock branch of assert_consistent_blind was unreachable. Every part writer hardcoded type="data", so a mock campaign's parts declared themselves data and a mock terminal assembly failed closed — only test fixtures ever built a type="mock" part. The run type is now plumbed to all six writers: CosmologyValidation carries run_type (set from config cosmo_val.type via cv_init_params) and its four writers stamp it; run_2pcf takes run_type= and a --run-type flag; run_2pcf_highres takes --run-type. papers/cosmo_val declares `type: data` explicitly, so the switch is visible where it is made. rho/tau was never stamped. The docstring said it passed through stamp_concealed_passthrough, but nothing called it for rho/tau — psf_systematics wrote the part with no commitment, so a data run's assemble_sacc died at sacc_io.load(allow_unblinded=False) on that part, before custody was ever checked. The rho_tau_stats rule now binds commitment.json on a data run and run_rho_tau passes it through calculate_rho_tau_stats, exactly as cv_cosebis and cv_pure_eb do. rho/tau carries no cosmological vector; the stamp shifts nothing and only clears the load gate. Tests: a mock campaign's parts assemble unconcealed, and the real rho/tau writer emits a part that loads without the escape hatch. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BRYg9fjMevgcsvgjh3KN6x --- papers/cosmo_val/config/config.yaml | 9 ++ src/sp_validation/cosmo_val/core.py | 9 ++ src/sp_validation/cosmo_val/cosebis.py | 2 +- src/sp_validation/cosmo_val/pseudo_cl.py | 2 +- .../cosmo_val/psf_systematics.py | 36 ++++++- src/sp_validation/cosmo_val/pure_eb.py | 9 +- .../tests/test_blinding_wiring.py | 95 ++++++++++++++++++- workflow/common.py | 16 ++++ workflow/rules/twopoint.smk | 25 ++++- workflow/scripts/run_2pcf.py | 16 +++- workflow/scripts/run_2pcf_highres.py | 15 ++- workflow/scripts/run_rho_tau.py | 9 +- 12 files changed, 224 insertions(+), 19 deletions(-) diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 232513c7..a2124302 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -19,6 +19,15 @@ versions: [ # CosmologyValidation suite parameters (cosmo_val.py) # --------------------------------------------------------------------------- cosmo_val: + # Campaign run type: "data" or "mock". One switch, two effects. It gates + # Smokescreen blind-at-birth (a data run blinds the three blindable parts and + # binds every ξ-derived consumer to the blinded siblings; a mock run bypasses + # blinding), and it is stamped as the SACC `type` of every part written. The + # two are the same decision: an unconcealed blindable part only enters the + # terminal assembly when it declares itself a mock, so a mock campaign must + # say so here for its parts to assemble at all. + type: data + # CosmologyValidation constructor npatch: 100 theta_min: 1.0 diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 0c9273d8..401d52af 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -98,6 +98,13 @@ class CosmologyValidation( noise debiasing, making those realizations reproducible run-to-run. cosmo_params : dict, optional Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. + run_type : {'data', 'mock'}, default 'data' + The campaign's run type, stamped as the SACC ``type`` of every part this + object writes. Custody state, not decoration: + ``blinding.assert_consistent_blind`` admits an unconcealed blindable part + into a terminal assembly only when it declares itself a mock, so a mock + campaign must be built with ``run_type='mock'`` for its parts to + assemble at all. Attributes ---------- @@ -224,6 +231,7 @@ def __init__( path_onecovariance=None, cosmo_params=None, blind=None, + run_type="data", ): self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method @@ -253,6 +261,7 @@ def __init__( self.nside_mask = nside_mask self.path_onecovariance = path_onecovariance self.blind = blind + self.run_type = run_type assert self.cell_method in ["map", "catalog"], ( "cell_method must be 'map' or 'catalog'" diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index 8ff2b18d..178bd971 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -203,7 +203,7 @@ def cosebis_to_sacc_part( from ..blinding import stamp_concealed_passthrough stamp_concealed_passthrough(s, commitment_path) - sacc_io.save(s, out_path, type="data") + sacc_io.save(s, out_path, type=self.run_type) def plot_cosebis( self, diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 2a55934a..cfaa349e 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -708,7 +708,7 @@ def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): cl_all, wsp, ) - sacc_io.save(s, out_path, type="data") + sacc_io.save(s, out_path, type=self.run_type) def plot_pseudo_cl(self): """ diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index 6f72ac1f..b81d8b1c 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -25,7 +25,12 @@ class PSFSystematicsMixin: - def calculate_rho_tau_stats(self): + def calculate_rho_tau_stats(self, commitment_path=None): + """Measure ρ/τ statistics per version and write each version's SACC part. + + ``commitment_path`` is custody plumbing forwarded to + :meth:`rho_tau_to_sacc_part`; see it for what it does. + """ out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" if not os.path.exists(out_dir): os.mkdir(out_dir) @@ -44,7 +49,12 @@ def calculate_rho_tau_stats(self): npatch=self.npatch, ) self.rho_tau_to_sacc_part( - ver, out_dir, base, rho_stat_handler, tau_stat_handler + ver, + out_dir, + base, + rho_stat_handler, + tau_stat_handler, + commitment_path=commitment_path, ) self.print_done("Rho stats finished") @@ -52,10 +62,24 @@ def calculate_rho_tau_stats(self): self._tau_stat_handler = tau_stat_handler def rho_tau_to_sacc_part( - self, version, out_dir, base, rho_stat_handler, tau_stat_handler + self, + version, + out_dir, + base, + rho_stat_handler, + tau_stat_handler, + commitment_path=None, ): """Write the ρ/τ SACC part for one version. + ρ/τ is a PSF diagnostic carrying no cosmological vector, so it is never + blinded — but on a data run it still has to pass the fail-closed load + gate that ``assemble_sacc`` opens every part through. + ``commitment_path`` is that seam: it stamps the part concealed under the + version's blind (:func:`blinding.stamp_concealed_passthrough`), values + untouched, so the assembly admits it. A mock run passes ``None`` and the + part stays unstamped. + ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage from the handler tables. The ``CovTauTh`` theory covariance ``cov_tau_{base}_th.npy`` — a ``(3·nbin, 3·nbin)`` plus-folded k-major block over ``{τ0, τ2, τ5}`` — is @@ -79,8 +103,12 @@ def rho_tau_to_sacc_part( tau_stat_handler.tau_stats, tau_cov_th=tau_cov_th, ) + if commitment_path is not None: + from ..blinding import stamp_concealed_passthrough + + stamp_concealed_passthrough(s, commitment_path) out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") - sacc_io.save(s, out_path, type="data") + sacc_io.save(s, out_path, type=self.run_type) @property def rho_stat_handler(self): diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 5424adf8..f0b41642 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -135,7 +135,12 @@ def calculate_pure_eb( return results def pure_eb_to_sacc_part( - self, version, out_path, results, eb_override=None, commitment_path=None + self, + version, + out_path, + results, + eb_override=None, + commitment_path=None, ): """Write the pure-E/B SACC part (six ``PURE_KEYS`` blocks + covariance). @@ -166,7 +171,7 @@ def pure_eb_to_sacc_part( from ..blinding import stamp_concealed_passthrough stamp_concealed_passthrough(s, commitment_path) - sacc_io.save(s, out_path, type="data") + sacc_io.save(s, out_path, type=self.run_type) def plot_pure_eb( self, diff --git a/src/sp_validation/tests/test_blinding_wiring.py b/src/sp_validation/tests/test_blinding_wiring.py index 706200cf..5bf5b0d7 100644 --- a/src/sp_validation/tests/test_blinding_wiring.py +++ b/src/sp_validation/tests/test_blinding_wiring.py @@ -16,9 +16,11 @@ """ import importlib.util +import json import os import subprocess import sys +import types from pathlib import Path import numpy as np @@ -103,7 +105,7 @@ def test_blindable_part_switches_on_run_type(monkeypatch): # --------------------------------------------------------------------------- # META = {"catalogue_version": "vSYNTH", "npatch": 1} # Two arbitrary-but-consistent hex stamps standing in for a real blind's -# sha256(seed) / config digest; the assembly only checks they agree across parts. +# seed commitment / config digest; the assembly only checks they agree across parts. _COMMIT = "a" * 64 _DIGEST = "b" * 64 @@ -117,12 +119,14 @@ def _spd(n, seed): return a @ a.T + n * np.eye(n) -def _data_parts(tmp_path, *, conceal, one_plaintext=False): - """Write the five per-statistic parts as ``type='data'``. +def _data_parts(tmp_path, *, conceal, one_plaintext=False, run_type="data"): + """Write the five per-statistic parts, stamped ``type=run_type``. ``conceal`` stamps every part with the shared blind (concealed=True). With ``one_plaintext`` the ξ± reporting part is left unconcealed — a blinded / - plaintext mix the assembly must refuse. + plaintext mix the assembly must refuse. ``run_type='mock'`` writes the parts + as a mock campaign's producers do, which is the only way an unconcealed + blindable part is allowed through the assembly. """ nz = {0: _nz()} theta = np.geomspace(1.0, 100.0, 6) @@ -162,7 +166,7 @@ def _data_parts(tmp_path, *, conceal, one_plaintext=False): if conceal and not (one_plaintext and name == "xi_reporting"): blinding._stamp_provenance(part, _COMMIT, "A", _DIGEST) p = tmp_path / f"{name}.sacc" - sio.save(part, str(p), type="data") + sio.save(part, str(p), type=run_type) paths[name] = str(p) return paths @@ -228,6 +232,87 @@ def test_data_assemble_stamps_the_draw_scheme_on_the_terminal_file(tmp_path): assert sio.load(str(out)).metadata["blind_draw_scheme"] == blinding.draw_scheme() +def test_mock_assemble_succeeds_without_a_blind(tmp_path): + """A mock campaign assembles its plaintext parts — the gate's mock branch is + reachable from real producers, not only from test fixtures. + + Every part writer stamps the campaign's run type (CosmologyValidation's + ``run_type``, ``run_2pcf``'s ``run_type=``, ``run_2pcf_highres``'s + ``--run-type``), so a ``mock`` campaign's parts declare themselves mocks and + ``assert_consistent_blind`` lets them through unconcealed. The same parts + stamped ``type='data'`` fail closed — that is + ``test_data_assemble_fails_closed_on_unblinded_part``. + """ + paths = _data_parts(tmp_path, conceal=False, run_type="mock") + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + assert "concealed" not in s.metadata + # No escape hatch: a mock part is not gated by the fail-closed loader. + assert sio.load(str(out)).metadata["type"] == "mock" + + +def test_rho_tau_part_is_stamped_concealed_from_the_commitment(tmp_path): + """ρ/τ carries no cosmological vector, but a data run's assembly still opens + it through the fail-closed load gate — so its writer must stamp it. + + This drives the real writer (``PSFSystematicsMixin.rho_tau_to_sacc_part``) + with the commitment the ``rho_tau_stats`` rule binds on a data run, and + checks the emitted part loads without the escape hatch. Without the stamp a + data run's ``assemble_sacc`` dies on the ρ/τ part before custody is ever + checked. + """ + from sp_validation.cosmo_val.psf_systematics import PSFSystematicsMixin + + blind_dir = tmp_path / "blind" + blind_dir.mkdir() + commitment = blinding.blind_init(str(blind_dir), log=lambda *_: None)["commitment"] + theta = np.geomspace(1.0, 100.0, 6) + rng = np.random.default_rng(11) + rho = {"theta": theta} + tau = {"theta": theta} + for k in sw.RHO_K: + for sign in ("p", "m"): + rho[f"rho_{k}_{sign}"] = rng.normal(size=6) * 1e-6 + rho[f"varrho_{k}_{sign}"] = rng.uniform(1e-14, 1e-13, 6) + for k in sw.TAU_K: + for sign in ("p", "m"): + tau[f"tau_{k}_{sign}"] = rng.normal(size=6) * 1e-6 + tau[f"vartau_{k}_{sign}"] = rng.uniform(1e-14, 1e-13, 6) + + class _Writer(PSFSystematicsMixin): + """The writer's collaborators, stubbed — the method under test is real.""" + + run_type = "data" + + def sacc_nz(self, version): + return {0: _nz()} + + def sacc_metadata(self, version): + return dict(META) + + def print_magenta(self, *args, **kwargs): + pass + + out_dir = tmp_path / "rho_tau_stats" + out_dir.mkdir() + _Writer().rho_tau_to_sacc_part( + "vSYNTH", + str(out_dir), + "vSYNTH", + types.SimpleNamespace(rho_stats=rho), + types.SimpleNamespace(tau_stats=tau), + commitment_path=commitment, + ) + + written = sio.load(str(out_dir / "rho_tau_vSYNTH.sacc")) + assert written.metadata["concealed"] is True + assert written.metadata["blind_draw_scheme"] == blinding.draw_scheme() + with open(commitment, encoding="utf-8") as f: + committed = json.load(f) + assert written.metadata["blind_commitment"] == committed["seed_commitment"] + assert written.metadata["blind_config_digest"] == committed["config_digest"] + + def test_assert_consistent_blind_rejects_divergent_commitments(tmp_path): """Two ξ± parts blinded under different commitments must never combine.""" nz = {0: _nz()} diff --git a/workflow/common.py b/workflow/common.py index 530b9212..d558773e 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -226,6 +226,19 @@ def get_shear_catalog(wildcards): # DAG-build time. test_blinding_wiring asserts the two stay in lockstep. +def run_type(): + """The campaign's run type, ``"data"`` or ``"mock"``. + + Every part writer stamps this as the SACC ``type`` metadata, which is what + ``blinding.assert_consistent_blind`` reads at assembly: a plaintext + blindable part may only assemble when it declares itself a mock. A function + rather than the ``RUN_TYPE`` global because ``from common import *`` binds + names before ``configure()`` runs, so only a call reads the configured + value. + """ + return RUN_TYPE + + def is_data_run(): """True when blinding is active (production data runs); False for mocks.""" return RUN_TYPE == "data" @@ -349,6 +362,9 @@ def cv_init_params(config, version_list=None): nrandom_cell=cv["nrandom_cell"], cell_method=cv["cell_method"], nside_mask=cv["nside_mask"], + # Stamped as the SACC `type` of every part the cv writes; RUN_TYPE reads + # from this same key (see configure()). + run_type=cv.get("type", "data"), ) if cv.get("path_onecovariance"): params["path_onecovariance"] = cv["path_onecovariance"] diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index bdfa1107..fe697893 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -24,6 +24,10 @@ rule xi: max_sep="{max_sep}", nbins="{nbins}", npatch="{npatch}", + # Stamped as the part's SACC `type`. An unconcealed blindable part only + # assembles when it declares itself a mock (assert_consistent_blind), so + # this is what makes a mock campaign's terminal assembly possible at all. + type=run_type(), resources: mem_mb=30000, disk_mb=20000, @@ -82,6 +86,7 @@ rule xi_highres: nbins=_INTEGRATION["nbins"], out=str(COSMO_VAL), scripts=WORKFLOW_SCRIPTS, + run_type=run_type(), threads: 24 resources: mem_mb=40000, @@ -90,7 +95,8 @@ rule xi_highres: "python {params.scripts}/run_2pcf_highres.py " "--version {params.version} --cat-config {params.cat_config} " "--min-sep {params.min_sep} --max-sep {params.max_sep} " - "--nbins {params.nbins} --npatch 1 --out {params.out}" + "--nbins {params.nbins} --npatch 1 --out {params.out} " + "--run-type {params.run_type}" rule run_cosmo_val: @@ -111,7 +117,23 @@ rule run_cosmo_val: """ +def rho_tau_inputs(w): + """ρ/τ has no blindable input; on a data run it binds the commitment. + + ρ/τ carries no cosmological vector, so it is never shifted — but the + fail-closed load gate assemble_sacc opens every part through admits only + concealed parts on a data run. Binding commitment.json lets the writer stamp + the part concealed pass-through (values untouched). A mock run binds nothing + and the part stays plaintext. + """ + if not is_data_run(): + return {} + return {"commitment": blind_state_paths(w.version)["commitment"]} + + rule rho_tau_stats: + input: + unpack(rho_tau_inputs), output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), @@ -127,6 +149,7 @@ rule rho_tau_stats: max_sep="{max_sep}", nbins="{nbins}", npatch="{npatch}", + type=run_type(), resources: mem_mb=30000, disk_mb=20000, diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 3e513479..b6281b74 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -39,6 +39,7 @@ def run_2pcf( cat_config, output_dir, sacc_out=None, + run_type="data", ): """Measure ξ±(θ) for ``ver`` and write its reporting SACC part. @@ -49,7 +50,10 @@ def run_2pcf( ``cat_config['paths']['output']`` so the ``.txt`` byproduct lands where lc expects. ``sacc_out`` is the exact destination for the reporting ξ± SACC part (the Snakemake-declared output); it defaults to ``{ver}_xi_reporting.sacc`` - under the resolved output directory for the CLI path. + under the resolved output directory for the CLI path. ``run_type`` + (``"data"`` or ``"mock"``) is stamped as the part's SACC ``type``: a + plaintext blindable part may only enter an assembly when it declares itself + a mock, so a mock campaign must say so here. Returns ------- @@ -85,7 +89,7 @@ def run_2pcf( out_path = sacc_out or os.path.join( output_dir or cv.cc["paths"]["output"], f"{ver}_xi_reporting.sacc" ) - sacc_io.save(s, out_path, type="data") + sacc_io.save(s, out_path, type=run_type) print(f"Wrote reporting ξ± SACC part: {out_path}") return gg @@ -107,6 +111,7 @@ def _from_snakemake(smk): # Write the SACC part exactly where the rule declares it (the .txt # byproduct still lands under the resolved output dir via _output_path). sacc_out=smk.output["xi_reporting"], + run_type=p.get("type", "data"), ) @@ -133,6 +138,12 @@ def _from_cli(argv=None): "--cat-config", required=True, help="Absolute path to cat_config.yaml" ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") + ap.add_argument( + "--run-type", + default="data", + choices=("data", "mock"), + help="Campaign run type stamped as the part's SACC `type`", + ) a = ap.parse_args(argv) run_2pcf( ver=a.ver, @@ -142,6 +153,7 @@ def _from_cli(argv=None): npatch=a.npatch, cat_config=a.cat_config, output_dir=a.out, + run_type=a.run_type, ) diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py index 6049e105..c5c546f5 100644 --- a/workflow/scripts/run_2pcf_highres.py +++ b/workflow/scripts/run_2pcf_highres.py @@ -91,6 +91,10 @@ NPATCH = None OUTPUT_DIR = None PATCH_FILE = None +# Campaign run type, stamped as the part's SACC `type`. Custody state, not +# decoration: a plaintext blindable part may only enter an assembly when it +# declares itself a mock (blinding.assert_consistent_blind). +RUN_TYPE = "data" def parse_args(argv=None): @@ -121,6 +125,12 @@ def parse_args(argv=None): "--max-sep", type=float, default=300.0, help="Max separation [arcmin]" ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") + ap.add_argument( + "--run-type", + default="data", + choices=("data", "mock"), + help="Campaign run type stamped as the part's SACC `type`", + ) return ap.parse_args(argv) @@ -233,7 +243,7 @@ def write_xi_integration_sacc(gg): variances=np.concatenate([gg.varxip, gg.varxim]), ) out_path = os.path.join(OUTPUT_DIR, f"{VERSION}_xi_integration.sacc") - sacc_io.save(s, out_path, type="data") + sacc_io.save(s, out_path, type=RUN_TYPE) log(f" Wrote {out_path}") @@ -297,10 +307,11 @@ def resolve_paths(ver): def main(): global CAT_PATH, VERSION, E1_COL, E2_COL, W_COL, REDSHIFT_PATH - global TMIN, TMAX, NBINS, NPATCH, OUTPUT_DIR, PATCH_FILE + global TMIN, TMAX, NBINS, NPATCH, OUTPUT_DIR, PATCH_FILE, RUN_TYPE args = parse_args() VERSION = args.version + RUN_TYPE = args.run_type NBINS = args.nbins NPATCH = args.npatch TMIN = args.min_sep diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index 9df0bbfd..7b64c418 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -44,9 +44,16 @@ theta_max=float(params["max_sep"]), nbins=int(params["nbins"]), npatch=int(params["npatch"]), + run_type=params.get("type", "data"), ) -cv.calculate_rho_tau_stats() +# On a data run the rule binds the version's commitment.json, which stamps the +# emitted ρ/τ part concealed pass-through — values untouched (ρ/τ carries no +# cosmological vector) but clearing the fail-closed load gate at assembly. A mock +# run binds no commitment and the part stays plaintext, stamped type='mock'. +commitment_path = snakemake.input.get("commitment") # type: ignore + +cv.calculate_rho_tau_stats(commitment_path=commitment_path) # Confirm CosmologyValidation produced the requested outputs. calculate_rho_tau_stats # writes the rho/tau FITS *and* the born-as-SACC rho_tau part (via From 334047a5fa9cbd6baa2f2abc928d021736d93fdf Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 02:05:41 +0200 Subject: [PATCH 034/160] Simplify the blinding rework: one home per fact sacc_io.save grows a commitment= kwarg and owns the passthrough stamp (three copy-pasted writer tails collapse); commitment_input() in workflow/common.py is the one place a data run binds commitment.json into rule inputs; the CLI's verify delegates draw-scheme checks to _assert_draw_scheme instead of re-wording them; _stamp_provenance loses its derivable scheme parameter. Draw-scheme rationale stated once (draw_scheme), mock-assembly rule once (assert_consistent_blind); dead escrow-recovery prose cut; doubled test assertions deduped; is_data_run reads run_type(). Container suite: 70/70 blinding+wiring+dry-run, 282 passed overall; 3 failures pre-existing and environmental (path checks on other users' scratch, stale-container glass import, a pure-eb value pin that fails identically on the unedited base). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01G9MahwJEQ1t9EuvUXijmy3 --- papers/cosmo_val/config/config.yaml | 10 ++--- scripts/blind_data_vector.py | 37 ++++++---------- src/sp_validation/blinding.py | 43 ++++++++----------- src/sp_validation/cosmo_val/core.py | 8 ++-- src/sp_validation/cosmo_val/cosebis.py | 12 ++---- .../cosmo_val/psf_systematics.py | 21 +++------ src/sp_validation/cosmo_val/pure_eb.py | 17 ++------ src/sp_validation/sacc_io.py | 12 +++++- src/sp_validation/tests/test_blinding.py | 9 ---- workflow/common.py | 18 ++++++-- workflow/rules/cosmo_val.smk | 19 +++----- workflow/rules/twopoint.smk | 23 +++------- workflow/scripts/run_2pcf.py | 5 +-- workflow/scripts/run_2pcf_highres.py | 3 +- 14 files changed, 89 insertions(+), 148 deletions(-) diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index a2124302..f3ead5ee 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -19,13 +19,9 @@ versions: [ # CosmologyValidation suite parameters (cosmo_val.py) # --------------------------------------------------------------------------- cosmo_val: - # Campaign run type: "data" or "mock". One switch, two effects. It gates - # Smokescreen blind-at-birth (a data run blinds the three blindable parts and - # binds every ξ-derived consumer to the blinded siblings; a mock run bypasses - # blinding), and it is stamped as the SACC `type` of every part written. The - # two are the same decision: an unconcealed blindable part only enters the - # terminal assembly when it declares itself a mock, so a mock campaign must - # say so here for its parts to assemble at all. + # Campaign run type: "data" or "mock". One switch: it gates Smokescreen + # blind-at-birth and is stamped as the SACC `type` of every part written + # (custody state at assembly — see blinding.assert_consistent_blind). type: data # CosmologyValidation constructor diff --git a/scripts/blind_data_vector.py b/scripts/blind_data_vector.py index 692ebd94..1050d226 100644 --- a/scripts/blind_data_vector.py +++ b/scripts/blind_data_vector.py @@ -9,9 +9,8 @@ (never printed, never written in plaintext), publishes a repo-committable ``commitment.json`` (the seed commitment + the config digest + the installed Smokescreen fork's draw scheme), and encrypts the seed into a Fernet bundle. -Those three, not the seed alone, are what reproduces a blind: seed and config -fix *which* shift, the draw scheme fixes *how* the seed becomes that shift. -``blind-part`` blinds one intermediate part SACC (reporting ξ±, integration ξ±, +Those three, not the seed alone, are what reproduces a blind (see +``blinding.draw_scheme``). ``blind-part`` blinds one intermediate part SACC (reporting ξ±, integration ξ±, or pseudo-Cℓ) under that fixed state, escrows the true vector into a per-part encrypted bundle beside the blinded output, and deletes the plaintext part. ``unblind`` verifies all three commitments and restores a true part @@ -113,8 +112,7 @@ def _verify(args): only that the file's three custody stamps agree with the commitment, and that the recorded draw scheme is the one this install implements. That last check makes the result environment-dependent by design: a machine carrying a - different Smokescreen reports a problem even when file and commitment agree - perfectly, because that machine could not unblind the file. + different Smokescreen could not unblind the file, so it reports a problem. Loads with ``allow_unblinded=True``: the whole job here is to report on a file's custody state, including the state where the file is not concealed at @@ -131,30 +129,19 @@ def _verify(args): problems.append("blind_commitment does not match the committed seed commitment") if s.metadata.get("blind_config_digest") != commitment["config_digest"]: problems.append("blind_config_digest does not match the committed digest") - # The draw scheme is checked three ways: file ↔ commitment, and both against - # the installed fork. A scheme mismatch is the one blinding failure with no - # numerical symptom — the unblind would subtract a different shift than was - # added and every other check here would still pass. - installed = blinding.draw_scheme() + # The draw scheme is checked two ways: file ↔ commitment, and the file's + # against the installed fork (see blinding.draw_scheme). file_scheme = s.metadata.get("blind_draw_scheme") committed = commitment.get("draw_scheme") - if file_scheme is None or committed is None: + if file_scheme != committed: problems.append( - "no draw-scheme record on the file and/or in the commitment — " - "this blind predates draw-scheme binding and cannot be verified " - "reproducible" - ) - elif int(file_scheme) != int(committed): - problems.append( - f"blind_draw_scheme {int(file_scheme)} does not match the committed " - f"draw_scheme {int(committed)}" - ) - elif int(file_scheme) != installed: - problems.append( - f"blind was drawn under Smokescreen DRAW_SCHEME={int(file_scheme)} " - f"but the installed fork implements DRAW_SCHEME={installed} — this " - "install cannot reproduce the shift" + f"blind_draw_scheme {file_scheme!r} does not match the committed " + f"draw_scheme {committed!r}" ) + try: + blinding._assert_draw_scheme(file_scheme, "the blinded file") + except ValueError as exc: + problems.append(str(exc)) if "seed_smokescreen" in s.metadata: problems.append("PLAINTEXT SEED LEAKED into file metadata (seed_smokescreen)") if problems: diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 34b648d2..247b20e9 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -46,9 +46,8 @@ and the installed fork's ``DRAW_SCHEME`` as a repo-committable ``commitment.json``, and encrypts the seed into a Fernet bundle (``smokescreen.encryption``) — the plaintext seed is never written. The - three together, not the seed alone, are what reproduces a blind: seed and - config fix *which* shift, the draw scheme fixes *how* the seed becomes that - shift. + three together, not the seed alone, are what reproduces a blind (see + :func:`draw_scheme`). Each :func:`blind_part` call reads that fixed state, conceals one part, escrows the part's true vector into its own encrypted bundle beside the @@ -61,14 +60,10 @@ ``blind_draw_scheme``. :func:`unblind_part` verifies all three against the commitment *before* subtracting anything, then restores the true part. - Custody has no back door, including for its own history: a blind fixed - before the draw scheme was bound into ``commitment.json`` carries no - scheme record, and :func:`_assert_draw_scheme` refuses it ahead of every - seed read, so neither :func:`blind_part` nor :func:`unblind_part` will run - on it. Such a blind is recoverable only by hand, by decrypting its per-part - escrow bundles (:func:`_read_encrypted_json`) and reading ``true_mean`` - back out. No such blind exists — the scheme has been bound since before - the first real blind — and the CLI deliberately offers no override. + Custody has no back door, including for its own history: a blind carrying no + scheme record is refused by :func:`_assert_draw_scheme` ahead of every seed + read, with no CLI override — none exists, the scheme having been bound + before the first real blind. """ import dataclasses @@ -228,7 +223,8 @@ def _assert_draw_scheme(recorded, what): ``what`` names the surface the scheme was read from, for the message. A missing record (``None``) is a failure, not a pass: a blind whose scheme - is unknown cannot be shown to be reproducible by this install. + is unknown cannot be shown to be reproducible by this install (see + :func:`draw_scheme`). """ installed = draw_scheme() if recorded is None: @@ -570,7 +566,7 @@ def unblind_sacc(blinded, seed, config=None, log=print): return part -def _stamp_provenance(s, commitment, label, config_digest, scheme=None): +def _stamp_provenance(s, commitment, label, config_digest): """Stamp the blind's public provenance; strip any leaked seed. ``blind_commitment`` (sha256 of the seed) ties the file to its blind @@ -582,17 +578,16 @@ def _stamp_provenance(s, commitment, label, config_digest, scheme=None): raw seed upstream Smokescreen's writer would stamp — is popped defensively: the seed must never ride a kept file. - ``scheme`` defaults to the installed fork's, which is the right answer - whenever this call is stamping a blind that was just computed. Callers - propagating an existing blind's provenance pass that blind's recorded - scheme instead. + The stamped scheme is always the installed fork's: every caller has already + checked the blind's recorded scheme against it (:func:`_assert_draw_scheme`), + so the two agree by the time this runs. """ s.metadata.pop("seed_smokescreen", None) s.metadata["concealed"] = True s.metadata["blind"] = label s.metadata["blind_commitment"] = commitment s.metadata["blind_config_digest"] = config_digest - s.metadata["blind_draw_scheme"] = draw_scheme() if scheme is None else int(scheme) + s.metadata["blind_draw_scheme"] = draw_scheme() def stamp_concealed_passthrough(s, commitment_path): @@ -628,7 +623,6 @@ def stamp_concealed_passthrough(s, commitment_path): commitment["seed_commitment"], commitment["label"], commitment["config_digest"], - scheme=commitment["draw_scheme"], ) return s @@ -808,12 +802,11 @@ def blind_init(blind_dir, config=None, label="A", log=print): def _read_seed(blind_dir, config): """Decrypt the seed bundle and verify it against the commitment. - The seed commitment, the config digest, and the committed draw scheme are all - checked before the seed is handed to any caller — a tampered bundle, a - drifted config or a Smokescreen that draws differently from the one that - fixed this blind all fail loud here, whether the caller is about to blind - or to unblind. The scheme check is what stops a re-blind of a later part - from landing a different hidden cosmology than the earlier parts got. + The seed commitment, the config digest, and the committed draw scheme (see + :func:`draw_scheme`) are all checked before the seed is handed to any caller + — a tampered bundle, a drifted config or a Smokescreen that draws + differently from the one that fixed this blind all fail loud here, whether + the caller is about to blind or to unblind. Returns ------- diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 401d52af..05ded922 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -100,11 +100,9 @@ class CosmologyValidation( Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. run_type : {'data', 'mock'}, default 'data' The campaign's run type, stamped as the SACC ``type`` of every part this - object writes. Custody state, not decoration: - ``blinding.assert_consistent_blind`` admits an unconcealed blindable part - into a terminal assembly only when it declares itself a mock, so a mock - campaign must be built with ``run_type='mock'`` for its parts to - assemble at all. + object writes. Custody state, not decoration: a mock campaign must be + built with ``run_type='mock'`` for its parts to assemble at all (see + ``blinding.assert_consistent_blind``). Attributes ---------- diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index 178bd971..4fbec2ba 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -185,10 +185,8 @@ def cosebis_to_sacc_part( to the part in place of ``result["En"]`` — re-derived from the integration ξ± SACC part at the fiducial scale cut (Bn and the covariance stay from the raw estimator ``result``). ``commitment_path`` stamps the part concealed - under that version's blind (via - :func:`blinding.stamp_concealed_passthrough`) before save, so a data run's - part clears the fail-closed load gate. With both ``None`` (mock runs) the - behaviour is unchanged. + under that version's blind (see :func:`sacc_io.save`). With both ``None`` + (mock runs) the behaviour is unchanged. """ result, scale_cut = self._fiducial_cosebis_result(results, fiducial_scale_cut) if en_override is not None: @@ -199,11 +197,7 @@ def cosebis_to_sacc_part( result, scale_cut, ) - if commitment_path is not None: - from ..blinding import stamp_concealed_passthrough - - stamp_concealed_passthrough(s, commitment_path) - sacc_io.save(s, out_path, type=self.run_type) + sacc_io.save(s, out_path, type=self.run_type, commitment=commitment_path) def plot_cosebis( self, diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index b81d8b1c..c7984c26 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -26,11 +26,7 @@ class PSFSystematicsMixin: def calculate_rho_tau_stats(self, commitment_path=None): - """Measure ρ/τ statistics per version and write each version's SACC part. - - ``commitment_path`` is custody plumbing forwarded to - :meth:`rho_tau_to_sacc_part`; see it for what it does. - """ + """Measure ρ/τ statistics per version and write each version's SACC part.""" out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" if not os.path.exists(out_dir): os.mkdir(out_dir) @@ -73,12 +69,9 @@ def rho_tau_to_sacc_part( """Write the ρ/τ SACC part for one version. ρ/τ is a PSF diagnostic carrying no cosmological vector, so it is never - blinded — but on a data run it still has to pass the fail-closed load - gate that ``assemble_sacc`` opens every part through. - ``commitment_path`` is that seam: it stamps the part concealed under the - version's blind (:func:`blinding.stamp_concealed_passthrough`), values - untouched, so the assembly admits it. A mock run passes ``None`` and the - part stays unstamped. + blinded; ``commitment_path`` stamps the part concealed under the + version's blind, values untouched (see :func:`sacc_io.save`), so a data + run's assembly admits it. A mock run passes ``None``. ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage from the handler tables. The ``CovTauTh`` theory covariance ``cov_tau_{base}_th.npy`` — a @@ -103,12 +96,8 @@ def rho_tau_to_sacc_part( tau_stat_handler.tau_stats, tau_cov_th=tau_cov_th, ) - if commitment_path is not None: - from ..blinding import stamp_concealed_passthrough - - stamp_concealed_passthrough(s, commitment_path) out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") - sacc_io.save(s, out_path, type=self.run_type) + sacc_io.save(s, out_path, type=self.run_type, commitment=commitment_path) @property def rho_stat_handler(self): diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index f0b41642..e7220c47 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -135,12 +135,7 @@ def calculate_pure_eb( return results def pure_eb_to_sacc_part( - self, - version, - out_path, - results, - eb_override=None, - commitment_path=None, + self, version, out_path, results, eb_override=None, commitment_path=None ): """Write the pure-E/B SACC part (six ``PURE_KEYS`` blocks + covariance). @@ -154,8 +149,8 @@ def pure_eb_to_sacc_part( — re-derived from the reporting + integration ξ± SACC parts (the covariance stays blind-invariant from the raw estimator ``results``). ``commitment_path`` stamps the part concealed under that version's blind - (via :func:`blinding.stamp_concealed_passthrough`) before save. With both - ``None`` (mock runs) the behaviour is unchanged. + (see :func:`sacc_io.save`). With both ``None`` (mock runs) the behaviour + is unchanged. """ theta = results["gg"].meanr source = eb_override if eb_override is not None else results @@ -167,11 +162,7 @@ def pure_eb_to_sacc_part( eb, covariance=results["cov"], ) - if commitment_path is not None: - from ..blinding import stamp_concealed_passthrough - - stamp_concealed_passthrough(s, commitment_path) - sacc_io.save(s, out_path, type=self.run_type) + sacc_io.save(s, out_path, type=self.run_type, commitment=commitment_path) def plot_pure_eb( self, diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 7dcb72a7..8d14e6ca 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -924,7 +924,7 @@ def update_statistic(s, sub): s.data[idx[0]].value = point.value -def save(s, path, *, type): +def save(s, path, *, type, commitment=None): """Write ``s`` to ``path`` (FITS), overwriting any existing file. Parameters @@ -937,6 +937,12 @@ def save(s, path, *, type): the pipeline computing the data vector — knows whether its input catalogue is a mock; there is deliberately no default. ``load`` refuses ``type='data'`` files that are not blinded. + commitment : str, optional + Path to the version's ``commitment.json``. When given, the file is + stamped concealed under that blind + (:func:`sp_validation.blinding.stamp_concealed_passthrough`, values + untouched) before writing — the seam every born-blinded or + blind-irrelevant part uses to clear the fail-closed load gate. """ if type not in ("data", "mock"): raise ValueError(f"type must be 'data' or 'mock'; got {type!r}") @@ -946,6 +952,10 @@ def save(s, path, *, type): f"refusing to re-stamp as {type!r}" ) s.metadata["type"] = type + if commitment is not None: + from . import blinding + + blinding.stamp_concealed_passthrough(s, commitment) s.save_fits(path, overwrite=True) diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 6d808895..f4aa7d58 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -336,18 +336,10 @@ def test_theory_config_ccl_params_exact_keyset(): # --------------------------------------------------------------------------- # def test_commitment_is_the_forks_domain_separated_digest(): """One definition of the commitment, and it is the fork's.""" - import hashlib - import smokescreen seed = "the-secret" assert bd.seed_commitment(seed) == smokescreen.seed_commitment(seed) - assert ( - bd.seed_commitment(seed) - == hashlib.sha256( - smokescreen.COMMITMENT_DOMAIN + seed.encode("utf-8") - ).hexdigest() - ) assert bd.seed_commitment("right") != bd.seed_commitment("wrong") @@ -370,7 +362,6 @@ def test_commitment_does_not_embed_the_rng_seed(): for seed in ("my_secret_seed", "the-secret", secrets.token_hex(16)): commitment = bd.seed_commitment(seed) assert int(commitment[:16], 16) != _normalize_seed(seed) - assert str(_normalize_seed(seed)) not in commitment def test_hidden_params_deterministic_and_in_envelope(): diff --git a/workflow/common.py b/workflow/common.py index d558773e..78224d02 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -230,8 +230,7 @@ def run_type(): """The campaign's run type, ``"data"`` or ``"mock"``. Every part writer stamps this as the SACC ``type`` metadata, which is what - ``blinding.assert_consistent_blind`` reads at assembly: a plaintext - blindable part may only assemble when it declares itself a mock. A function + ``blinding.assert_consistent_blind`` reads at assembly. A function rather than the ``RUN_TYPE`` global because ``from common import *`` binds names before ``configure()`` runs, so only a call reads the configured value. @@ -241,7 +240,7 @@ def run_type(): def is_data_run(): """True when blinding is active (production data runs); False for mocks.""" - return RUN_TYPE == "data" + return run_type() == "data" def blind_state_dir(version): @@ -262,6 +261,19 @@ def blind_state_paths(version): } +def commitment_input(version): + """Input mapping binding a version's commitment.json, on data runs only. + + A part writer stamps its output concealed from that file (sacc_io.save's + `commitment=`), which is what lets a born-blinded or blind-irrelevant part + clear the fail-closed load gate the terminal assembly opens every part + through. A mock run binds nothing and the part stays plaintext. + """ + if not is_data_run(): + return {} + return {"commitment": blind_state_paths(version)["commitment"]} + + def blinded_path(part_path): """The *_blinded sibling blind_part writes beside a plaintext part. diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 705ac9bc..049b30b8 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -370,29 +370,24 @@ rule cv_pseudo_cl: # On a data run the COSEBIs / pure-E/B parts re-derive their E-mode vector from # the *blinded* integration ξ± (COSEBIs) or blinded reporting + integration ξ± -# (pure-E/B), and stamp the derived part concealed from the version's -# commitment.json. blindable_part returns the plaintext part on a mock run and -# the blinded part on a data run, so the ξ± inputs bind unconditionally for every -# version; only the commitment is gated on is_data_run(). +# (pure-E/B). blindable_part returns the plaintext part on a mock run and the +# blinded part on a data run, so the ξ± inputs bind unconditionally for every +# version; see common.commitment_input for the commitment. def cv_cosebis_inputs(w): - inputs = { + return { "xi": cv_xi_txt(w.version), "xi_integration": blindable_part(cv_xi_integration_sacc(w.version)), + **commitment_input(w.version), } - if is_data_run(): - inputs["commitment"] = blind_state_paths(w.version)["commitment"] - return inputs def cv_pure_eb_inputs(w): - inputs = { + return { "xi": cv_xi_txt(w.version), "xi_reporting": blindable_part(cv_xi_reporting_sacc(w.version)), "xi_integration": blindable_part(cv_xi_integration_sacc(w.version)), + **commitment_input(w.version), } - if is_data_run(): - inputs["commitment"] = blind_state_paths(w.version)["commitment"] - return inputs rule cv_pure_eb: diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index fe697893..31b1dd5a 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -24,9 +24,8 @@ rule xi: max_sep="{max_sep}", nbins="{nbins}", npatch="{npatch}", - # Stamped as the part's SACC `type`. An unconcealed blindable part only - # assembles when it declares itself a mock (assert_consistent_blind), so - # this is what makes a mock campaign's terminal assembly possible at all. + # Stamped as the part's SACC `type` — custody state at assembly + # (see blinding.assert_consistent_blind). type=run_type(), resources: mem_mb=30000, @@ -117,23 +116,11 @@ rule run_cosmo_val: """ -def rho_tau_inputs(w): - """ρ/τ has no blindable input; on a data run it binds the commitment. - - ρ/τ carries no cosmological vector, so it is never shifted — but the - fail-closed load gate assemble_sacc opens every part through admits only - concealed parts on a data run. Binding commitment.json lets the writer stamp - the part concealed pass-through (values untouched). A mock run binds nothing - and the part stays plaintext. - """ - if not is_data_run(): - return {} - return {"commitment": blind_state_paths(w.version)["commitment"]} - - rule rho_tau_stats: + # ρ/τ has no blindable input; it binds only the commitment, to stamp its + # part concealed pass-through (see common.commitment_input). input: - unpack(rho_tau_inputs), + unpack(lambda w: commitment_input(w.version)), output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index b6281b74..7f39660c 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -51,9 +51,8 @@ def run_2pcf( expects. ``sacc_out`` is the exact destination for the reporting ξ± SACC part (the Snakemake-declared output); it defaults to ``{ver}_xi_reporting.sacc`` under the resolved output directory for the CLI path. ``run_type`` - (``"data"`` or ``"mock"``) is stamped as the part's SACC ``type``: a - plaintext blindable part may only enter an assembly when it declares itself - a mock, so a mock campaign must say so here. + (``"data"`` or ``"mock"``) is stamped as the part's SACC ``type`` — custody + state at assembly (see ``blinding.assert_consistent_blind``). Returns ------- diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py index c5c546f5..985c0ef4 100644 --- a/workflow/scripts/run_2pcf_highres.py +++ b/workflow/scripts/run_2pcf_highres.py @@ -92,8 +92,7 @@ OUTPUT_DIR = None PATCH_FILE = None # Campaign run type, stamped as the part's SACC `type`. Custody state, not -# decoration: a plaintext blindable part may only enter an assembly when it -# declares itself a mock (blinding.assert_consistent_blind). +# decoration (see blinding.assert_consistent_blind). RUN_TYPE = "data" From 28e237d47550ecea22992406932423b1c2dc18a7 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 02:57:26 +0200 Subject: [PATCH 035/160] blinding: default label "smokescreen", not "A" (collides with legacy blind vocab) --- src/sp_validation/blinding.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 247b20e9..97fb7121 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -488,7 +488,7 @@ def _concealed(s): return bool(s.metadata.get("concealed")) -def blind_sacc(part, seed, config=None, label="A", log=print): +def blind_sacc(part, seed, config=None, label="smokescreen", log=print): """Return a blinded copy of a part SACC (covariance and tags untouched). Per blindable block present (a standalone part carries exactly one — @@ -744,7 +744,7 @@ def part_paths(part_path): } -def blind_init(blind_dir, config=None, label="A", log=print): +def blind_init(blind_dir, config=None, label="smokescreen", log=print): """Fix the blind for one catalogue version: seed, commitment, seed bundle. Runs once per catalogue version: From cd630c55ca7b3a02ed1d45ccab00d6f1651f7e79 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 02:58:24 +0200 Subject: [PATCH 036/160] Restore cosmo_inference/ to develop state (stale-base residue reverted newer develop edits) --- cosmo_inference/.gitignore | 1 + cosmo_inference/README.md | 4 +- .../scripts/chain_postprocessing.py | 24 +- cosmo_inference/scripts/cosmosis_fitting.py | 12 +- cosmo_inference/scripts/k_analysis.py | 308 ++++++++++++++++++ 5 files changed, 328 insertions(+), 21 deletions(-) create mode 100644 cosmo_inference/scripts/k_analysis.py diff --git a/cosmo_inference/.gitignore b/cosmo_inference/.gitignore index 0b53f63e..cb5922b7 100644 --- a/cosmo_inference/.gitignore +++ b/cosmo_inference/.gitignore @@ -1,6 +1,7 @@ plots/ .ipynb_checkpoints/ data/ +cosmosis_config/output/* *.png *.pdf *.sh \ No newline at end of file diff --git a/cosmo_inference/README.md b/cosmo_inference/README.md index 6da1b94c..5d753010 100644 --- a/cosmo_inference/README.md +++ b/cosmo_inference/README.md @@ -4,7 +4,7 @@ by Lisa Goh and Sacha Guerrini, CEA Paris-Saclay This folder contains the files neccessary to run the cosmological inference pipeline on the UNIONS galaxy catalogues. ### Requirements -To run the pipeline, one would need to have installed [CosmoSIS](https://cosmosis.readthedocs.io/en/latest/) and [CosmoCov](https://github.com/CosmoLike/CosmoCov). To PSF leakage parameters, the fork of [cosmosis-standard-library](https://github.com/sachaguer/cosmosis-standard-library/) of Sacha Guerrini has to be used. +To run the pipeline, one would need to have installed [CosmoSIS](https://cosmosis.readthedocs.io/en/latest/). To sample the PSF leakage parameters, the fork of [cosmosis-standard-library](https://github.com/sachaguer/cosmosis-standard-library/) of Sacha Guerrini has to be used. ### To Run The inference pipeline is now orchestrated through Python. Run the main Snakemake workflow from the parent directory: @@ -15,7 +15,7 @@ snakemake -j inference_fiducial This will automatically execute all steps: 1. Calculate 2PCF ($\xi_{pm}$) via `cosmo_val.py` -2. Compute covariance matrices using CosmoCov +2. Compute covariance matrices using CosmoCov 3. Prepare CosmoSIS data (FITS) via `cosmosis_fitting.py` 4. Run CosmoSIS inference diff --git a/cosmo_inference/scripts/chain_postprocessing.py b/cosmo_inference/scripts/chain_postprocessing.py index 83cddf4b..d1462902 100644 --- a/cosmo_inference/scripts/chain_postprocessing.py +++ b/cosmo_inference/scripts/chain_postprocessing.py @@ -7,6 +7,7 @@ import os import subprocess +import cs_util.cosmo as cs_cosmo import matplotlib.pyplot as plt import numpy as np from astropy.io import fits @@ -281,16 +282,15 @@ def compute_best_fit_xi_from_cell(output_folder, root, best_fit_params, theta_ra output_folder + "{}/best_fit/shear_cl/bin_1_1.txt".format(root) ) - import pyccl as ccl - - cosmo = ccl.Cosmology( - Omega_c=best_fit_params["omch2"] / (best_fit_params["h0"] / 100) ** 2, - Omega_b=best_fit_params["ombh2"] / (best_fit_params["h0"] / 100) ** 2, - h=best_fit_params["h0"] / 100, - n_s=best_fit_params["n_s"], - sigma8=best_fit_params["SIGMA_8"], - baryonic_effects=None, - extra_parameters={ + cosmo = cs_cosmo.get_cosmo( + camb_params={ + "H0": best_fit_params["h0"], + "ombh2": best_fit_params["ombh2"], + "omch2": best_fit_params["omch2"], + "ns": best_fit_params["n_s"], + "sigma8": best_fit_params["SIGMA_8"], + }, + extra_params={ "camb": { "halofit_version": "mead2020_feedback", "HMCode_logT_AGN": best_fit_params["logt_agn"], @@ -298,9 +298,7 @@ def compute_best_fit_xi_from_cell(output_folder, root, best_fit_params, theta_ra }, ) - theta_deg = np.rad2deg(theta_rad) - xi_p = ccl.correlation(cosmo, ell=ell, C_ell=shear_cl, theta=theta_deg, type="GG+") - xi_m = ccl.correlation(cosmo, ell=ell, C_ell=shear_cl, theta=theta_deg, type="GG-") + xi_p, xi_m = cs_cosmo.c_ell_to_xi(cosmo, np.rad2deg(theta_rad) * 60, ell, shear_cl) os.makedirs( output_folder + "{}/best_fit/shear_xi_minus".format(root), exist_ok=True diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 6cfe8be8..7396ae35 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -3,8 +3,8 @@ """Prepare CosmoSIS inputs from UNIONS validation outputs. The script lives in ``cosmo_inference/scripts``. By default it reads templates -from ``cosmo_inference/cosmosis_config`` and writes data products beneath -``cosmo_inference/data`` and ``cosmo_inference/cosmosis_config``. Override +from ``cosmo_inference/cosmosis_config/templates`` and writes data products beneath +``cosmo_inference/data`` and ``cosmo_inference/cosmosis_config/output``. Override ``--template-dir`` or ``--output-root`` to use alternative locations. """ @@ -399,8 +399,8 @@ def _generate_ini_file( modifications.append((r"^\[output\]", output_section)) pipeline_section = ( - f"[pipeline]\nvalues = cosmosis_config/{values_file}\npriors = " - f"cosmosis_config/{priors_file}" + f"[pipeline]\nvalues = cosmosis_config/templates/{values_file}\npriors = " + f"cosmosis_config/templates/{priors_file}" ) modifications.append((r"^\[pipeline\]", pipeline_section)) @@ -617,7 +617,7 @@ def parse_args(): parser.add_argument( "--template-dir", type=str, - default=str(cosmo_inference_root / "cosmosis_config"), + default=str(cosmo_inference_root / "cosmosis_config" / "templates"), help=( "Directory containing CosmoSIS template INI files (defaults to the " "cosmosis_config folder next to this script)." @@ -636,7 +636,7 @@ def parse_args(): template_dir_path = Path(args.template_dir).expanduser().resolve() output_basename_path = Path(output_basename) data_dir_root = output_root_path / "data" / output_basename_path - config_dir_root = output_root_path / "cosmosis_config" + config_dir_root = output_root_path / "cosmosis_config" / "output" data_dir_root.mkdir(parents=True, exist_ok=True) config_dir_root.mkdir(parents=True, exist_ok=True) out_file_path = data_dir_root / f"cosmosis_{args.cosmosis_root}.fits" diff --git a/cosmo_inference/scripts/k_analysis.py b/cosmo_inference/scripts/k_analysis.py new file mode 100644 index 00000000..d3939c10 --- /dev/null +++ b/cosmo_inference/scripts/k_analysis.py @@ -0,0 +1,308 @@ +import sys +from multiprocessing import Pool + +import astropy.constants as const +import astropy.units as u +import camb +import numpy as np +import scipy.integrate as integrate +from cs_util.cosmo import PLANCK18 +from scipy import interpolate +from scipy.special import j0, jn + +###################################################################################################### + +###################################################################################################### + + +def process_theta(theta, nz_file, output_root): + """Compute shear correlation functions for a single angular scale. + + For a given angular separation, this function computes the weak-lensing + correlation functions xi+ and xi- over a range of maximum wavenumbers + (kmax). The calculation includes nonlinear matter power spectra from + CAMB and optionally intrinsic-alignment contributions. Results are + appended to output text files. + + Parameters + ---------- + theta : float + Angular separation in arcminutes. + nz_file : str + Path to the source redshift distribution file. The file must contain + two columns giving redshift and n(z). + output_root : str + Prefix of the output files. Results are written to + ``{output_root}_xip.txt`` and ``{output_root}_xim.txt``. + + Returns + ------- + float + The input angular separation, returned for bookkeeping when running + in parallel. + """ + + def Hz(z): + """Return the Hubble expansion rate. + + Computes the Hubble parameter assuming a flat LCDM cosmology. + + Parameters + ---------- + z : float or ndarray + Redshift. + + Returns + ------- + float or ndarray + Hubble parameter in km s^-1 Mpc^-1. + """ + return H0 * np.sqrt(Omega_m * (1 + z) ** 3 + (1 - Omega_m)) + + def rz_interp(want_z): + """Create an interpolation between redshift and comoving distance. + + Computes the line-of-sight comoving distance by numerical integration + and returns an interpolation function in either direction. + + Parameters + ---------- + want_z : bool + If True, return an interpolator mapping comoving distance to + redshift. Otherwise return an interpolator mapping redshift to + comoving distance. + + Returns + ------- + scipy.interpolate.interp1d + Interpolation function relating redshift and comoving distance. + """ + + def hz_integrand(zz): + return c / Hz(zz) + + rz_ref = np.array([integrate.quad(hz_integrand, 0, z)[0] for z in zs]) + + if want_z == True: + return interpolate.interp1d( + rz_ref, zs, bounds_error=False, fill_value="extrapolate" + ) + else: + return interpolate.interp1d( + zs, rz_ref, bounds_error=False, fill_value="extrapolate" + ) + + def W_gg(z, rz): + """Compute the lensing efficiency kernel. + + Evaluates the lensing kernel for the supplied source redshift + distribution. + + Parameters + ---------- + z : float + Lens redshift. + rz : callable + Function returning comoving distance as a function of redshift. + + Returns + ------- + float + Weak-lensing efficiency kernel evaluated at z. + """ + z_integrate = np.linspace(z, zmax, n) + r_zmin = rz(z) + nz_int = som_nz_interp(z_integrate) * (1 - r_zmin / rz(z_integrate)) + prefactor = 3 * H0**2 * Omega_m * (1 + z) * r_zmin / (2 * c**2) + + return prefactor * integrate.simpson(nz_int, x=z_integrate) + + def C_ell(ell, kmax, want_IA): + """Compute the angular power spectrum. + + Calculates the Limber-approximated cosmic shear power spectrum, + optionally including intrinsic-alignment (GI and II) contributions. + + Parameters + ---------- + ell : float + Angular multipole. + kmax : float + Maximum wavenumber used to truncate the Limber integral. + want_IA : bool + If True, include intrinsic-alignment contributions. + + Returns + ------- + float + Total cosmic shear angular power spectrum at the specified multipole. + """ + z_min = rz_interp_wantz((ell + 0.5) / kmax) + z_valid = zs[zs >= z_min] + + if len(z_valid) == 0: + return 0.0 + + rzs = rz_interp_noz(z_valid) + W_ggs = W_gg_interp(z_valid) + Pks = pkz_nl_interp((z_valid, (ell + 0.5) / rzs)) + Hzs = Hz(z_valid) + + gg_integrand = c * W_ggs**2 * Pks / (Hzs * rzs**2) + C_ell_gg = integrate.simpson(gg_integrand, x=z_valid) + + if want_IA == True: + Dzs = pkz_lin_interp((z_valid, (ell + 0.5) / rzs)) / pkz_lin_interp( + (0, (ell + 0.5) / rzs) + ) + P_ia = -A_IA * c1 * Omega_m / Dzs + W_ias = Hzs * som_nz_interp(z_valid) / c + + gI_integrand = c * W_ggs * Pks * W_ias * P_ia / (Hzs * rzs**2) + II_integrand = c * Pks * W_ias**2 * P_ia**2 / (Hzs * rzs**2) + + C_ell_gI = integrate.simpson(gI_integrand, x=z_valid) + C_ell_II = integrate.simpson(II_integrand, x=z_valid) + + return C_ell_gg + C_ell_gI + C_ell_II + + return C_ell_gg + + def xi(theta_rad, kmax, want_IA): + """Compute the shear correlation functions. + + Evaluates the real-space shear correlation functions xi+ and xi- + by Hankel-transforming the convergence power spectrum. + + Parameters + ---------- + theta_rad : float + Angular separation in radians. + kmax : float + Maximum wavenumber used in the Limber integration. + want_IA : bool + If True, include intrinsic-alignment contributions. + + Returns + ------- + tuple of float + The pair (xi_plus, xi_minus). + """ + C_ell_vals = np.array([C_ell(ell, kmax, want_IA) for ell in ells]) + + xip_integrand = ells * C_ell_vals * j0(ells * theta_rad) + xim_integrand = ells * C_ell_vals * jn(4, ells * theta_rad) + + return integrate.simpson(xip_integrand, x=ells) / ( + 2 * np.pi + ), integrate.simpson(xim_integrand, x=ells) / (2 * np.pi) + + ########################################################################################### + + c = const.c.to("km/s") + H0 = PLANCK18["h"] * 100 + Omega_m = PLANCK18["Omega_m"] + + A_IA = 0.83 + c1 = 5e-14 * (u.Mpc**3.0) / u.solMass + + zmin = 1e-5 + zmax = 4 + n = 500 + zs = np.linspace(zmin, zmax, n) + ells = np.linspace(2, 1e5, int(1e5 - 1)) + + kmaxs = np.logspace(-4, 2, 200) + theta_rad = theta * (np.pi / (180 * 60)) + + ombh2 = PLANCK18["Omega_b"] * PLANCK18["h"] ** 2 + omch2 = (PLANCK18["Omega_m"] - PLANCK18["Omega_b"]) * PLANCK18["h"] ** 2 + pars = camb.set_params( + H0=H0, + ombh2=ombh2, + omch2=omch2, + mnu=PLANCK18["m_nu"], + As=PLANCK18["As"], + ns=PLANCK18["n_s"], + halofit_version="mead2020_feedback", + lmax=3000, + WantTransfer=True, + ) + + nz_z, som_nz = np.loadtxt(f"{nz_file}", unpack=True) + som_nz_interp = interpolate.interp1d( + nz_z, som_nz, bounds_error=False, fill_value=None + ) + + pars.set_matter_power(redshifts=np.linspace(zmin, zmax, 150), kmax=200) + results = camb.get_results(pars) + results.calc_power_spectra(pars) + k_nonlin, z_nonlin, pk_nonlin = results.get_nonlinear_matter_power_spectrum( + hubble_units=False, k_hunit=False + ) + + pkz_nl_interp = interpolate.RegularGridInterpolator( + (z_nonlin, k_nonlin), pk_nonlin, bounds_error=False, fill_value=None + ) + + k_lin, z_lin, pk_lin = results.get_linear_matter_power_spectrum( + hubble_units=False, k_hunit=False + ) + + pkz_lin_interp = interpolate.RegularGridInterpolator( + (z_lin, k_lin), pk_lin, bounds_error=False, fill_value=None + ) + + rz_interp_wantz = rz_interp(True) + rz_interp_noz = rz_interp(False) + W_gg_vals = np.array([W_gg(z, rz_interp_noz) for z in zs]) + W_gg_interp = interpolate.interp1d( + zs, W_gg_vals, bounds_error=False, fill_value="extrapolate" + ) + + ########################################################################################### + xis = np.array([xi(theta_rad, kmax, True) for kmax in kmaxs]) + xip = xis[:, 0] + xim = xis[:, 1] + + # Write results immediately to avoid thread conflicts + with open(f"{output_root}_xip.txt", "a") as f: + new_arr = np.concatenate(([theta], xip)) + np.savetxt(f, new_arr, fmt="%.8e") + + with open(f"{output_root}_xim.txt", "a") as f: + new_arr = np.concatenate(([theta], xim)) + np.savetxt(f, new_arr, fmt="%.8e") + + return theta + + +########################################################################################### + +if __name__ == "__main__": + """Run the shear-correlation calculation in parallel. + + The script expects three command-line arguments: + + 1. Block index specifying which subset of angular scales to process. + 2. Path to the source redshift distribution file. + 3. Output file prefix. + + The 50 angular scales between 1 and 20 arcmin are divided into + blocks of 10 values. Each block is processed in parallel using + multiprocessing, with one worker per angular scale. Each worker + computes xi+ and xi- over the predefined range of kmax values and + appends the results to the output files. + """ + i = int(sys.argv[1]) + nz_file = sys.argv[2] + output_root = sys.argv[3] + + thetas = np.linspace(1, 20, 50) + theta_block = thetas[i * 10 : (i + 1) * 10] + + # Run in parallel to speed up calculations for multiple angular scales + with Pool(processes=10) as pool: + pool.starmap( + process_theta, [(theta, nz_file, output_root) for theta in theta_block] + ) From 16db5b24b25cb037fe309a15e250457edb82e971 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 03:03:27 +0200 Subject: [PATCH 037/160] One binning-agnostic xi rule (grid + covariance are config, not a second rule) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reporting and integration ξ± measurements were two rules and two drivers for one computation. Collapse them: rule `xi` measures any binning, files are named by that binning (.txt + .sacc), and the grid label + covariance treatment are resolved from the wildcards via XI_GRIDS (reporting → no covariance, the ξ block arrives at assembly from CosmoCov; integration → DiagonalCovariance from TreeCorr varxip/varxim). Workflows with no cosmo_val block fall back to their fiducial grids; an unnamed binning measures as plain reporting. run_2pcf.py gains --grid/--covariance and drops run_2pcf_highres.py entirely (its hand-rolled catalogue loader, cat_config resolver and bare-host MPI path were a second implementation of CosmologyValidation.calculate_2pcf). The SACC part now stamps the npatch actually measured. calculate_2pcf writes patch results + covariance only for npatch > 1, so the fine grid no longer serialises a dense (2*nbins)^2 block that nothing reads. Renames {version}_xi_reporting_.sacc and {version}_xi_integration.sacc to {version}_xi_.sacc; consumers go through cv_xi_sacc(version, grid). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014hSQTC6WwTH1p9w4FuKJGz --- papers/bmodes/scripts/run_xi_sweep.py | 12 +- papers/cosmo_val/config/config.yaml | 5 +- pyproject.toml | 4 +- src/sp_validation/cosmo_val/real_space.py | 15 +- .../tests/test_bmodes_workflow_dry_run.py | 4 +- src/sp_validation/tests/test_cli_seams.py | 11 +- workflow/common.py | 4 +- workflow/rules/cosmo_val.smk | 47 +- workflow/rules/twopoint.smk | 150 ++++--- workflow/scripts/run_2pcf.py | 65 ++- workflow/scripts/run_2pcf_highres.py | 416 ------------------ 11 files changed, 181 insertions(+), 552 deletions(-) delete mode 100644 workflow/scripts/run_2pcf_highres.py diff --git a/papers/bmodes/scripts/run_xi_sweep.py b/papers/bmodes/scripts/run_xi_sweep.py index 0aa5f859..5e411c34 100644 --- a/papers/bmodes/scripts/run_xi_sweep.py +++ b/papers/bmodes/scripts/run_xi_sweep.py @@ -38,7 +38,9 @@ GRIDS = { "reporting": dict(min_sep=1.0, max_sep=250.0, nbins=20, npatch=1), - "integration": dict(min_sep=0.5, max_sep=300.0, nbins=1000, npatch=1), + "integration": dict( + min_sep=0.5, max_sep=300.0, nbins=1000, npatch=1, covariance="diagonal" + ), } @@ -73,14 +75,14 @@ def _from_cli(argv=None): for ver in versions: for grid in a.grids: # The sweep consumes only the .txt dump (cosebis_version_comparison - # reconstructs it by binning). run_2pcf is born-as-SACC, so give its - # reporting part a grid-qualified name — the default {ver}_xi_reporting.sacc - # carries no binning, so the two grids per version would collide. + # reconstructs it by binning). run_2pcf is born-as-SACC; its default + # part name carries the binning, so the two grids per version land + # in distinct files without an explicit sacc_out. run_2pcf( ver=ver, cat_config=a.cat_config, output_dir=a.out, - sacc_out=os.path.join(a.out, f"{ver}_xi_reporting_{grid}.sacc"), + grid=grid, **GRIDS[grid], ) diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 232513c7..fac4cdbb 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -58,8 +58,9 @@ cosmo_val: kmax: 20 kmax_extrapolate: 500 - # Integration-grid ξ± (the shared fine grid measured once per version by rule - # xi_highres as {version}_xi_integration.sacc). Both estimators consume this one + # Integration-grid ξ± (the shared fine grid measured once per version by the + # binning-agnostic `xi` rule, which resolves this binning to grid='integration' + # and attaches a DiagonalCovariance). Both estimators consume this one # part: pure-E/B uses the full range (it must strictly contain the reporting # grid [1, 250]); COSEBIs scale-cuts it up to 0.9. Owned here, not inside either # consumer's block. Decoupled from covariance.smk's own FIDUCIAL grid. diff --git a/pyproject.toml b/pyproject.toml index 9c067178..75e91dd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -155,8 +155,8 @@ glass = [ # Snakemake workflow and cross-validation runners are available. workflow = [ "snakemake", - # run_2pcf_highres.py drives the MPI convergence run; the container ships - # OpenMPI (/opt/ompi) so mpi4py builds against it. + # Optional MPI runners (the container ships OpenMPI at /opt/ompi, so mpi4py + # builds against it). "mpi4py", # NOTE: workflow/scripts/cv_*.py also import `cv_runner`, which is not # published or resolvable (no public repo found) — left undeclared pending diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index a0852d77..ce3b7ecc 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -47,9 +47,10 @@ def calculate_2pcf(self, ver, npatch=None, **treecorr_config): created during the process. - The ``.txt`` TreeCorr dump is the only raw byproduct written here (read back by the covariance machinery and the skip-if-exists). The - analysis ξ± data product is born as SACC in the Snakemake scripts - (``run_2pcf.py`` coarse / ``run_2pcf_highres.py`` fine), which call - ``xi_to_sacc``; there is no DES-style ξ FITS writer anymore. + analysis ξ± data product is born as SACC in ``run_2pcf.py`` (one + binning-agnostic driver for both the reporting and the fine + integration grid), which calls ``xi_to_sacc``; there is no + DES-style ξ FITS writer anymore. """ self.print_magenta(f"Computing {ver} ξ±") @@ -98,7 +99,13 @@ def calculate_2pcf(self, ver, npatch=None, **treecorr_config): # Process the catalog & write the correlation functions gg.process(cat_gal) - gg.write(out_fname, write_patch_results=True, write_cov=True) + # Patch results + the jackknife covariance only make sense (and are + # only affordable) for npatch > 1: at npatch=1 var_method is "shot", + # so the "covariance" is just the varxip/varxim already written as + # per-bin columns, while the dense (2*nbins)^2 block would dominate + # the file on the fine integration grid (nbins ~ 1000). + write_cov = int(npatch) > 1 + gg.write(out_fname, write_patch_results=write_cov, write_cov=write_cov) # Add correlation object to class if not hasattr(self, "cat_ggs"): diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index cc6a293c..136be047 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -94,5 +94,7 @@ def test_cosmo_val_workflow_assemble_dry_runs(): assert "rule assemble_sacc:" in out, out assert f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" in out, out assert f"pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits" in out, out - for part in ("_xi_reporting_", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): + # The ξ± part is named by its reporting binning (one binning-agnostic `xi` + # rule serves the reporting and integration grids alike). + for part in ("_xi_minsep=", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): assert part in out, f"missing {part} part in assemble DAG:\n{out}" diff --git a/src/sp_validation/tests/test_cli_seams.py b/src/sp_validation/tests/test_cli_seams.py index 069d8ccc..4eaaa2de 100644 --- a/src/sp_validation/tests/test_cli_seams.py +++ b/src/sp_validation/tests/test_cli_seams.py @@ -40,16 +40,17 @@ def test_run_xi_sweep_run_2pcf_call_binds(): run_2pcf_mod = _load(root / "workflow/scripts/run_2pcf.py", "run_2pcf_seam") sig = inspect.signature(run_2pcf_mod.run_2pcf) # Exactly the keyword set run_xi_sweep._from_cli passes (grid params spread - # from GRIDS: min_sep/max_sep/nbins/npatch). + # from GRIDS: min_sep/max_sep/nbins/npatch, plus covariance on the fine grid). sig.bind( ver="V", cat_config="/cfg.yaml", output_dir="/out", - sacc_out="/out/V_xi_reporting_reporting.sacc", - min_sep=1.0, - max_sep=250.0, - nbins=20, + grid="integration", + min_sep=0.5, + max_sep=300.0, + nbins=1000, npatch=1, + covariance="diagonal", ) # And the removed kwarg must NOT bind (guards against a silent re-add). with pytest.raises(TypeError): diff --git a/workflow/common.py b/workflow/common.py index 1a168041..7d0ec2c4 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -10,8 +10,8 @@ # Snakefile, and so resolves to the generic workflow dir of the running checkout # regardless of which paper composes it — unlike workflow.basedir, which under # `module` composition reflects the composing paper). Rules that shell out to a -# script directly (the MPI xi_highres run can't go through Snakemake's `script:` -# directive) interpolate this instead of a hardcoded pure_eb/ compat-symlink +# script directly (rather than through Snakemake's `script:` directive) +# interpolate this instead of a hardcoded pure_eb/ compat-symlink # path. /automnt/n17data is the automount of the container-bound /n17data. WORKFLOW_SCRIPTS = os.path.join(os.path.dirname(os.path.realpath(__file__)), "scripts") diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 0cac5126..3923e5d0 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -52,13 +52,7 @@ def cv_xi_txt(version): Mirrors the out_fname f-string in cosmo_val.calculate_2pcf: {ver}_xi_minsep=..._maxsep=..._nbins=..._npatch=...txt """ - return str( - COSMO_VAL - / ( - f"{version}_xi_minsep={CV['theta_min']}_maxsep={CV['theta_max']}" - f"_nbins={CV['nbins']}_npatch={CV['npatch']}.txt" - ) - ) + return str(COSMO_VAL / f"{version}_xi_{xi_binning('reporting')}.txt") def cv_rho_stats(version): @@ -154,29 +148,20 @@ def cv_rho_tau_sacc(version): ) -def cv_xi_reporting_sacc(version): - """Reporting ξ± SACC part the xi rule (run_2pcf.py) writes for a version. - - Carries the reporting-binning suffix so requesting it binds the xi job's - wildcards (the rule's txt + reporting .sacc outputs share one wildcard set). - """ - return str( - COSMO_VAL - / ( - f"{version}_xi_reporting_minsep={CV['theta_min']}_maxsep={CV['theta_max']}" - f"_nbins={CV['nbins']}_npatch={CV['npatch']}.sacc" - ) - ) - +def cv_xi_sacc(version, grid): + """ξ± SACC part the `xi` rule writes for a version on a named grid. -def cv_xi_integration_sacc(version): - """Integration-grid ξ± SACC part the xi_highres rule writes, per version. + Named by its binning (xi_binning, twopoint.smk), which is what binds the xi + job's wildcards; the grid label and the covariance treatment are resolved + from that binning by the rule. - Intermediate per-statistic part (grid='integration', its own DiagonalCovariance - from TreeCorr varxip/varxim). NOT folded into the terminal {version}.sacc (see - #247 ruling) — COSEBIs and pure-E/B consume it directly. + grid='reporting' is the analysis part (no covariance block until assembly + injects the CosmoCov one); grid='integration' is the fine-grid part COSEBIs + and pure-E/B consume, carrying its own DiagonalCovariance from TreeCorr + varxip/varxim. The integration part is intermediate: it stays standalone and + is NOT folded into the terminal {version}.sacc (see #247 ruling). """ - return str(COSMO_VAL / f"{version}_xi_integration.sacc") + return str(COSMO_VAL / f"{version}_xi_{xi_binning(grid)}.sacc") def cv_analysis_sacc(version): @@ -372,8 +357,8 @@ rule cv_pure_eb: """Pure E/B-mode decomposition for one version (config-space).""" input: xi=lambda w: cv_xi_txt(w.version), - xi_reporting=lambda w: cv_xi_reporting_sacc(w.version), - xi_integration=lambda w: cv_xi_integration_sacc(w.version), + xi_reporting=lambda w: cv_xi_sacc(w.version, "reporting"), + xi_integration=lambda w: cv_xi_sacc(w.version, "integration"), output: npz=cv_pure_eb_npz("{version}"), sacc=cv_pure_eb_sacc("{version}"), @@ -397,7 +382,7 @@ rule cv_cosebis: """COSEBIs E/B decomposition for one version (config-space, fine binning).""" input: xi=lambda w: cv_xi_txt(w.version), - xi_integration=lambda w: cv_xi_integration_sacc(w.version), + xi_integration=lambda w: cv_xi_sacc(w.version, "integration"), output: npz=cv_cosebis_npz("{version}"), sacc=cv_cosebis_sacc("{version}"), @@ -489,7 +474,7 @@ def cv_assemble_inputs(version): config toggles the harmonic-space BB into the analysis. """ parts = dict( - xi_reporting=cv_xi_reporting_sacc(version), + xi_reporting=cv_xi_sacc(version, "reporting"), cosebis=cv_cosebis_sacc(version), pure_eb=cv_pure_eb_sacc(version), rho_tau=cv_rho_tau_sacc(version), diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 69c516e9..8d449ffc 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -1,20 +1,84 @@ # Two-point data-vector rules: xi, rho/tau, and pseudo-Cl products. -# WORKFLOW_SCRIPTS (from common.py) is the generic workflow's scripts dir, -# resolved from the running checkout — used by the raw-shell MPI xi_highres rule. + +# --------------------------------------------------------------------------- +# ξ± angular grids +# --------------------------------------------------------------------------- +# A grid IS a binning: (min_sep, max_sep, nbins, npatch) plus how the +# born-as-SACC part carries its covariance. The reporting grid is the analysis +# one (its ξ covariance is injected at assembly from CosmoCov, so the part is +# written bare); the integration grid is the fine grid COSEBIs and pure-E/B +# integrate over, whose only covariance estimate is TreeCorr's shot-noise +# varxip/varxim — attached as a DiagonalCovariance. +# +# Both grids are measured by the single `xi` rule below: files are named by +# binning, so the grid label and the covariance mode are *resolved* from the +# wildcards rather than duplicated into a second rule. Workflows that carry no +# cosmo_val block (e.g. papers/bmodes) fall back to their fiducial grids; a +# binning matching no named grid is measured as a plain reporting-style +# measurement (no covariance). +def _xi_grids(): + cv = config.get("cosmo_val", {}) + reporting = ( + { + "min_sep": cv["theta_min"], + "max_sep": cv["theta_max"], + "nbins": cv["nbins"], + "npatch": cv["npatch"], + } + if cv + else {k: FIDUCIAL[k] for k in ("min_sep", "max_sep", "nbins", "npatch")} + ) + integration = dict( + cv.get("integration") + or { + "min_sep": FIDUCIAL["min_sep_int"], + "max_sep": FIDUCIAL["max_sep_int"], + "nbins": FIDUCIAL["nbins_int"], + } + ) + integration.setdefault("npatch", 1) + return { + "reporting": {**reporting, "covariance": "none"}, + "integration": {**integration, "covariance": "diagonal"}, + } + + +XI_GRIDS = _xi_grids() +XI_DEFAULT_GRID = ("reporting", "none") + + +def xi_binning(grid): + """The `minsep=..._maxsep=..._nbins=..._npatch=...` tag of a named grid.""" + g = XI_GRIDS[grid] + return ( + f"minsep={g['min_sep']}_maxsep={g['max_sep']}" + f"_nbins={g['nbins']}_npatch={g['npatch']}" + ) + + +def xi_grid_of(wildcards): + """(grid label, covariance mode) for the binning a job was requested with.""" + key = (wildcards.min_sep, wildcards.max_sep, wildcards.nbins, wildcards.npatch) + for name, g in XI_GRIDS.items(): + if tuple(str(g[k]) for k in ("min_sep", "max_sep", "nbins", "npatch")) == key: + return name, g["covariance"] + return XI_DEFAULT_GRID rule xi: + """TreeCorr ξ±(θ) for one version on one angular grid. + + Binning-agnostic: the reporting and integration measurements are the same + job with different wildcards. The raw TreeCorr .txt byproduct (read back by + the covariance machinery and by the skip-if-exists) and the born-as-SACC + part are named by that binning, so a request for either binds unambiguously + — and the grid label + covariance treatment come from XI_GRIDS. + """ input: catalog=get_shear_catalog, output: - # Raw TreeCorr .txt byproduct (read back by covariance + skip-if-exists) - # and the born-as-SACC reporting ξ± part (no covariance until the - # assemble_sacc rule injects the CosmoCov block). Both outputs carry the - # same reporting-binning wildcards — Snakemake requires every output of a - # rule to share one wildcard set, and it keeps the reporting .sacc name - # self-describing so requesting it binds the xi job unambiguously. txt=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), - xi_reporting=str(COSMO_VAL / "{version}_xi_reporting_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), + sacc=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 24 params: ver="{version}", @@ -22,73 +86,19 @@ rule xi: max_sep="{max_sep}", nbins="{nbins}", npatch="{npatch}", + cat_config=CAT_CONFIG, + grid=lambda w: xi_grid_of(w)[0], + covariance=lambda w: xi_grid_of(w)[1], resources: - mem_mb=30000, + # The fine integration grid needs more memory and wall time than the + # ~20-bin reporting one; scale on nbins rather than splitting the rule. + mem_mb=lambda w: 40000 if int(w.nbins) > 100 else 30000, disk_mb=20000, - runtime=360, + runtime=lambda w: 600 if int(w.nbins) > 100 else 360, script: "../scripts/run_2pcf.py" -# Integration-grid ξ± measured by xi_highres. The cosmo_val paper owns a dedicated -# cosmo_val.integration block ([0.08, 300] @ 1000 bins); other workflows sharing -# this file (e.g. papers/bmodes, whose config carries no cosmo_val section) fall -# back to their fiducial integration grid. Evaluated at parse time, so the lookup -# must not assume the cosmo_val key exists. -_INTEGRATION = config.get("cosmo_val", {}).get("integration") or { - "min_sep": FIDUCIAL["min_sep_int"], - "max_sep": FIDUCIAL["max_sep_int"], - "nbins": FIDUCIAL["nbins_int"], -} - - -rule xi_highres: - """High-resolution integration-grid xi for COSEBIs + pure-E/B, per version. - - Intermediate born-as-SACC part: {version}_xi_integration.sacc (a - DiagonalCovariance from TreeCorr varxip/varxim). COSEBIs and pure-E/B consume - it; it stays a standalone per-part file and does not join the terminal - {version}.sacc (see #247 ruling). The raw .txt dump is kept as a convergence - byproduct. - - In-container single-process TreeCorr: at the config-driven nbins_int=1000 grid - this is a normal single-node job (the global container: in the Snakefile makes - a plain shell: run in-container). run_2pcf_highres.py runs its single-process - path when not launched under mpiexec. The historical 10k-bin bare-host MPI path - is removed as unnecessary. - """ - input: - catalog=get_shear_catalog, - output: - # Only the uniquely-named SACC part is tracked. The raw TreeCorr .txt dump - # run_2pcf_highres.py writes ({version}_xi_minsep=..._nbins=..._npatch=1.txt) - # is left UNDECLARED: it is a convergence byproduct nothing in the DAG - # consumes (cv_xi_txt is the reporting grid), and declaring it would collide - # with rule xi's wildcard txt output (same filename pattern) — an - # AmbiguousRuleException. Shared integration grid (cosmo_val.integration: - # [0.08, 300] at 1000 bins) so the single part serves both consumers: - # pure-E/B needs it to strictly contain its reporting grid down to 0.08; - # COSEBIs scale-cuts on the same part. Decoupled from covariance.smk. - xi_integration=str(COSMO_VAL / "{version}_xi_integration.sacc"), - params: - version="{version}", - cat_config=CAT_CONFIG, - min_sep=_INTEGRATION["min_sep"], - max_sep=_INTEGRATION["max_sep"], - nbins=_INTEGRATION["nbins"], - out=str(COSMO_VAL), - scripts=WORKFLOW_SCRIPTS, - threads: 24 - resources: - mem_mb=40000, - runtime=600, - shell: - "python {params.scripts}/run_2pcf_highres.py " - "--version {params.version} --cat-config {params.cat_config} " - "--min-sep {params.min_sep} --max-sep {params.max_sep} " - "--nbins {params.nbins} --npatch 1 --out {params.out}" - - rule run_cosmo_val: """Full CosmoVal diagnostic suite.""" output: diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 3e513479..10dafc53 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -12,19 +12,29 @@ --cat-config /path/to/cosmo_val/cat_config.yaml \ --out -The measurement itself is unchanged — ``CosmologyValidation.calculate_2pcf`` -does the TreeCorr work and writes the ``.txt`` dump (a raw byproduct the -covariance machinery reads back). The analysis ξ± data product is then born as -SACC here: ``{ver}_xi_reporting.sacc``, a *part* on the reporting grid via -``xi_to_sacc(grid="reporting", ...)`` carrying ``theta_nom``/``npairs``/``weight`` -tags but NO covariance (the ξ block is supplied at assembly from the CosmoCov -theory covariance). ``output_dir`` is passed explicitly (rather than via the -``COSMO_VAL`` env hook) so lc can point each run at its own ``{output}`` tree. +The measurement is binning-agnostic: the reporting and the fine integration +grids are the same compute with different ``--min-sep/--max-sep/--nbins``. +``CosmologyValidation.calculate_2pcf`` does the TreeCorr work and writes the +``.txt`` dump (a raw byproduct the covariance machinery and the convergence +consumers read back); the ξ± data product is then born as SACC here, a *part* +named by its binning and tagged with its ``grid``: + +* ``--grid reporting`` (default) — ``--covariance none``: no covariance block, + because the ξ block is supplied at assembly from the CosmoCov theory + covariance. +* ``--grid integration`` — ``--covariance diagonal``: a ``DiagonalCovariance`` + from TreeCorr ``varxip``/``varxim``, the only covariance estimate available at + npatch=1, which is what COSEBIs and pure-E/B consume. + +``output_dir`` is passed explicitly (rather than via the ``COSMO_VAL`` env hook) +so lc can point each run at its own ``{output}`` tree. """ import argparse import os +import numpy as np + from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation from sp_validation.cosmo_val.sacc_writers import xi_to_sacc @@ -39,6 +49,8 @@ def run_2pcf( cat_config, output_dir, sacc_out=None, + grid="reporting", + covariance="none", ): """Measure ξ±(θ) for ``ver`` and write its reporting SACC part. @@ -60,6 +72,8 @@ def run_2pcf( versions=[ver], catalog_config=cat_config, output_dir=output_dir, + # so the SACC provenance metadata stamps the npatch actually measured + npatch=npatch, ) gg = cv.calculate_2pcf( ver=ver, @@ -69,24 +83,29 @@ def run_2pcf( nbins=nbins, ) - # Born-as-SACC reporting ξ± part: no covariance here (added at assembly from - # the CosmoCov theory covariance). theta = meanr; theta_nom = rnom. + # Born-as-SACC ξ± part. theta = meanr; theta_nom = rnom. + if covariance not in ("none", "diagonal"): + raise ValueError(f"unknown covariance mode {covariance!r}") s = xi_to_sacc( cv.sacc_nz(ver), cv.sacc_metadata(ver), gg.meanr, gg.xip, gg.xim, - grid="reporting", + grid=grid, theta_nom=gg.rnom, npairs=gg.npairs, weight=gg.weight, + variances=( + np.concatenate([gg.varxip, gg.varxim]) if covariance == "diagonal" else None + ), ) out_path = sacc_out or os.path.join( - output_dir or cv.cc["paths"]["output"], f"{ver}_xi_reporting.sacc" + output_dir or cv.cc["paths"]["output"], + f"{ver}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc", ) sacc_io.save(s, out_path, type="data") - print(f"Wrote reporting ξ± SACC part: {out_path}") + print(f"Wrote {grid} ξ± SACC part: {out_path}") return gg @@ -104,9 +123,13 @@ def _from_snakemake(smk): # class defaults (./cat_config.yaml, COSMO_VAL env) otherwise. cat_config=p.get("cat_config", "./cat_config.yaml"), output_dir=p.get("output_dir", None), + # Grid label + covariance treatment are resolved by the rule from the + # binning wildcards (workflow/rules/twopoint.smk XI_GRIDS). + grid=p.get("grid", "reporting"), + covariance=p.get("covariance", "none"), # Write the SACC part exactly where the rule declares it (the .txt # byproduct still lands under the resolved output dir via _output_path). - sacc_out=smk.output["xi_reporting"], + sacc_out=smk.output["sacc"], ) @@ -133,6 +156,18 @@ def _from_cli(argv=None): "--cat-config", required=True, help="Absolute path to cat_config.yaml" ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") + ap.add_argument( + "--grid", + default="reporting", + choices=["reporting", "integration"], + help="SACC grid tag of the measured part", + ) + ap.add_argument( + "--covariance", + default="none", + choices=["none", "diagonal"], + help="Covariance carried by the SACC part (diagonal: varxip/varxim)", + ) a = ap.parse_args(argv) run_2pcf( ver=a.ver, @@ -142,6 +177,8 @@ def _from_cli(argv=None): npatch=a.npatch, cat_config=a.cat_config, output_dir=a.out, + grid=a.grid, + covariance=a.covariance, ) diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py deleted file mode 100644 index 6049e105..00000000 --- a/workflow/scripts/run_2pcf_highres.py +++ /dev/null @@ -1,416 +0,0 @@ -#!/usr/bin/env python -""" -High-resolution ξ± measurement for COSEBIS/pure-EB integration. - -Computes TreeCorr GGCorrelation on the fine integration angular grid (default -1000 log bins, config-driven) required for accurate COSEBIS/pure-EB mode -integration. Uses MPI for patch-pair distribution across nodes when available; -falls back to multi-threaded single-process otherwise (the default at 1000 bins). - -Reference: Asgari et al. 2017 motivates a fine integration grid; the B-modes -paper found no substantial 1k-vs-10k difference, so the operational default is -1000 bins (see config nbins_int). - -Usage: - # MPI (via Slurm submission script): - mpiexec --map-by ppr:1:node python run_2pcf_highres.py \ - --cat-config /path/to/cosmo_val/cat_config.yaml --out - - # Single-process fallback: - python run_2pcf_highres.py \ - --cat-config /path/to/cosmo_val/cat_config.yaml --out -""" - -import argparse -import os -import time - -import numpy as np -import treecorr -from astropy.io import fits - -# sacc_io depends only on numpy + sacc (no healpy/cs_util), so the born-as-SACC -# integration ξ± write works on the bare-host MPI path too, where the full cosmo_val -# stack is unavailable. -from sp_validation import sacc_io -from sp_validation.cosmo_val.sacc_writers import xi_to_sacc - -try: - # In-container path: full sp_validation stack available. - from sp_validation.cosmo_val import CosmologyValidation - - _HAVE_COSMO_VAL = True -except ImportError: - # Bare-host path (host OpenMPI + host python for an optional MPI run): the - # full sp_validation stack (cs_util.plots -> healpy/healsparse) is not - # installed. This measurement only needs the shear catalog path + column - # names, which are a pure cat_config.yaml lookup — resolve them standalone. - CosmologyValidation = None - _HAVE_COSMO_VAL = False - -# --------------------------------------------------------------------------- -# MPI setup (graceful fallback) -# --------------------------------------------------------------------------- -try: - from mpi4py import MPI - - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - size = comm.Get_size() - USE_MPI = size > 1 -except ImportError: - comm = None - rank = 0 - size = 1 - USE_MPI = False - -# --------------------------------------------------------------------------- -# Configuration -# --------------------------------------------------------------------------- -# Shear response (R=1 for all SP catalogs) -R = 1.0 - -# Detect threads from Slurm or fall back to OS count -NUM_THREADS = int(os.environ.get("SLURM_CPUS_PER_TASK", os.cpu_count() or 24)) - -# The catalog path, ellipticity/weight columns, TreeCorr grid, patch count and -# output directory are resolved from the CLI in main() (defaults reproduce the -# historical hardcoded values for a no-arg run). They are declared here as -# module globals so the rank-aware helpers below resolve them at call time; the -# catalog path + columns come from cat_config + version exactly as run_2pcf.py -# resolves them (via CosmologyValidation). -CAT_PATH = None -VERSION = None -E1_COL = None -E2_COL = None -W_COL = None -REDSHIFT_PATH = None # n(z) file for the SACC tracer -TMIN = None # arcmin -TMAX = None # arcmin -NBINS = None -NPATCH = None -OUTPUT_DIR = None -PATCH_FILE = None - - -def parse_args(argv=None): - """CLI mirroring run_xi_sweep's signature; defaults reproduce prior behavior.""" - ap = argparse.ArgumentParser( - description="High-resolution TreeCorr ξ± measurement for COSEBIS integration." - ) - ap.add_argument( - "--config", - default=None, - help="Path to bmodes config.yaml (accepted for signature parity with " - "run_xi_sweep; not read by this measurement).", - ) - ap.add_argument( - "--cat-config", required=True, help="Absolute path to cat_config.yaml" - ) - ap.add_argument( - "--version", - default="SP_v1.4.6.3_leak_corr", - help="Catalog version key in cat_config", - ) - ap.add_argument("--nbins", type=int, default=1000, help="Number of log bins") - ap.add_argument("--npatch", type=int, default=50, help="TreeCorr patch count") - ap.add_argument( - "--min-sep", type=float, default=0.5, help="Min separation [arcmin]" - ) - ap.add_argument( - "--max-sep", type=float, default=300.0, help="Max separation [arcmin]" - ) - ap.add_argument("--out", required=True, help="Output directory (lc {output})") - return ap.parse_args(argv) - - -def log(msg): - """Print with timestamp on rank 0 only.""" - if rank == 0: - print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) - - -def load_catalog(): - """Load shear catalog and apply mean subtraction.""" - log(f"Loading catalog: {CAT_PATH}") - hdul = fits.open(CAT_PATH, memmap=True) - data = hdul[1].data - - ra = np.array(data["ra"], dtype=np.float64) - dec = np.array(data["dec"], dtype=np.float64) - e1 = np.array(data[E1_COL], dtype=np.float64) - e2 = np.array(data[E2_COL], dtype=np.float64) - w = np.array(data[W_COL], dtype=np.float64) - hdul.close() - - log(f" {len(ra):,} galaxies loaded") - - # Additive bias: c = _w (R=1 for SP catalogs) - c1 = np.average(e1 / R, weights=w) - c2 = np.average(e2 / R, weights=w) - log(f" Additive bias: c1={c1:.6e}, c2={c2:.6e}") - - # Calibrated shear: g = (e - c) / R - g1 = (e1 - c1) / R - g2 = (e2 - c2) / R - - return ra, dec, g1, g2, w - - -def _wait_for_file(path, timeout=300, interval=1.0): - """Block until `path` is visible to this node, defeating NFS dir caching. - - On a multi-node run the rank that wrote `path` sees it immediately, but - peer nodes can carry a stale negative directory-cache entry past an MPI - Barrier. Re-listing the parent directory forces an NFS attribute refresh; - poll that until the entry appears (or raise after `timeout` seconds). - """ - parent = os.path.dirname(path) or "." - name = os.path.basename(path) - waited = 0.0 - while waited < timeout: - try: - if name in os.listdir(parent): - return - except FileNotFoundError: - pass - time.sleep(interval) - waited += interval - raise TimeoutError(f"patch-center file not visible after {timeout}s: {path}") - - -def compute_patch_centers(ra, dec): - """Compute patch centers from subsampled catalog (rank 0 only).""" - if os.path.exists(PATCH_FILE): - log(f"Using existing patch centers: {PATCH_FILE}") - return - - if rank != 0: - return - - log(f"Computing patch centers (npatch={NPATCH}) from 1% subsample...") - rng = np.random.default_rng(42) - n_sub = max(len(ra) // 100, NPATCH * 100) - idx = rng.choice(len(ra), size=n_sub, replace=False) - - cat_sub = treecorr.Catalog( - ra=ra[idx], - dec=dec[idx], - ra_units="degrees", - dec_units="degrees", - npatch=NPATCH, - ) - cat_sub.write_patch_centers(PATCH_FILE) - log(f" Wrote patch centers to {PATCH_FILE}") - del cat_sub - - -def write_xi_integration_sacc(gg): - """Write the integration-grid ξ± SACC part (``{version}_xi_integration.sacc``). - - This is an intermediate per-statistic part — COSEBIs and pure-E/B consume it. - It stays a standalone per-part file and does not join the terminal - ``{version}.sacc`` (see #247 ruling). It carries a ``DiagonalCovariance`` from - TreeCorr ``varxip``/``varxim`` - (npatch=1 leaves shot-noise variance as the only covariance estimate). - Both run paths land here: in-container this uses the full SACC stack; on the - bare-host MPI run only ``sacc_io`` + the n(z) file are needed (no healpy). - """ - z, nz = np.loadtxt(REDSHIFT_PATH, unpack=True) - metadata = { - "catalogue_version": VERSION, - "sp_validation_version": _sp_validation_version(), - "npatch": 1, - } - s = xi_to_sacc( - {0: (z, nz)}, - metadata, - gg.meanr, - gg.xip, - gg.xim, - grid="integration", - theta_nom=gg.rnom, - variances=np.concatenate([gg.varxip, gg.varxim]), - ) - out_path = os.path.join(OUTPUT_DIR, f"{VERSION}_xi_integration.sacc") - sacc_io.save(s, out_path, type="data") - log(f" Wrote {out_path}") - - -def _sp_validation_version(): - """Best-effort package version for the SACC metadata (empty if unavailable).""" - try: - from sp_validation import __version__ - - return __version__ - except Exception: - return "" - - -def resolve_shear_config(cat_config_path, version): - """Standalone shear-config resolver (bare-host fallback for CosmologyValidation). - - Reproduces exactly the ``cc[version]["shear"]`` fields this measurement reads - (path, e1_col, e2_col, w_col), replicating CosmologyValidation's two - transforms: (1) subdir-relative path resolution, and (2) the ``_leak_corr`` - virtual version — deep-copy the base version and swap - e1_col/e2_col -> e1_col_corrected/e2_col_corrected. See - sp_validation/cosmo_val/core.py. - """ - import copy - - import yaml - - with open(cat_config_path) as fh: - cc = yaml.load(fh, Loader=yaml.FullLoader) - - def resolve_paths(ver): - subdir = os.fspath(cc[ver]["subdir"]) - for section in cc[ver].values(): - if isinstance(section, dict) and "path" in section: - p = section["path"] - if not os.path.isabs(p): - section["path"] = os.path.join(subdir, p) - - leak_suffix = "_leak_corr" - if version in cc: - resolve_paths(version) - elif version.endswith(leak_suffix): - base = version[: -len(leak_suffix)] - if base not in cc: - raise ValueError(f"Base version '{base}' not in cat_config for '{version}'") - resolve_paths(base) - base_shear = cc[base]["shear"] - if "e1_col_corrected" not in base_shear or "e2_col_corrected" not in base_shear: - raise ValueError( - f"{base} lacks e1_col_corrected/e2_col_corrected; cannot form {version}" - ) - cc[version] = copy.deepcopy(cc[base]) - cc[version]["shear"]["e1_col"] = base_shear["e1_col_corrected"] - cc[version]["shear"]["e2_col"] = base_shear["e2_col_corrected"] - resolve_paths(version) - else: - raise ValueError(f"Version '{version}' not found in cat_config") - - return cc[version]["shear"] - - -def main(): - global CAT_PATH, VERSION, E1_COL, E2_COL, W_COL, REDSHIFT_PATH - global TMIN, TMAX, NBINS, NPATCH, OUTPUT_DIR, PATCH_FILE - - args = parse_args() - VERSION = args.version - NBINS = args.nbins - NPATCH = args.npatch - TMIN = args.min_sep - TMAX = args.max_sep - OUTPUT_DIR = args.out - - # Resolve catalog path + ellipticity/weight columns from cat_config + version - # exactly as run_2pcf.py does (applies the _leak_corr column swap and the - # subdir path resolution). In-container this uses CosmologyValidation; on the - # bare-host MPI fallback it uses the standalone cat_config resolver, which is - # byte-identical for the shear-config fields this measurement reads. - if _HAVE_COSMO_VAL: - cv = CosmologyValidation( - versions=[VERSION], catalog_config=args.cat_config, output_dir=OUTPUT_DIR - ) - shear_cfg = cv.cc[VERSION]["shear"] - else: - shear_cfg = resolve_shear_config(args.cat_config, VERSION) - CAT_PATH = shear_cfg["path"] - E1_COL = shear_cfg["e1_col"] - E2_COL = shear_cfg["e2_col"] - W_COL = shear_cfg["w_col"] - REDSHIFT_PATH = shear_cfg["redshift_path"] - - PATCH_FILE = os.path.join( - OUTPUT_DIR, - f"patch_centers_{VERSION}_{NPATCH}_{TMIN}_{TMAX}.dat", - ) - - t0 = time.time() - - log("=" * 60) - log("High-resolution ξ± measurement") - log(f" MPI: {'yes' if USE_MPI else 'no'} (ranks={size})") - log(f" Config: {NBINS:,} bins, [{TMIN}, {TMAX}] arcmin") - log(f" Patches: {NPATCH}, Threads/rank: {NUM_THREADS}") - log(f" Version: {VERSION}") - log("=" * 60) - - # All ranks load catalog (needed for TreeCorr patch assignment) - ra, dec, g1, g2, w = load_catalog() - - # Compute patch centers (rank 0 only; others wait) - compute_patch_centers(ra, dec) - if USE_MPI: - comm.Barrier() - # Cross-node visibility: rank 0 wrote PATCH_FILE on its node, but on a - # multi-node allocation the other ranks' nodes may not see it yet (NFS - # close-to-open + negative-dir caching persists past the Barrier). Poll - # with a forced directory refresh until it appears before reading it. - _wait_for_file(PATCH_FILE) - - # Create TreeCorr catalog with patch centers - log("Creating TreeCorr catalog with patches...") - cat = treecorr.Catalog( - ra=ra, - dec=dec, - g1=g1, - g2=g2, - w=w, - ra_units="degrees", - dec_units="degrees", - patch_centers=PATCH_FILE, - ) - cat.load() - cat.get_patches() - log(f" Catalog ready ({cat.nobj:,} objects, {cat.npatch} patches)") - - # Free raw arrays (TreeCorr holds its own copy) - del ra, dec, g1, g2, w - - # Compute GG correlation - log("Computing GGCorrelation...") - gg = treecorr.GGCorrelation( - min_sep=TMIN, - max_sep=TMAX, - nbins=NBINS, - sep_units="arcminutes", - verbose=2, - ) - - process_kwargs = {"num_threads": NUM_THREADS} - if USE_MPI: - process_kwargs["comm"] = comm - - gg.process(cat, **process_kwargs) - log(f" Correlation complete ({time.time() - t0:.0f}s elapsed)") - - # Write output (rank 0 only) - if rank == 0: - out_txt = os.path.join( - OUTPUT_DIR, - f"{VERSION}_xi_minsep={TMIN}_maxsep={TMAX}_nbins={NBINS}_npatch=1.txt", - ) - # Write only the main per-bin correlation. The convergence consumer - # (cosebis_binning_comparison.py) reads just the per-bin columns - # (np.loadtxt max_rows=nbins); the fine-grid jackknife cov is used - # nowhere. write_patch_results/write_cov=True serialised a full - # (2*nbins)^2 cov + patch blocks that nothing reads and also cost the - # estimate_cov compute; drop both. The patches - # still parallelise gg.process; gg.xip/gg.xim (values, FITS) are - # unaffected. - gg.write(out_txt, write_patch_results=False, write_cov=False) - log(f" Wrote {out_txt}") - - write_xi_integration_sacc(gg) - - elapsed = time.time() - t0 - log(f"Done! Total time: {elapsed / 3600:.1f}h ({elapsed:.0f}s)") - - -if __name__ == "__main__": - main() From b43796893046988ebc2935f8b646eb46531f45ce Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 03:05:26 +0200 Subject: [PATCH 038/160] Restore develop files wrongly deleted by stale-base cleanup (realspace paper, image-sims, mask configs, candide profile) --- config/calibration/mask_v1.X.4_im_sim.yaml | 70 ++ .../mask_v1.X.9_im_sim.overlay.yaml | 97 +++ config/calibration/mask_v1.X.9_im_sim.yaml | 77 ++ papers/realspace/S8_om_sigma8_whisker.py | 549 +++++++++++++ papers/realspace/best_fit_xipm.py | 497 ++++++++++++ papers/realspace/contours.py | 745 ++++++++++++++++++ papers/realspace/cov_masking.py | 82 ++ papers/realspace/get_chi2.py | 564 +++++++++++++ papers/realspace/get_chi2_glass_mock.py | 468 +++++++++++ papers/realspace/get_prior_psf_leakage.py | 163 ++++ papers/realspace/glass_mock_hist.py | 458 +++++++++++ papers/realspace/nonlin_k_analysis.py | 104 +++ scripts/compute_m_bias_image_sims.py | 361 +++++++++ scripts/diagnostics_image_sims.py | 227 ++++++ src/sp_validation/image_sims.py | 324 ++++++++ src/sp_validation/tests/test_image_sims.py | 253 ++++++ src/sp_validation/tests/test_mask_overlay.py | 85 ++ workflow/image_sims/Snakefile | 46 ++ workflow/image_sims/config.yaml | 89 +++ workflow/profiles/candide/config.yaml | 85 ++ workflow/rules/image_sims.smk | 568 +++++++++++++ workflow/scripts/im_build_manifest.py | 244 ++++++ workflow/scripts/im_compose_mask.py | 109 +++ 23 files changed, 6265 insertions(+) create mode 100644 config/calibration/mask_v1.X.4_im_sim.yaml create mode 100644 config/calibration/mask_v1.X.9_im_sim.overlay.yaml create mode 100644 config/calibration/mask_v1.X.9_im_sim.yaml create mode 100644 papers/realspace/S8_om_sigma8_whisker.py create mode 100644 papers/realspace/best_fit_xipm.py create mode 100644 papers/realspace/contours.py create mode 100644 papers/realspace/cov_masking.py create mode 100644 papers/realspace/get_chi2.py create mode 100644 papers/realspace/get_chi2_glass_mock.py create mode 100644 papers/realspace/get_prior_psf_leakage.py create mode 100644 papers/realspace/glass_mock_hist.py create mode 100644 papers/realspace/nonlin_k_analysis.py create mode 100644 scripts/compute_m_bias_image_sims.py create mode 100644 scripts/diagnostics_image_sims.py create mode 100644 src/sp_validation/image_sims.py create mode 100644 src/sp_validation/tests/test_image_sims.py create mode 100644 src/sp_validation/tests/test_mask_overlay.py create mode 100644 workflow/image_sims/Snakefile create mode 100644 workflow/image_sims/config.yaml create mode 100644 workflow/profiles/candide/config.yaml create mode 100644 workflow/rules/image_sims.smk create mode 100644 workflow/scripts/im_build_manifest.py create mode 100644 workflow/scripts/im_compose_mask.py diff --git a/config/calibration/mask_v1.X.4_im_sim.yaml b/config/calibration/mask_v1.X.4_im_sim.yaml new file mode 100644 index 00000000..794566d4 --- /dev/null +++ b/config/calibration/mask_v1.X.4_im_sim.yaml @@ -0,0 +1,70 @@ +# Config file for masking and calibration. +# Standard cuts without coverage mask, type v1.X.4. + +# General parameters (can also given on command line) +params: + input_path: shape_catalog_comprehensive_ngmix.hdf5 + cmatrices: False + sky_regions: False + verbose: True + +# Masks +## Using columns in 'dat' group (ShapePipe flags) +dat: + # SExtractor flags + - col_name: FLAGS + label: SE FLAGS + kind: equal + value: 0 + + # Number of epochs + - col_name: N_EPOCH + label: r"$n_{\rm epoch}$" + kind: greater_equal + value: 2 + + # Magnitude range + - col_name: mag + label: mag range + kind: range + value: [15, 30] + + # ngmix flags + - col_name: NGMIX_MOM_FAIL + label: "ngmix moments failure" + kind: equal + value: 0 + + # invalid PSF ellipticities + - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + label: "bad PSF ellipticity comp 1" + kind: not_equal + value: -10 + - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + label: "bad PSF ellipticity comp 2" + kind: not_equal + value: -10 + +# Metacal parameters +metacal: + # Ellipticity dispersion + sigma_eps_prior: 0.34 + + # Signal-to-noise range + gal_snr_min: 10 + gal_snr_max: 500 + + # Relative-size (hlr / hlr_psf) range + gal_rel_size_min: 0.5 + gal_rel_size_max: 3 + + # Correct relative size for ellipticity? + gal_size_corr_ell: False + + # Weight for global response matrix, None for unweighted mean. + # Unweighted for image sims: no weights anywhere in sim m-bias (#227). + global_R_weight: null + + # Subtract additive bias (mean shear)? Use False for constant-shear + # image sims + additive_correction: False diff --git a/config/calibration/mask_v1.X.9_im_sim.overlay.yaml b/config/calibration/mask_v1.X.9_im_sim.overlay.yaml new file mode 100644 index 00000000..94c5b9ae --- /dev/null +++ b/config/calibration/mask_v1.X.9_im_sim.overlay.yaml @@ -0,0 +1,97 @@ +# Declared delta: image-sim mask/calibration config vs the data config. +# +# The image-sim calibration reuses the *data* mask config +# (mask_v1.X.9.yaml) and changes only what the sims genuinely differ on. +# Rather than maintain a second full copy that can silently drift from the +# base, this overlay states -- as a list of block operations on the base +# file -- exactly which pieces the sims drop or change, and one line of why +# for each. `im_compose_mask.py` applies these ops to mask_v1.X.9.yaml and +# reproduces mask_v1.X.9_im_sim.yaml byte-for-byte; a test locks that, so the +# runtime file and this declaration cannot diverge. +# +# Each op anchors to a block of base text (matched verbatim, and required to +# occur exactly once) and either drops it or replaces it. `why` is prose for +# the human reader; the compose ignores it. Ordering follows the base file. + +base: mask_v1.X.9.yaml + +ops: + # --- params ------------------------------------------------------------ + - why: >- + Sims read the ShapePipe FITS catalogue staged in the run dir, not the + survey-wide comprehensive HDF5 on /n17data. + replace: | + input_path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_comprehensive_struc_2024_v1.X.c.hdf5 + with: | + input_path: shape_catalog_comprehensive_ngmix.fits + + # --- dat cuts ---------------------------------------------------------- + - why: >- + No ShapePipe coverage/mask flags on sims: IMAFLAGS_ISO is a survey + artefact (external masks, bright-star haloes) the sims do not carry. + drop: |2 + + # ShapePipe flags + - col_name: IMAFLAGS_ISO + label: SP mask + kind: equal + value: 0 + + - why: >- + Same cuts, but flag the grammar: the sims run the ShapePipe-v2 PSF + columns (scalar G1/G2), so the comment is made explicit here. + replace: |2 + # invalid PSF ellipticities + with: |2 + # invalid PSF ellipticities (ShapePipe-v2 grammar: scalar G1/G2 components) + + # --- dat_ext (post-processing / coverage masks) ------------------------ + - why: >- + No coverage masks on sims: the whole dat_ext group (Stars, manual mask, + r-band footprint, Maximask) is survey post-processing with no analogue + in the simulated tiles. + drop: |2 + + ## Using columns in 'dat_ext' group (post-processing flags) + dat_ext: + + # Stars + - col_name: 4_Stars + label: "Stars" + kind: equal + value: False + + # Manual mask + - col_name: 8_Manual + label: "manual mask" + kind: equal + value: False + + # r-band footprint + - col_name: 64_r + label: "r-band imaging" + kind: equal + value: False + + # Maximask + - col_name: 1024_Maximask + label: "maximask" + kind: equal + value: False + + # --- metacal ----------------------------------------------------------- + - why: >- + Unweighted for image sims: no weights anywhere in the sim m-bias (#227), + and the w_des-weighted global R had only N_eff ~ 20-100 objects. + replace: |2 + # Weight for global response matrix, None for unweighted mean + global_R_weight: w + with: |2 + # Weight for global response matrix, None for unweighted mean. + # Unweighted for image sims: no weights anywhere in sim m-bias (#227), + # and the w_des-weighted R had N_eff ~ 20-100 objects. + global_R_weight: null + + # Subtract additive bias (mean shear)? Use False for constant-shear + # image sims + additive_correction: False diff --git a/config/calibration/mask_v1.X.9_im_sim.yaml b/config/calibration/mask_v1.X.9_im_sim.yaml new file mode 100644 index 00000000..1c6ff672 --- /dev/null +++ b/config/calibration/mask_v1.X.9_im_sim.yaml @@ -0,0 +1,77 @@ +# Config file for masking and calibration. +# Less conservative cuts, type v1.X.9 (e.g. for matching with spectroscopic sample). + +# General parameters (can also given on command line) +params: + input_path: shape_catalog_comprehensive_ngmix.fits + cmatrices: False + sky_regions: False + verbose: True + +# Masks +## Using columns in 'dat' group (ShapePipe flags) +dat: + # SExtractor flags + - col_name: FLAGS + label: SE FLAGS + kind: smaller_equal + value: 2 + + # Duplicate objects + - col_name: overlap + label: tile overlap + kind: equal + value: True + + # Number of epochs + - col_name: N_EPOCH + label: r"$n_{\rm epoch}$" + kind: greater_equal + value: 1 + + # Magnitude range + - col_name: mag + label: mag range + kind: range + value: [15, 30] + + # ngmix flags + - col_name: NGMIX_MCAL_TYPES_FAIL + label: "ngmix moments failure" + kind: equal + value: 0 + + # invalid PSF ellipticities (ShapePipe-v2 grammar: scalar G1/G2 components) + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR + label: "bad PSF ellipticity comp 1" + kind: not_equal + value: -10 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR + label: "bad PSF ellipticity comp 2" + kind: not_equal + value: -10 + +# Metacal parameters +metacal: + # Ellipticity dispersion + sigma_eps_prior: 0.34 + + # Signal-to-noise range + gal_snr_min: 5 + gal_snr_max: 500 + + # Relative-size (hlr / hlr_psf) range + gal_rel_size_min: 0.25 + gal_rel_size_max: 10 + + # Correct relative size for ellipticity? + gal_size_corr_ell: False + + # Weight for global response matrix, None for unweighted mean. + # Unweighted for image sims: no weights anywhere in sim m-bias (#227), + # and the w_des-weighted R had N_eff ~ 20-100 objects. + global_R_weight: null + + # Subtract additive bias (mean shear)? Use False for constant-shear + # image sims + additive_correction: False diff --git a/papers/realspace/S8_om_sigma8_whisker.py b/papers/realspace/S8_om_sigma8_whisker.py new file mode 100644 index 00000000..9f0aecae --- /dev/null +++ b/papers/realspace/S8_om_sigma8_whisker.py @@ -0,0 +1,549 @@ +# +# This notebook plots the whisker plot of $S_8$, $\Omega_m$ and $\sigma_8$ + + +import os +import sys + +# Trick to plot with tex +os.environ["LD_LIBRARY_PATH"] = "" +os.environ["CONDA_PREFIX"] = "/home/guerrini/.conda/envs/sp_validation_3.11" + +import warnings + +import matplotlib.pyplot as plt +import numpy as np +import seaborn as sns +from getdist import plots + +sys.path.append("/home/guerrini/sp_validation/cosmo_inference/scripts") + +import chain_postprocessing as cp + +plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") + +plt.rc("text", usetex=True) + +sns.set_palette("husl") + +g = plots.get_subplot_plotter(width_inch=30) +g.settings.axes_fontsize = 60 +g.settings.axes_labelsize = 60 +g.settings.alpha_filled_add = 0.7 +g.settings.legend_fontsize = 60 + + +# SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE +root_dir = "/n09data/guerrini/output_chains/" +root_external = f"{root_dir}/ext_data/" +blind = "B" + +roots = [ + f"SP_v1.4.6.3_{blind}_fiducial_config", + f"SP_v1.4.6.3_leak_corr_{blind}", + "Planck18", + "DES Y6", + "KiDS-Legacy_bandpowers", + "KiDS-Legacy_cosebis", + "KiDS-Legacy_xipm", + "HSC_Y3", + "HSC_Y3_cell", + f"SP_v1.4.6.3_{blind}_small_scales_config", + f"SP_v1.4.6.3_{blind}_flat_alpha_beta_config", + f"SP_v1.4.6.3_{blind}_no_xi_sys_config", + f"SP_v1.4.6.3_{blind}_no_leak_corr_config", + f"SP_v1.4.6.3_{blind}_flat_delta_z_config", + f"SP_v1.4.6.3_{blind}_no_delta_z_config", + f"SP_v1.4.6.3_{blind}_flat_ia_config", + f"SP_v1.4.6.3_{blind}_no_ia_config", + f"SP_v1.4.6.3_{blind}_no_m_bias_config", + f"SP_v1.4.6.3_{blind}_unmasked_covmat_config", + f"SP_v1.4.6.3_{blind}_halofit_config", + f"SP_v1.4.6.3_{blind}_no_baryons_config", + f"SP_v1.4.6.3_{blind}_nautilus_config", + f"SP_v1.4.6.3_{blind}_planck_config", + f"SP_v1.4.6.3_{blind}_planck_desi_config", +] + +legend_labels = [ + r"UNIONS-3500 $\xi_{\pm}(\theta)$ (This work)", + r"UNIONS-3500 $C_\ell$ (Guerrini et al. 2026)", + r"$\textit{Planck}$ 2018", + r"DES Y6 $\xi_{\pm}$, NLA", + r"KiDS-Legacy Bandpowers ($C_{\rm E}$)", + r"KiDS-Legacy COSEBIs ($E_n$)", + r"KiDS-Legacy $\xi_{\pm}(\theta)$", + r"HSC-Y3 $\xi_{\pm}(\theta)$", + r"HSC-Y3 $C_\ell$", + r"$\xi_+$ small scales, $\theta$=[5,83] arcmin", + r"Flat $\alpha_{\rm{PSF}}$ and $\beta_{\rm{PSF}}$ priors", + r"No $\xi^{\rm sys}_{\pm}$", + r"No leakage correction", + r"Flat $\Delta z$ priors", + r"No $\Delta z$", + r"Flat $A_{\rm IA}$ prior", + r"No $A_{\rm IA}$", + r"No $m$ bias", + r"Unmasked covmat", + r"$\texttt{Halofit}$", + r"$\texttt{HMCode}$ no baryons", + r"Nautilus sampler", + r"UNIONS-3500 + $\textit{Planck}$", + r"UNIONS-3500 + $\textit{Planck}$ + DESI BAO", +] + +categories = [ + "configuration", + "harmonic", + "external", + "external", + "external", + "external", + "external", + "external", + "external", + "configuration", + "configuration", + "configuration", + "configuration", + "configuration", + "configuration", + "configuration", + "configuration", + "configuration", + "configuration", + "configuration", + "configuration", + "configuration", + "configuration", + "configuration", +] +colours = [ + "darkorange", + "royalblue", + "violet", + "black", + "black", + "black", + "black", + "black", + "black", + "forestgreen", + "forestgreen", + "forestgreen", + "forestgreen", + "forestgreen", + "forestgreen", + "forestgreen", + "forestgreen", + "forestgreen", + "forestgreen", + "forestgreen", + "forestgreen", + "forestgreen", + "forestgreen", + "forestgreen", +] + + +chains = [] +for i, root in enumerate(roots): + category = categories[i] + if root == "DES Y6": + continue + if category != "external": + if category == "configuration": + path_samples = os.path.join(root_dir, f"{root}/samples_{root}.txt") + path_getdist = os.path.join(root_dir, f"{root}/getdist_{root}") + elif category == "harmonic": + path_samples = os.path.join( + root_dir, f"{root}/{root}/samples_{root}_cell.txt" + ) + path_getdist = os.path.join(root_dir, f"{root}/{root}/getdist_{root}") + elif category == "external_compute_sample": + path_samples = os.path.join(root_dir, f"ext_data/{root}/samples_{root}.txt") + path_getdist = os.path.join(root_dir, f"ext_data/{root}/getdist_{root}") + else: + raise ValueError(f"The category, {category}, of {root} is not correct") + if "nautilus" not in root: + cp.load_samples_and_write_paramnames( + path_samples, path_getdist + ".paramnames" + ) + cp.write_samples_getdist_format(path_samples, path_getdist + ".txt") + else: + cp.load_samples_and_write_paramnames( + path_samples, path_getdist + ".paramnames", chain_type="nautilus" + ) + cp.write_samples_getdist_format( + path_samples, path_getdist + ".txt", chain_type="nautilus" + ) + chains.append(cp.load_chain(path_getdist, smoothing_scale=0.5)) + else: + path_getdist = os.path.join(root_dir, f"ext_data/{root}/getdist_{root}") + chains.append(cp.load_chain(path_getdist)) + + +name_list = [ + "OMEGA_M", + "ombh2", + "h0", + "n_s", + "SIGMA_8", + "S_8", + "s_8_input", + "logt_agn", + "a", + "m1", + "bias_1", +] +label_list = [ + r"\Omega_{\rm m}", + r"\omega_b h^2", + r"h_0", + r"n_s", + r"\sigma_8", + r"S_8", + r"S_8", + r"\log T_{\rm AGN}", + r"A_{\rm IA}", + r"m_1", + r"\Delta z_1", +] + +for i, chain in enumerate(chains): + print(legend_labels[i]) + param_names = chain.getParamNames() + for name, label in zip(name_list, label_list): + try: + param_names.parWithName(name).label = label + except Exception: + warnings.warn(f"Parameter {name} not found in chain {roots[i]}.") + + +# Micro management of external chains + +# Account for the missing parameter conventions + +idx = roots.index("KiDS-Legacy_xipm") +cp.derive_parameter_S8(chains[idx]) + +idx = roots.index("KiDS-Legacy_bandpowers") +cp.derive_parameter_S8(chains[idx]) + +idx = roots.index("KiDS-Legacy_cosebis") +cp.derive_parameter_S8(chains[idx]) + +# OMEGA_M not in HSC_Y3_cell +idx = roots.index("HSC_Y3_cell") +cp.adjust_paramname_chain(chains[idx], "omega_m", "OMEGA_M", r"\Omega_{\rm m}") + + +param_values = np.array( + [ + "# Expt", + "Colour", + "S8_Mean", + "S8_low", + "S8_high", + "sigma_8_Mean", + "sigma_8_low", + "sigma_8_high", + "Omega_m_Mean", + "Omega_m_low", + "Omega_m_high", + ] +) +escaped = np.char.replace(legend_labels, "\\", "\\\\") + +for i, root in enumerate(roots): + chain = chains[i] + if root == "DES Y6": + param_values = np.vstack( + ( + param_values, + [ + escaped[i], + colours[i], + 0.798, + 0.015, + 0.014, + 0.763, + 0.057, + 0.050, + 0.332, + 0.040, + 0.035, + ], + ) + ) + else: + best_fit_params = cp.extract_best_fit_params(chain, best_fit_method="2Dkde") + margestats = chain.getMargeStats() + + s8_stats = margestats.parWithName("S_8") + sigma8_stats = margestats.parWithName("SIGMA_8") + omegam_stats = margestats.parWithName("OMEGA_M") + + param_values = np.vstack( + ( + param_values, + [ + escaped[i], + colours[i], + best_fit_params["S_8"], + best_fit_params["S_8"] - s8_stats.limits[0].lower, + s8_stats.limits[0].upper - best_fit_params["S_8"], + best_fit_params["SIGMA_8"], + best_fit_params["SIGMA_8"] - sigma8_stats.limits[0].lower, + sigma8_stats.limits[0].upper - best_fit_params["SIGMA_8"], + best_fit_params["OMEGA_M"], + best_fit_params["OMEGA_M"] - omegam_stats.limits[0].lower, + omegam_stats.limits[0].upper - best_fit_params["OMEGA_M"], + ], + ) + ) +print(param_values) +np.savetxt( + f"{root_dir}/param_values.txt", + param_values, + fmt=["%s" for i in range(11)], + delimiter=";", +) + + +# Load the value of the parameters +cosmo = np.loadtxt( + f"{root_dir}/param_values.txt", + dtype={ + "names": ( + "Expt", + "colour", + "s8_mean", + "s8_low", + "s8_high", + "sigma8_mean", + "sigma8_low", + "sigma8_high", + "omegam_mean", + "omegam_low", + "omegam_high", + ), + "formats": ( + "U250", + "U20", + "U20", + "U20", + "U20", + "U20", + "U20", + "U20", + "U20", + "U20", + "U20", + ), + }, + skiprows=1, + delimiter=";", +) +expt = np.char.replace(cosmo["Expt"], "\\\\", "\\") +colours = cosmo["colour"] +s8_mean = cosmo["s8_mean"].astype(np.float64) +s8_low = cosmo["s8_low"].astype(np.float64) +s8_high = cosmo["s8_high"].astype(np.float64) +sigma8_mean = cosmo["sigma8_mean"].astype(np.float64) +sigma8_low = cosmo["sigma8_low"].astype(np.float64) +sigma8_high = cosmo["sigma8_high"].astype(np.float64) +omegam_mean = cosmo["omegam_mean"].astype(np.float64) +omegam_low = cosmo["omegam_low"].astype(np.float64) +omegam_high = cosmo["omegam_high"].astype(np.float64) + + +from matplotlib.gridspec import GridSpec + +fig = plt.figure(figsize=(13, 8)) +gs = GridSpec(1, 3, width_ratios=[1, 0.5, 0.5]) +ax1 = fig.add_subplot(gs[0]) +ax2 = fig.add_subplot(gs[1], sharey=ax1) +ax3 = fig.add_subplot(gs[2], sharey=ax1) + +axs = [ax1, ax2, ax3] + +params = [ + (s8_mean, s8_low, s8_high, r"$S_8$"), + (sigma8_mean, sigma8_low, sigma8_high, r"$\sigma_8$"), + (omegam_mean, omegam_low, omegam_high, r"$\Omega_{\rm m}$"), +] +reference = r"UNIONS-3500 $\xi_{\pm}(\theta)$ (This work)" + +separation_after = [ + r"UNIONS-3500 $C_\ell$ (Guerrini et al. 2026)", + r"HSC-Y3 $C_\ell$", + r"$\xi_+$ small scales, $\theta$=[5,83] arcmin", + r"Unmasked covmat", + r"$\texttt{HMCode}$ no baryons", + r"Nautilus sampler", +] +list_section_index = [r"(ii)", r"(iii)", r"(iv)", r"(v)", r"(vi)", r"(vii)"] + +preliminary_watermark = False +blind_axes = False +row_spacing = 0.2 + +index_ref = np.where(expt == reference)[0][0] + +y = np.arange(len(expt)) +for ax, param in zip(axs, params): + means, lows, highs, label = param + for i, mean, low, high, color in zip(y, means, lows, highs, colours): + ax.errorbar( + mean, + 0.05 + i * row_spacing, + xerr=np.array([low, high])[:, None], + fmt="o", + color=color, + ecolor=color, + elinewidth=2, + capsize=3, + ) + ax.set_xlabel(label, fontsize=14) + + ax.grid(False) + ax.tick_params(axis="y", left=False, labelleft=False) + if label == r"$S_8$": + ax.axvspan( + s8_mean[index_ref] - s8_low[index_ref], + s8_mean[index_ref] + s8_high[index_ref], + color=colours[index_ref], + alpha=0.2, + ) + ax.set_xlim(0.6, 1.35) + if blind_axes: + ref_tick = np.mean(s8_mean[:4]) + ax.set_xticks([ref_tick + i * 0.1 for i in range(-5, 5)], labels=[]) + elif label == r"$\sigma_8$": + ax.axvspan( + sigma8_mean[index_ref] - sigma8_low[index_ref], + sigma8_mean[index_ref] + sigma8_high[index_ref], + color=colours[index_ref], + alpha=0.2, + ) + ax.set_xlim(0.5, 1.35) + if blind_axes: + ref_tick = np.mean(sigma8_mean[:4]) + ax.set_xticks([ref_tick + i * 0.2 for i in range(-2, 2)], labels=[]) + elif label == r"$\Omega_{\rm m}$": + ax.axvspan( + omegam_mean[index_ref] - omegam_low[index_ref], + omegam_mean[index_ref] + omegam_high[index_ref], + color=colours[index_ref], + alpha=0.2, + ) + ax.set_xlim(0.1, 0.5) + if blind_axes: + ref_tick = np.mean(omegam_mean[:4]) + ax.set_xticks([ref_tick + i * 0.1 for i in range(-2, 3)], labels=[]) + + +ax1.set_yticks(0.01 + y * row_spacing) +ax1.set_yticklabels([]) +for label, color in zip(expt, colours): + if "This work" in label: + label_bold = ( + r"$\bf{UNIONS}$-$\bf{3500}$ $\xi_{\pm}(\theta)$ $\bf{(This\ work)}$" + ) + ax1.text( + -0.6, + 0.05 + row_spacing * np.where(expt == label)[0][0], + label_bold, + fontsize=12, + ha="left", + va="center", + color=color, + ) + else: + ax1.text( + -0.6, + 0.05 + row_spacing * np.where(expt == label)[0][0], + label, + fontsize=12, + ha="left", + va="center", + color=color, + ) + if label != reference: + index = np.where(expt == label)[0][0] + s8_tension = cp.get_sigma_tension( + s8_mean[index], + s8_low[index], + s8_high[index], + s8_mean[index_ref], + s8_low[index_ref], + s8_high[index_ref], + ) + sign_str = "+" if s8_tension > 0 else "-" + ax1.text( + 1.32, + 0.05 + row_spacing * index, + rf"${sign_str}{np.abs(s8_tension):.2f}" + r"\, \sigma$", + fontsize=10, + ha="right", + va="center", + color=color, + ) +# Add separation lines +for i, sep in enumerate(separation_after): + print(sep) + index_sep = np.where(expt == sep)[0][0] + ax2.axhline( + row_spacing * (index_sep + 1) - 0.07, + color="black", + linestyle="dotted", + linewidth=1, + ) + ax3.axhline( + row_spacing * (index_sep + 1) - 0.07, + color="black", + linestyle="dotted", + linewidth=1, + ) + ax1.axhline( + row_spacing * (index_sep + 1) - 0.07, + xmin=-1.8, + color="black", + linestyle="dotted", + linewidth=1, + clip_on=False, + ) + ax1.text( + -0.61, + row_spacing * (index_sep + 1) + 0.05, + list_section_index[i], + fontsize=12, + fontweight="bold", + va="center", + ha="right", + ) + + +# --- Add section label (i)) --- +ax1.text(-0.61, 0.05, r"(i)", fontsize=12, fontweight="bold", va="center", ha="right") + +if preliminary_watermark: + plt.figtext( + 0.5, + 0.5, + "PRELIMINARY", + fontsize=50, + color="gray", + ha="center", + va="center", + alpha=0.3, + rotation=330, + ) + +plt.gca().invert_yaxis() + +plt.tight_layout() + +# #Save pdf +plt.savefig("./../../results/S8_whisker_plot.pdf", bbox_inches="tight") diff --git a/papers/realspace/best_fit_xipm.py b/papers/realspace/best_fit_xipm.py new file mode 100644 index 00000000..ebd66f27 --- /dev/null +++ b/papers/realspace/best_fit_xipm.py @@ -0,0 +1,497 @@ +import os +import sys + +sys.path.append("/home/guerrini/sp_validation/cosmo_inference/scripts") + +import chain_postprocessing as cp +import matplotlib.pyplot as plt +import matplotlib.scale as mscale +import numpy as np +import seaborn as sns +from astropy.io import fits +from getdist import plots + +plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") + +from sp_validation.rho_tau import SquareRootScale + +mscale.register_scale(SquareRootScale) + +plt.rcParams["text.usetex"] = True + +sns.set_palette("husl") + +g = plots.get_subplot_plotter(width_inch=30) +g.settings.axes_fontsize = 40 +g.settings.axes_labelsize = 40 +g.settings.alpha_filled_add = 0.7 +g.settings.legend_fontsize = 50 + +# Directory where the chains are located +root_dir = "/n09data/guerrini/output_chains" + +# THE BLIND TO USE FOR THE PLOTS +blind = "B" +catalog_version = "SP_v1.4.6.3" +fiducial_root_cell = f"SP_v1.4.6.3_leak_corr_{blind}" +label_fiducial_cell = r"UNIONS $C_{\ell}$" +fiducial_root_xi_data = f"SP_v1.4.6.3_leak_corr_{blind}_masked" +fiducial_root_xi_chains = f"SP_v1.4.6.3_{blind}_fiducial_config" +label_fiducial_xi = r"UNIONS $\xi_{\pm}$" + +# Path to the ini files used +path_ini_files = "/home/guerrini/sp_validation/cosmo_inference/cosmosis_config" +path_datavectors = "/home/guerrini/sp_validation/cosmo_inference/data/" +path_output_chains = "/n09data/guerrini/output_chains/" + + +data_cell = fits.open( + os.path.join( + path_datavectors, f"{fiducial_root_cell}/cosmosis_{fiducial_root_cell}.fits" + ) +) + +data_xi = fits.open( + os.path.join( + path_datavectors, + f"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}/cosmosis_{fiducial_root_xi_data}.fits", + ) +) + +path_samples_fiducial_cell = os.path.join( + path_output_chains, + fiducial_root_cell, + fiducial_root_cell, + f"samples_{fiducial_root_cell}_cell.txt", +) +path_gd_fiducial_cell = os.path.join( + path_output_chains, + fiducial_root_cell, + fiducial_root_cell, + f"getdist_{fiducial_root_cell}_cell", +) +cp.load_samples_and_write_paramnames( + path_samples_fiducial_cell, path_gd_fiducial_cell + ".paramnames" +) +cp.write_samples_getdist_format( + path_samples_fiducial_cell, path_gd_fiducial_cell + ".txt", chain_type="polychord" +) + +chain_fiducial_cell = cp.load_chain(path_gd_fiducial_cell, smoothing_scale=0.5) + +best_fit_params_fiducial_cell = cp.extract_best_fit_params( + chain_fiducial_cell, best_fit_method="2Dkde" +) + +cp.compute_best_fit( + path_ini_files, + best_fit_params_fiducial_cell, + fiducial_root_cell, + is_harmonic=True, + blind=blind, +) +path_samples_fiducial_xi = os.path.join( + path_output_chains, + fiducial_root_xi_chains, + f"samples_{fiducial_root_xi_chains}.txt", +) + +path_gd_fiducial_xi = os.path.join( + path_output_chains, fiducial_root_xi_chains, f"getdist_{fiducial_root_xi_chains}" +) +cp.load_samples_and_write_paramnames( + path_samples_fiducial_xi, path_gd_fiducial_xi + ".paramnames" +) +cp.write_samples_getdist_format( + path_samples_fiducial_xi, path_gd_fiducial_xi + ".txt", chain_type="polychord" +) + +chain_fiducial_xi = cp.load_chain(path_gd_fiducial_xi, smoothing_scale=0.5) + +best_fit_params_fiducial_xi = cp.extract_best_fit_params( + chain_fiducial_xi, best_fit_method="2Dkde" +) + +ini_file_root = os.path.join( + path_ini_files, + f"config_space_v1.4.6.3_fiducial/pipeline/blind_{blind}/fiducial.ini", +) +cp.compute_best_fit( + path_ini_files, + best_fit_params_fiducial_xi, + fiducial_root_xi_chains, + is_harmonic=False, + blind=blind, + ini_file_root=ini_file_root, +) + +root_to_plot = [ + fiducial_root_xi_chains, + fiducial_root_cell, +] + +labels = [ + r"UNIONS $\xi_\pm(\theta)$", + r"UNIONS $C_\ell$", +] + +line_args = [ + {"color": "royalblue", "linestyle": "-"}, + {"color": "orange", "linestyle": "-"}, +] + +properties = {} + +properties = cp.update_properties_w_roots( + properties, fiducial_root_cell, path_ini_files, with_configuration=False +) +properties = cp.update_properties_w_roots( + properties, + fiducial_root_xi_chains, + path_ini_files, + with_configuration=True, + path_to_this_ini=ini_file_root, +) + + +root_to_plot = [fiducial_root_cell, fiducial_root_xi_chains] +labels = [r"Best fit $C_\ell$", r"Best fit $\xi_\pm(\theta)$"] +path_best_fit_xi_theta = os.path.join( + path_output_chains, fiducial_root_xi_chains, "best_fit/shear_xi_plus/theta.txt" +) + +theta_rad = np.loadtxt(path_best_fit_xi_theta) +theta_min = 1 +theta_max = 250 + +cp.compute_best_fit_xi_from_cell( + path_output_chains, fiducial_root_cell, best_fit_params_fiducial_cell, theta_rad +) + +data = fits.open( + os.path.join( + path_datavectors, + f"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}/cosmosis_{fiducial_root_xi_data}.fits", + ) +) +bbox_to_anchor_xip = (0.685, 0.09) +bbox_to_anchor_xim = (0.3, 0.65) +xi_p_data = data["XI_PLUS"].data +xi_m_data = data["XI_MINUS"].data +cov_mat = data["COVMAT"].data + +# Plot hyperparameter +loc_legend = "lower center" + +fig, [ax, ax2] = plt.subplots(1, 2, figsize=(20, 8)) + +theta, xi_p, xi_m = xi_p_data["ANG"], xi_p_data["VALUE"], xi_m_data["VALUE"] +ax.errorbar( + theta, + theta * xi_p, + yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])), + fmt="o", + label=r"UNIONS $\xi_+$ data", + color="black", + capsize=2, +) +ax2.errorbar( + theta, + theta * xi_m, + yerr=theta + * np.sqrt( + np.diag(cov_mat[len(theta) : 2 * len(theta), len(theta) : 2 * len(theta)]) + ), + fmt="o", + label=r"UNIONS $\xi_-$ data", + color="black", + capsize=2, +) + +for idx, (label, root) in enumerate(zip(labels, root_to_plot)): + # Read the results + theta = ( + ( + np.loadtxt( + path_output_chains + "{}/best_fit/shear_xi_plus/theta.txt".format(root) + ) + ) + * 180 + / np.pi + * 60 + ) + xi_plus = np.loadtxt( + path_output_chains + "{}/best_fit/shear_xi_plus/bin_1_1.txt".format(root) + ) + xi_minus = np.loadtxt( + path_output_chains + "{}/best_fit/shear_xi_minus/bin_1_1.txt".format(root) + ) + if r"$C_\ell$" not in label: + xi_sys_plus = np.loadtxt( + path_output_chains + "{}/best_fit/xi_sys/shear_xi_plus.txt".format(root) + ) + xi_sys_minus = np.loadtxt( + path_output_chains + "{}/best_fit/xi_sys/shear_xi_minus.txt".format(root) + ) + theta_xi_sys = ( + np.loadtxt(path_output_chains + "{}/best_fit/xi_sys/theta.txt".format(root)) + * 180 + / np.pi + * 60 + ) + + xi_sys_plus = np.interp(theta, theta_xi_sys, xi_sys_plus) + xi_sys_minus = np.interp(theta, theta_xi_sys, xi_sys_minus) + xi_plus += xi_sys_plus + xi_minus += xi_sys_minus + + mask = (theta > theta_min) & (theta < theta_max) + theta = theta[mask] + ax.plot( + theta, + theta * xi_plus[mask], + label=r"Best fit $\xi_+(\theta)$", + **line_args[idx], + lw=2.5, + ) + ax.plot( + theta, + theta * xi_sys_plus[mask], + label=r"Best fit $\xi^{\rm sys}_{+}(\theta)$", + c="r", + ) + ax2.plot( + theta, + theta * xi_minus[mask], + label=r"Best fit $\xi_-(\theta)$", + **line_args[idx], + lw=2.5, + ) + ax2.plot( + theta, + theta * xi_sys_minus[mask], + label=r"Best fit $\xi^{\rm sys}_{-}(\theta)$", + c="r", + ) + + else: + mask = (theta > theta_min) & (theta < theta_max) + theta = theta[mask] + ax.plot(theta, theta * xi_plus[mask], label=label, **line_args[idx], lw=2.5) + ax2.plot(theta, theta * xi_minus[mask], label=label, **line_args[idx], lw=2.5) + +# XI PLUS PLOT SETTINGS + +# Plot the scale cuts for different k_max +ax.axvline(x=5, color="gray", linestyle="--", alpha=0.7) +ax.axhline(y=0, color="black", linestyle="--", alpha=0.7) + +ymin = ax.get_ylim()[0] +ymax = ax.get_ylim()[1] +# Shadowing cut scaled +ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color="gray", alpha=0.2) +ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color="gray", alpha=0.2) + +ax.set_ylim(ymin, ymax) + +# Add labels directly under the tick +ax.text( + 4.5, + 0.47e-4, + r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", + ha="center", + va="top", + fontsize=20, + rotation=90, +) + +ax.set_ylabel(r"$\theta \xi_\pm$", fontsize=26) +ax.set_xlabel(r"$\theta$ (arcmin)", fontsize=26) +ax.set_xlim([theta.min() - 0.1, theta.max() + 20]) +ax.set_title(r"$\xi_+(\theta)$", fontsize=26) +ax.set_xscale("log") +ax.set_xticks(np.array([1, 10, 100])) +ax.tick_params(axis="x", which="minor", length=2, width=0.8) +ax.tick_params(axis="both", which="major", labelsize=24) +ax.tick_params(axis="both", which="minor", labelsize=20) +ax.yaxis.get_offset_text().set_fontsize(24) +ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) +ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=20) + +# XI_MINUS PLOT SETTINGS + +# Plot the scale cuts for different k_max +ax2.axvline(x=50, color="gray", linestyle="--", alpha=0.7) +ax2.axhline(y=0, color="black", linestyle="--", alpha=0.7) + +ymin = ax2.get_ylim()[0] +ymax = ax2.get_ylim()[1] +# Shadowing cut scaled +ax2.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color="gray", alpha=0.2) +ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color="gray", alpha=0.2) + +ax2.set_ylim(ymin, ymax) + +# Add labels directly under the tick +ax2.text( + 45, + 1.15e-4, + r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", + ha="center", + va="top", + fontsize=20, + rotation=90, +) + +# ax2.set_ylabel(r'$\theta \xi_-$', fontsize=16) +ax2.set_xlabel(r"$\theta$ (arcmin)", fontsize=26) +ax2.set_xlim([theta.min() - 0.1, theta.max() + 20]) +ax2.set_xscale("log") +ax2.set_title(r"$\xi_-(\theta)$", fontsize=26) +ax2.set_xticks(np.array([1, 10, 100])) +ax2.tick_params(axis="x", which="minor", length=2, width=0.8) +ax2.tick_params(axis="both", which="major", labelsize=24) +ax2.tick_params(axis="both", which="minor", labelsize=20) +ax2.yaxis.get_offset_text().set_fontsize(24) +ax2.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) +ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=20) + +plt.savefig("./../../results/best_fit_xipm_SP_v1.4.6.3_B.pdf", bbox_inches="tight") + + +root_to_plot = [fiducial_root_xi_chains] +labels = [r"Best fit $\tau_{0,2}(\theta)$"] + +bbox_to_anchor_xip = (0.285, 0.7) +bbox_to_anchor_xim = (0.3, 0.65) +tau0_data = data["TAU_0_PLUS"].data +tau2_data = data["TAU_2_PLUS"].data +cov_mat = data["COVMAT"].data + +# Plot hyperparameter + +fig, [ax, ax2] = plt.subplots(1, 2, figsize=(20, 8)) + +theta, tau0, tau2 = tau0_data["ANG"], tau0_data["VALUE"], tau2_data["VALUE"] +ax.errorbar( + theta, + theta * tau0, + yerr=theta + * np.sqrt( + np.diag( + cov_mat[2 * len(theta) : 3 * len(theta), 2 * len(theta) : 3 * len(theta)] + ) + ), + fmt="o", + label=r"UNIONS $\tau_{0,+}$", + color="black", + capsize=2, +) +ax2.errorbar( + theta, + theta * tau2, + yerr=theta + * np.sqrt( + np.diag( + cov_mat[3 * len(theta) : 4 * len(theta), 3 * len(theta) : 4 * len(theta)] + ) + ), + fmt="o", + label=r"UNIONS $\tau_{2,+}$", + color="black", + capsize=2, +) + +for idx, (label, root) in enumerate(zip(labels, root_to_plot)): + # Read the results + theta = ( + ( + np.loadtxt( + path_output_chains + "{}/best_fit/tau_0_plus/theta.txt".format(root) + ) + ) + * 180 + / np.pi + * 60 + ) + tau0_plus = np.loadtxt( + path_output_chains + "{}/best_fit/tau_0_plus/bin_1_1.txt".format(root) + ) + tau2_plus = np.loadtxt( + path_output_chains + "{}/best_fit/tau_2_plus/bin_1_1.txt".format(root) + ) + + mask = (theta > theta_min) & (theta < theta_max) + theta = theta[mask] + ax.plot( + theta, + theta * tau0_plus[mask], + label=r"Best fit $\tau_{0,+}(\theta)$", + c="orange", + lw=2.5, + ) + ax2.plot( + theta, + theta * tau2_plus[mask], + label=r"Best fit $\tau_{2,+}(\theta)$", + c="orange", + lw=2.5, + ) + +# XI PLUS PLOT SETTINGS + +# Plot the scale cuts for different k_max +ax.axhline(y=0, color="black", linestyle="--", alpha=0.7) + +ymin = ax.get_ylim()[0] +ymax = ax.get_ylim()[1] + +ax.set_ylim(ymin, ymax) + +ax.set_ylabel(r"$\theta\tau_{0,2}$", fontsize=26) +ax.set_xlabel(r"$\theta$ (arcmin)", fontsize=26) +ax.set_xlim([theta.min() - 0.1, theta.max() + 20]) +ax.set_title(r"$\tau_{0,+}(\theta)$", fontsize=26) +ax.set_xscale("log") +ax.set_xticks(np.array([1, 10, 100])) +ax.tick_params(axis="x", which="minor", length=2, width=0.8) +ax.tick_params(axis="both", which="major", labelsize=24) +ax.tick_params(axis="both", which="minor", labelsize=20) +ax.yaxis.get_offset_text().set_fontsize(24) +ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) +ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=20) + +# XI_MINUS PLOT SETTINGS + +# Plot the scale cuts for different k_max +ax2.axhline(y=0, color="black", linestyle="--", alpha=0.7) + +ymin = ax2.get_ylim()[0] +ymax = ax2.get_ylim()[1] +# Shadowing cut scaled +ax2.fill_betweenx( + y=[ymin, ymax], + x1=0, + x2=12, + color="gray", + alpha=0.2, + label=r"$B$-mode informed scale cut", +) +ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=250, color="gray", alpha=0.2) + +ax2.set_ylim(ymin, ymax) + +# ax2.set_ylabel(r'$\theta \xi_-$', fontsize=16) +ax2.set_xlabel(r"$\theta$ (arcmin)", fontsize=26) +ax2.set_xlim([theta.min() - 0.1, theta.max() + 20]) +ax2.set_xscale("log") +ax2.set_title(r"$\tau_{2,+}(\theta)$", fontsize=26) +ax2.set_xticks(np.array([1, 10, 100])) +ax2.tick_params(axis="x", which="minor", length=2, width=0.8) +ax2.tick_params(axis="both", which="major", labelsize=24) +ax2.tick_params(axis="both", which="minor", labelsize=20) +ax2.yaxis.get_offset_text().set_fontsize(24) +ax2.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) +ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=20) + +plt.savefig("./../../results/best_fit_tau_02_SP_v1.4.6.3_B.pdf", bbox_inches="tight") diff --git a/papers/realspace/contours.py b/papers/realspace/contours.py new file mode 100644 index 00000000..78b988f4 --- /dev/null +++ b/papers/realspace/contours.py @@ -0,0 +1,745 @@ +# # 2D contour plots +# +# This notebook produces the plots for all the 2D contours in the results section. + + +import os.path + +import matplotlib.pyplot as plt +import numpy as np +import seaborn as sns +from astropy.io import fits +from getdist import plots + +plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") + +plt.rcParams["text.usetex"] = True + +sns.set_palette("husl") +g = plots.get_subplot_plotter(width_inch=30) +g.settings.axes_fontsize = 70 +g.settings.axes_labelsize = 80 +g.settings.alpha_filled_add = 0.7 +g.settings.legend_fontsize = 70 + + +# SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE + +root_dir = "/n09data/guerrini/output_chains/" +path_datavectors = "/home/guerrini/sp_validation/cosmo_inference/data/" +path_output_chains = "/n09data/guerrini/output_chains/" + +data = fits.open( + os.path.join( + path_datavectors, + "SP_v1.4.6.3_config/SP_v1.4.6.3_B/cosmosis_SP_v1.4.6.3_leak_corr_B_masked.fits", + ) +) + +roots_fid = { + "SP_v1.4.6.3_leak_corr_B": r"UNIONS-3500 $C_\ell$", + "SP_v1.4.6.3_B_fiducial_config": r"UNIONS-3500 $\xi_\pm$ (This work) ", + "KiDS-Legacy_xipm": r"KiDS-Legacy $\xi_\pm$", + "HSC_Y3": r"HSC-Y3 $\xi_\pm$", + "Planck18": r"$\textit{Planck}$ 2018", +} + +roots_full = { + "SP_v1.4.6.3_B_fiducial_config": r"UNIONS-3500 $\xi_\pm$ (This work) ", +} + +roots_ia = { + "SP_v1.4.6.3_B_fiducial_config": r"Gaussian $A_{\rm{IA}}$ prior", + "SP_v1.4.6.3_B_flat_ia_config": r"Flat $A_{\rm{IA}}$ prior", + "SP_v1.4.6.3_B_no_ia_config": r"No IA", +} + +roots_ext = { + "SP_v1.4.6.3_B_fiducial_config": r"UNIONS-3500 $\xi_\pm$", + "SP_v1.4.6.3_B_planck_config": r"UNIONS-3500 $\xi_\pm$ + CMB", + "SP_v1.4.6.3_B_planck_desi_config": r"UNIONS-3500 $\xi_\pm$ + CMB + BAO", + "Planck18": r"$\textit{Planck}$ 2018", +} + +roots_dz = { + "SP_v1.4.6.3_B_fiducial_config": r"Gaussian $\Delta z$ prior", + "SP_v1.4.6.3_B_flat_delta_z_config": r"Flat $\Delta z$ prior", + "SP_v1.4.6.3_B_no_delta_z_config": r"No $\Delta z$ modelling", +} + +roots_psf = { + "SP_v1.4.6.3_B_flat_alpha_beta_config": r"Flat $\alpha$ and $\beta$ priors", + "SP_v1.4.6.3_B_fiducial_config": r"Gaussian $\alpha$ and $\beta$ priors", + "SP_v1.4.6.3_B_no_xi_sys_config": r"No $\xi^{\rm sys}$ included", + "SP_v1.4.6.3_B_no_leak_corr_config": r"No object-wise leakage correction", +} + +roots_scale = { + "SP_v1.4.6.3_B_fiducial_config": r"$\xi_+$: $\theta=[12,83]$", + "SP_v1.4.6.3_B_small_scales_config": r"$\xi_+$: $\theta=[5,83]$", +} + +roots_nonlin = { + "SP_v1.4.6.3_B_fiducial_config": r"Fiducial (\texttt{HMCode2020}, $\log(T_{\rm AGN})$)", + "SP_v1.4.6.3_B_no_baryons_config": r"\texttt{HMCode2020} no baryons", + "SP_v1.4.6.3_B_halofit_config": r"\texttt{Halofit}", +} +roots = roots_ext + + +# ## Retrieve the chains + + +# READ CHAIN + +chains = [] + +for i, root in enumerate(list(roots.keys())): + burnin = 0 + if "SP" not in root: + chain = g.samples_for_root( + root_dir + "ext_data/{}/getdist_{}".format(root, root), + cache=False, + settings={ + "ignore_rows": burnin, + # 'smooth_scale_2D':0.2, + # 'smooth_scale_1D':0.2 + }, + ) + p = chain.getParams() + if hasattr(p, "S_8") == False: + omega_m = chain.getParams().OMEGA_M + sigma_8 = chain.getParams().SIGMA_8 + + s_8 = sigma_8 * (omega_m / 0.3) ** 0.5 + + chain.addDerived(s_8, name="S_8", label=r"S_8") + + p = chain.paramNames.parWithName("S_8") + + elif "config" in root: + if os.path.isfile(root_dir + "{}/getdist_{}.txt".format(root, root)) == False: + samples = np.loadtxt(root_dir + "{}/samples_{}.txt".format(root, root)) + + if "nautilus" in root: + weights = np.exp(samples[:, -3]) + neglogL = samples[:, -2] - samples[:, -1] + + samples = np.column_stack((weights, neglogL, samples[:, 0:-3])) + elif "mh" in root: + samples = np.column_stack( + ( + np.ones_like(samples[:, -1]), + np.log(samples[:, -1]) - np.log(samples[:, -2]), + samples[:, 0:-2], + ) + ) + burnin = 0.3 + else: + samples = np.column_stack( + (samples[:, -1], samples[:, -3], samples[:, 0:-4]) + ) + + np.savetxt(root_dir + "{}/getdist_{}.txt".format(root, root), samples) + + chain = g.samples_for_root( + root_dir + "{}/getdist_{}".format(root, root), + cache=False, + settings={ + "ignore_rows": burnin, + # 'smooth_scale_2D':0.2, + # 'smooth_scale_1D':0.2 + }, + ) + else: + if ( + os.path.isfile( + root_dir + "{}/{}/getdist_{}_cell.txt".format(root, root, root) + ) + == False + ): + samples = np.loadtxt( + root_dir + "{}/{}/samples_{}_cell.txt".format(root, root, root) + ) + + if "nautilus" in root: + weights = np.exp(samples[:, -3]) + neglogL = samples[:, -2] - samples[:, -1] + + samples = np.column_stack((weights, neglogL, samples[:, 0:-3])) + elif "mh" in root: + samples = np.column_stack( + ( + np.ones_like(samples[:, -1]), + np.log(samples[:, -1]) - np.log(samples[:, -2]), + samples[:, 0:-2], + ) + ) + burnin = 0.3 + else: + samples = np.column_stack( + (samples[:, -1], samples[:, -3], samples[:, 0:-4]) + ) + + np.savetxt( + root_dir + "{}/{}/getdist_{}_cell.txt".format(root, root, root), samples + ) + + chain = g.samples_for_root( + root_dir + "{}/{}/getdist_{}_cell".format(root, root, root), + cache=False, + settings={ + "ignore_rows": burnin, + # 'smooth_scale_2D':0.2, + # 'smooth_scale_1D':0.2 + }, + ) + p = chain.getParams() + + chains.append(chain) + + +name_list = [ + "OMEGA_M", + "ombh2", + "h0", + "n_s", + "SIGMA_8", + "S_8", + "logt_agn", + "a", + "m1", + "bias_1", + "alpha", + "beta", + "omch2", +] +label_list = [ + r"\Omega_{\rm m}", + r"\omega_{\rm b}", + r"h", + r"n_{\rm s}", + r"\sigma_8", + r"S_8", + r"\log T_{\rm AGN}", + r"A_{\rm IA}", + r"m_1", + r"\Delta z", + r"\alpha_{\rm PSF}", + r"\beta_{\rm PSF}", + r"\omega_{\rm c}", +] + +for chain in chains: + param_names = chain.getParamNames() + p = chain.getParams() + for name, label in zip(name_list, label_list): + if hasattr(p, name): + param_names.parWithName(name).label = label + +legend_labels = list(roots.values()) + + +# ## Plot the chains + + +# ### FIDUCIAL PLOT + + +colours = [ + "royalblue", + "orange", + "crimson", + "forestgreen", + "indigo", +] + +linestyle = ["solid", "solid", "solid", "solid", "solid"] + +line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] + +# FIDUCIAL PLOT +g.triangle_plot( + chains, + ["SIGMA_8", "S_8", "OMEGA_M"], # + legend_labels=legend_labels, + line_args=line_args, + contour_colors=colours, + label_order=[1, 0, 2, 3, 4], + filled=[True, True, False, False, True], +) + +g.export("./../../results/SP_v1.4.6.3_B_fiducial_config_contour_plot.pdf") + + +# ### FULL PLOT + + +g.settings.axes_fontsize = 40 +g.settings.axes_labelsize = 50 + +colours = [ + "orange", +] + +linestyle = [ + "solid", +] + +line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] + +# FIDUCIAL PLOT +g.triangle_plot( + chains, + [ + "OMEGA_M", + "ombh2", + "h0", + "n_s", + "SIGMA_8", + "S_8", + "logt_agn", + "a", + "m1", + "bias_1", + ], + legend_labels=legend_labels, + line_args=line_args, + contour_colors=colours, + filled=True, +) + +g.export("./../../results/SP_v1.4.6.3_B_fiducial_config_contour_plot_full.pdf") + + +# ### IA PLOT + + +colours = [ + "orange", + "royalblue", + "forestgreen", +] + +linestyle = [ + "solid", + "solid", + "solid", +] + +line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] + +g.triangle_plot( + chains, + ["S_8", "OMEGA_M", "a"], # + legend_labels=legend_labels, + line_args=line_args, + contour_args={"alpha": 0.6}, + contour_colors=colours, + filled=[True, False, True], +) + +g.export("./../../results/SP_v1.4.6.3_B_fiducial_config_contour_plot_ia.pdf") + + +# ### PSF PLOT + + +colours = [ + "royalblue", + "orange", + "hotpink", + "slategray", +] + +linestyle = [ + "solid", + "solid", + "solid", + "solid", +] + +line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] + +g.triangle_plot( + chains, + ["S_8", "OMEGA_M", "alpha", "beta"], # + legend_labels=legend_labels, + line_args=line_args, + contour_args=[{"alpha": 1}, {"alpha": 0.6}, {"alpha": 0.8}, {"alpha": 0.8}], + contour_colors=colours, + legend_loc="upper right", + label_order=[1, 0, 2, 3], + filled=[False, True, True, True], +) + +g.subplots[3, 2].scatter( + 0.005, 0.81, color="k", marker="X", s=400, label="Fiducial config best-fit" +) +g.subplots[3, 2].scatter( + 0.022, 0.798, color="k", marker="P", s=400, label="Fiducial config best-fit" +) + +g.export("./../../results/SP_v1.4.6.3_B_fiducial_config_contour_plot_psf.pdf") + + +# ### DELTA Z PLOT + + +colours = [ + "orange", + "royalblue", + "indigo", +] + +linestyle = [ + "solid", + "solid", + "solid", +] + +line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] +g.triangle_plot( + chains, + ["S_8", "OMEGA_M", "bias_1"], # + legend_labels=legend_labels, + line_args=line_args, + contour_args=[{"alpha": 1.0}, {"alpha": 0.9}, {"alpha": 0.5}], + contour_colors=colours, + filled=[True, False, True], +) + +g.export("./../../results/SP_v1.4.6.3_B_fiducial_config_contour_plot_dz.pdf") + + +# ### EXTERNAL DATA + + +colours = [ + "orange", + "royalblue", + "crimson", + "forestgreen", +] + +linestyle = [ + "solid", + "solid", + "solid", + "solid", + "solid", +] + +line_args = [dict(color=col, ls=ls) for col, ls in zip(colours, linestyle)] + +g = plots.get_subplot_plotter(width_inch=10) +g.settings.axes_fontsize = 25 +g.settings.axes_labelsize = 25 +g.settings.legend_fontsize = 22 + +g.plot_2d( + chains, + ["S_8", "OMEGA_M", "SIGMA_8"], # + line_args=line_args, + contour_colors=colours, + legend_labels=legend_labels, + alphas=[0.7, 1.0, 1.0, 1.0], + filled=[True, True, True, False], +) + +g.add_y_bands(0.2975, 0.0086, alpha2=0, color="k", label="BAO") +g.add_legend(legend_labels, legend_loc="upper right") + +g.export("./../../results/SP_v1.4.6.3_B_fiducial_config_contour_plot_ext.pdf") + + +# ### Small scales + + +colours = [ + "orange", + "dodgerblue", +] + +linestyle = [ + "solid", + "solid", +] + +line_args = [dict(color=col, ls=ls) for col, ls in zip(colours, linestyle)] + +g = plots.get_subplot_plotter(width_inch=9) +g.settings.axes_fontsize = 25 +g.settings.axes_labelsize = 25 +g.settings.alpha_filled_add = 0.7 +g.settings.legend_fontsize = 30 + +g.plot_2d( + chains, + ["S_8", "OMEGA_M"], # + line_args=line_args, + contour_args=[{"alpha": 0.7}, {"alpha": 1.0}], + contour_colors=colours, + filled=[True, True], +) +g.add_legend(legend_labels, legend_loc="upper right") + +g.export("./../../results/SP_v1.4.6.3_B_fiducial_config_contour_plot_scales.pdf") + + +# ### BBN Prior + + +from getdist.gaussian_mixtures import Gaussian1D + +colours = [ + "orange", + "royalblue", +] + +linestyle = [ + "solid", + "solid", +] + +line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] + +# BBN PRIOR +bbn_prior = Gaussian1D( + mean=0.02218, + sigma=0.00055, + name="ombh2", + labels=[r"\omega_{\rm b}"], + label="BBN prior", +) +bbn_chain = bbn_prior.MCSamples(3000, label="BBN prior") + +g.triangle_plot( + chains + [bbn_chain], + name_list, + legend_labels=legend_labels, + line_args=line_args, + contour_colors=colours, + filled=[True, False], +) + + +# ## Plot the best-fit $\xi_\pm$ + + +xi_p_data = data["XI_PLUS"].data +xi_m_data = data["XI_MINUS"].data +cov_mat = data["COVMAT"].data + +labels = roots_scale.values() + +bbox_to_anchor_xip = (0.685, 0.09) +bbox_to_anchor_xim = (0.3, 0.65) +theta_min = 1.0 +theta_max = 250.0 +loc_legend = "lower center" + + +colours = [ + "orange", + "dodgerblue", +] + +linestyle = [ + "solid", + "solid", +] + +line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] + +labels = roots_scale.values() + +fig, ax = plt.subplots(1, 1, figsize=(11, 7)) + +theta, xi_p, xi_m = xi_p_data["ANG"], xi_p_data["VALUE"], xi_m_data["VALUE"] +ax.errorbar( + theta, + theta * xi_p, + yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])), + fmt="o", + color="black", + capsize=2, +) + +for idx, (label, root) in enumerate(zip(labels, roots_scale)): + # Read the results + theta = ( + ( + np.loadtxt( + path_output_chains + "{}/best_fit/shear_xi_plus/theta.txt".format(root) + ) + ) + * 180 + / np.pi + * 60 + ) + xi_plus = np.loadtxt( + path_output_chains + "{}/best_fit/shear_xi_plus/bin_1_1.txt".format(root) + ) + xi_minus = np.loadtxt( + path_output_chains + "{}/best_fit/shear_xi_minus/bin_1_1.txt".format(root) + ) + xi_sys_plus = np.loadtxt( + path_output_chains + "{}/best_fit/xi_sys/shear_xi_plus.txt".format(root) + ) + xi_sys_minus = np.loadtxt( + path_output_chains + "{}/best_fit/xi_sys/shear_xi_minus.txt".format(root) + ) + theta_xi_sys = ( + np.loadtxt(path_output_chains + "{}/best_fit/xi_sys/theta.txt".format(root)) + * 180 + / np.pi + * 60 + ) + + xi_sys_plus = np.interp(theta, theta_xi_sys, xi_sys_plus) + xi_sys_minus = np.interp(theta, theta_xi_sys, xi_sys_minus) + xi_plus += xi_sys_plus + xi_minus += xi_sys_minus + + mask = (theta > theta_min) & (theta < theta_max) + theta = theta[mask] + ax.plot(theta, theta * xi_plus[mask], label=label, **line_args[idx]) + +ymin = ax.get_ylim()[0] +ymax = ax.get_ylim()[1] + +ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color="gray", alpha=0.2) +ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=5, color="gray", alpha=0.7) +ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=300, color="gray", alpha=0.2) + +ax.set_ylim(ymin, ymax) + +ax.set_ylabel(r"$\theta \xi_\pm$", fontsize=26) +ax.set_xlabel(r"$\theta$ (arcmin)", fontsize=26) +ax.set_xlim([theta.min() - 0.1, theta.max() + 20]) +ax.set_title(r"$\xi_+(\theta)$", fontsize=26) +ax.set_xscale("log") +ax.set_xticks(np.array([1, 10, 100])) +ax.tick_params(axis="x", which="minor", length=2, width=0.8) +ax.tick_params(axis="both", which="major", labelsize=24) +ax.tick_params(axis="both", which="minor", labelsize=20) +ax.yaxis.get_offset_text().set_fontsize(24) +ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) +ax.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xip, fontsize=20) + + +plt.savefig("./../../results/scale_cut_xipm_SP_v1.4.6.3_B.pdf", bbox_inches="tight") + + +labels = roots_nonlin.values() + +colours = ["orange", "hotpink", "teal"] + +linestyle = ["solid", "solid", "dashed"] + +line_args = [dict(color=col, ls=ls, lw=2) for col, ls in zip(colours, linestyle)] + +fig, [ax, ax2] = plt.subplots(2, 1, figsize=(11, 14)) + +theta, xi_p, xi_m = xi_p_data["ANG"], xi_p_data["VALUE"], xi_m_data["VALUE"] +ax.errorbar( + theta, + theta * xi_p, + yerr=theta * np.sqrt(np.diag(cov_mat[: len(theta), : len(theta)])), + fmt="o", + color="black", + capsize=2, +) +ax2.errorbar( + theta, + theta * xi_m, + yerr=theta + * np.sqrt( + np.diag(cov_mat[len(theta) : 2 * len(theta), len(theta) : 2 * len(theta)]) + ), + fmt="o", + color="black", + capsize=2, +) + +for idx, (label, root) in enumerate(zip(labels, roots_nonlin)): + # Read the results + theta = ( + ( + np.loadtxt( + path_output_chains + "{}/best_fit/shear_xi_plus/theta.txt".format(root) + ) + ) + * 180 + / np.pi + * 60 + ) + xi_plus = np.loadtxt( + path_output_chains + "{}/best_fit/shear_xi_plus/bin_1_1.txt".format(root) + ) + xi_minus = np.loadtxt( + path_output_chains + "{}/best_fit/shear_xi_minus/bin_1_1.txt".format(root) + ) + xi_sys_plus = np.loadtxt( + path_output_chains + "{}/best_fit/xi_sys/shear_xi_plus.txt".format(root) + ) + xi_sys_minus = np.loadtxt( + path_output_chains + "{}/best_fit/xi_sys/shear_xi_minus.txt".format(root) + ) + theta_xi_sys = ( + np.loadtxt(path_output_chains + "{}/best_fit/xi_sys/theta.txt".format(root)) + * 180 + / np.pi + * 60 + ) + + xi_sys_plus = np.interp(theta, theta_xi_sys, xi_sys_plus) + xi_sys_minus = np.interp(theta, theta_xi_sys, xi_sys_minus) + xi_plus += xi_sys_plus + xi_minus += xi_sys_minus + + mask = (theta > theta_min) & (theta < theta_max) + theta = theta[mask] + ax.plot(theta, theta * xi_plus[mask], label=label, **line_args[idx]) + ax2.plot(theta, theta * xi_minus[mask], label=label, **line_args[idx]) + +ymin = ax.get_ylim()[0] +ymax = ax.get_ylim()[1] +ax.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color="gray", alpha=0.2) +ax.fill_betweenx(y=[ymin, ymax], x1=83, x2=300, color="gray", alpha=0.2) + +ax.set_ylim(ymin, ymax) + +ax.set_ylabel(r"$\theta \xi_\pm$", fontsize=26) +ax.set_xlabel(r"$\theta$ (arcmin)", fontsize=26) +ax.set_xlim([theta.min() - 0.1, theta.max() + 20]) +ax.set_title(r"$\xi_+(\theta)$", fontsize=26) +ax.set_xscale("log") +ax.set_xticks(np.array([1, 10, 100])) +ax.tick_params(axis="x", which="minor", length=2, width=0.8) +ax.tick_params(axis="both", which="major", labelsize=24) +ax.tick_params(axis="both", which="minor", labelsize=20) +ax.yaxis.get_offset_text().set_fontsize(24) +ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) + + +ymin = ax2.get_ylim()[0] +ymax = ax2.get_ylim()[1] +ax2.fill_betweenx(y=[ymin, ymax], x1=0, x2=12, color="gray", alpha=0.2) +ax2.fill_betweenx(y=[ymin, ymax], x1=83, x2=3000, color="gray", alpha=0.2) + +ax2.set_ylim(ymin, ymax) +ax2.set_xlabel(r"$\theta$ (arcmin)", fontsize=26) +ax2.set_xlim([theta.min() - 0.1, theta.max()]) +ax2.set_xscale("log") +ax2.set_title(r"$\xi_-(\vartheta)$", fontsize=26) +ax2.set_xticks(np.array([1, 10, 100])) +ax2.tick_params(axis="x", which="minor", length=2, width=0.8) +ax2.tick_params(axis="both", which="major", labelsize=24) +ax2.tick_params(axis="both", which="minor", labelsize=20) +ax2.yaxis.get_offset_text().set_fontsize(24) +ax2.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) +ax2.legend(loc=loc_legend, bbox_to_anchor=bbox_to_anchor_xim, fontsize=20) + +plt.savefig("./../../results/nonlin_xipm_SP_v1.4.6.3_B.pdf", bbox_inches="tight") diff --git a/papers/realspace/cov_masking.py b/papers/realspace/cov_masking.py new file mode 100644 index 00000000..210638cb --- /dev/null +++ b/papers/realspace/cov_masking.py @@ -0,0 +1,82 @@ +# # Covmat mask analysis +# +# This notebook creates the plots to look at the ratio of the covaraiance matrices when applying the mask or not + + +import os + +import healpy as hp +import matplotlib.pyplot as plt +import numpy as np +import seaborn as sns + +plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") + +plt.rcParams["axes.labelsize"] = 18 +plt.rcParams["xtick.labelsize"] = 18 +plt.rcParams["ytick.labelsize"] = 18 + +plt.rcParams["text.usetex"] = True +sns.set_palette("husl") + +cat_dir = "/n17data/UNIONS/WL/v1.4.x/" +catalog_ver = "v1.4.6.3" +blind = "B" + +nside = 8192 +npix = hp.nside2npix(nside) + +data_dir = "/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/data/" +curr_dir = os.getcwd() + + +# PLOT 2D MAP OF COVMAT masked vs unmasked RATIOS +nbins = 20 +ndata = nbins * 2 +full_ratio = np.zeros((ndata, ndata)) + +cov = np.loadtxt(data_dir + f"/covs/cov_SP_{catalog_ver}_{blind}.txt") +cov_masked = np.loadtxt(data_dir + f"/covs/cov_masked_SP_{catalog_ver}_{blind}.txt") + +for i in range(ndata): + for j in range(ndata): + full_ratio[i][j] = cov_masked[i][j] / cov[i][j] + +fig = plt.figure() +ax = fig.add_subplot(1, 1, 1) +extent = (0, ndata, ndata, 0) + +vmin, vmax = np.percentile(full_ratio, [1, 99]) + +im3 = ax.imshow(full_ratio, cmap="RdBu_r", vmin=vmin, vmax=vmax, extent=extent) + +cbar = fig.colorbar(im3, ax=ax, fraction=0.046, pad=0.04) + +ax.text(int(ndata / 4), ndata + 5, r"$\xi_+$", fontsize=15) +ax.text(3 * int(ndata / 4), ndata + 5, r"$\xi_-$", fontsize=15) +ax.text(-8, int(ndata / 4), r"$\xi_+$", fontsize=15, rotation=90) +ax.text(-8, 3 * int(ndata / 4), r"$\xi_-$", fontsize=15, rotation=90) +ax.set_xticks([0, 10, 20, 30, 40]) +ax.set_yticks([0, 10, 20, 30, 40]) +ax.set_yticklabels(["1'", "125'", "250'", "125'", "250'"]) +ax.set_xticklabels(["1'", "125'", "250'", "125'", "250'"]) +plt.axvline(x=int(ndata / 2), color="white", linewidth=1.0) +plt.axhline(y=int(ndata / 2), color="white", linewidth=1.0) + +plt.savefig( + f"./../../results/covmat_masked_unmasked_ratio_{catalog_ver}_{blind}.pdf", + bbox_inches="tight", +) + + +theta = np.linspace(1, 250, 20) +plt.axhline(y=1, color="k", ls="--") +plt.plot(theta, np.diag(cov_masked)[:20] / np.diag(cov)[:20], label=r"$\xi_+$") +plt.plot(theta, np.diag(cov_masked)[20:] / np.diag(cov)[20:], label=r"$\xi_-$") + +plt.xlabel(r"$\theta$ (arcmin)") +plt.ylabel("Cov masked / Cov unmasked") +plt.legend(fontsize=20) +plt.savefig( + "./../../results/covmat_masked_unmasked_ratio_diag.pdf", bbox_inches="tight" +) diff --git a/papers/realspace/get_chi2.py b/papers/realspace/get_chi2.py new file mode 100644 index 00000000..c87a33da --- /dev/null +++ b/papers/realspace/get_chi2.py @@ -0,0 +1,564 @@ +import configparser +import os +import re +import subprocess +import sys + +import matplotlib.pyplot as plt +import numpy as np +import scipy.stats as stats +from astropy.io import fits +from getdist import plots +from scipy.interpolate import interp1d + +sys.path.append("/home/guerrini/sp_validation/cosmo_inference/scripts") + +import chain_postprocessing + +plt.rc("mathtext", fontset="stix") +plt.rc("font", family="sans-serif") + +g = plots.get_subplot_plotter(width_inch=30) +g.settings.axes_fontsize = 30 +g.settings.axes_labelsize = 30 +g.settings.alpha_filled_add = 0.7 +g.settings.legend_fontsize = 40 + +# #SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE +root_dir = "/n09data/guerrini/output_chains/" +blind = "B" + +roots = [ + f"SP_v1.4.6.3_{blind}_fiducial_config", + f"SP_v1.4.6.3_{blind}_small_scales_config", + f"SP_v1.4.6.3_{blind}_flat_alpha_beta_config", + f"SP_v1.4.6.3_{blind}_no_xi_sys_config", + f"SP_v1.4.6.3_{blind}_no_leak_corr_config", + f"SP_v1.4.6.3_{blind}_flat_delta_z_config", + f"SP_v1.4.6.3_{blind}_no_delta_z_config", + f"SP_v1.4.6.3_{blind}_flat_ia_config", + f"SP_v1.4.6.3_{blind}_no_ia_config", + f"SP_v1.4.6.3_{blind}_no_m_bias_config", + f"SP_v1.4.6.3_{blind}_unmasked_covmat_config", + f"SP_v1.4.6.3_{blind}_halofit_config", + f"SP_v1.4.6.3_{blind}_no_baryons_config", + f"SP_v1.4.6.3_{blind}_nautilus_config", + f"SP_v1.4.6.3_{blind}_planck_config", + f"SP_v1.4.6.3_{blind}_planck_desi_config", +] + +catalog_versions = [ + f"SP_v1.4.6.3_config/SP_v1.4.6.3_{blind}", +] + +catalog_sub_versions = [ + f"SP_v1.4.6.3_leak_corr_{blind}_masked", + f"SP_v1.4.6.3_leak_corr_{blind}_masked", + f"SP_v1.4.6.3_leak_corr_{blind}_masked", + f"SP_v1.4.6.3_leak_corr_{blind}_masked", + f"SP_v1.4.6.3_{blind}_masked", + f"SP_v1.4.6.3_leak_corr_{blind}_masked", + f"SP_v1.4.6.3_leak_corr_{blind}_masked", + f"SP_v1.4.6.3_leak_corr_{blind}_masked", + f"SP_v1.4.6.3_leak_corr_{blind}_masked", + f"SP_v1.4.6.3_leak_corr_{blind}_masked", + f"SP_v1.4.6.3_leak_corr_{blind}", + f"SP_v1.4.6.3_leak_corr_{blind}_masked", + f"SP_v1.4.6.3_leak_corr_{blind}_masked", + f"SP_v1.4.6.3_leak_corr_{blind}_masked", + f"SP_v1.4.6.3_leak_corr_{blind}_masked", + f"SP_v1.4.6.3_leak_corr_{blind}_masked", +] +output_folder = "/n09data/guerrini/output_chains/" + +path_ini_files = "/home/guerrini/sp_validation/cosmo_inference/cosmosis_config/" + + +ini_roots = [ + f"blind_{blind}/fiducial", + f"blind_{blind}/small_scales", + f"blind_{blind}/flat_alpha_beta", + f"blind_{blind}/no_xi_sys", + f"blind_{blind}/no_leak_corr", + f"blind_{blind}/flat_delta_z", + f"blind_{blind}/no_delta_z", + f"blind_{blind}/flat_ia", + f"blind_{blind}/no_ia", + f"blind_{blind}/no_m_bias", + f"blind_{blind}/unmasked_covmat", + f"blind_{blind}/halofit", + f"blind_{blind}/no_baryons", + f"blind_{blind}/nautilus", + f"blind_{blind}/planck", + f"blind_{blind}/planck_desi", +] + +properties = {} + +for i, root in enumerate(roots): + print(root) + config = configparser.ConfigParser() + config.optionxform = str # Preserve case sensitivity of option names + config.read( + path_ini_files + + "config_space_v1.4.6.3_fiducial/pipeline/" + + ini_roots[i] + + ".ini" + ) + add_xi_sys = config["2pt_like"]["add_xi_sys"] + lower_bound_xi_plus, upper_bound_xi_plus = map( + float, config["2pt_like"]["angle_range_XI_PLUS_1_1"].split() + ) + lower_bound_xi_minus, upper_bound_xi_minus = map( + float, config["2pt_like"]["angle_range_XI_MINUS_1_1"].split() + ) + + properties[root] = { + "add_xi_sys": add_xi_sys, + "lower_bound_xi_plus": lower_bound_xi_plus, + "upper_bound_xi_plus": upper_bound_xi_plus, + "lower_bound_xi_minus": lower_bound_xi_minus, + "upper_bound_xi_minus": upper_bound_xi_minus, + } + + +# ## Retrieve the chains + + +# READ CHAIN + +chains = [] + +for i, root in enumerate(roots): + burnin = 0 + + if os.path.isfile(root_dir + "{}/getdist_{}.txt".format(root, root)) == False: + samples = np.loadtxt(root_dir + "{}/samples_{}.txt".format(root, root)) + + if "nautilus" in root: + samples = np.column_stack( + ( + np.exp(samples[:, -3]), + samples[:, -1] - samples[:, -2], + samples[:, 0:-3], + ) + ) + elif "mh" in root: + samples = np.column_stack( + ( + np.ones_like(samples[:, -1]), + np.log(samples[:, -1]) - np.log(samples[:, -2]), + samples[:, 0:-2], + ) + ) + burnin = 0.3 + else: + samples = np.column_stack( + (samples[:, -1], samples[:, -3], samples[:, 0:-4]) + ) + + np.savetxt(root_dir + "{}/getdist_{}.txt".format(root, root), samples) + + chain = g.samples_for_root( + root_dir + "{}/getdist_{}".format(root, root), + cache=False, + settings={ + "ignore_rows": burnin, + "smooth_scale_2D": 0.5, + "smooth_scale_1D": 0.5, + }, + ) + p = chain.getParams() + + chains.append(chain) + + +param_list = [ + "OMEGA_M", + "ombh2", + "h0", + "n_s", + "SIGMA_8", + "s_8_input", + "logt_agn", + "a", + "m1", + "bias_1", + "alpha", + "beta", + "omch2", + "m", + "a_planck", +] +label_list = [ + r"\Omega_m", + r"\omega_b", + "h_0", + "n_s", + r"\sigma_8", + "S_8", + "log T_{AGN}", + "A_{IA}", + "m_1", + r"\Delta z_1", + "\\alpha_{PSF}", + "\\beta_{PSF}", + r"\omega_c", + "M", + "A_{\rm Planck}", +] + +for chain in chains: + param_names = chain.getParamNames() + for name, label in zip(param_list, label_list): + if param_names.parWithName(name) is not None: + param_names.parWithName(name).label = label + + +# ## Extract the best fit parameters + + +best_fit = {} + +for root, chain in zip(roots, chains): + print(root) + p = chain.getParams() + + best_fit[root] = chain_postprocessing.extract_best_fit_params( + chain, best_fit_method="2Dkde" + ) + + for param_name in best_fit[root].keys(): + high_68, low_68, high_95, low_95 = chain_postprocessing.compute_limits( + chain, param_name + ) + if param_name == "S_8": + print(f"{best_fit[root][param_name]}") + + +# ## Run `Cosmosis` in test mode to get the data vectors + + +if not os.path.exists(path_ini_files + "/values_empty.ini"): + content = """[cosmological_parameters] + +tau = 0.0544 +w = -1.0 +mnu = 0.06 +omega_k = 0.0 +wa = 0.0 + +[halo_model_parameters] + +[intrinsic_alignment_parameters] + +[shear_calibration_parameters] + +[nofz_shifts] + +[psf_leakage_parameters] +""" + + with open(path_ini_files + "/values_empty.ini", "w") as f: + f.write(content) + f.close() + + print("File created successfully") + + +section_map = { + "omch2": "cosmological_parameters", + "ombh2": "cosmological_parameters", + "h0": "cosmological_parameters", + "n_s": "cosmological_parameters", + "tau": "cosmological_parameters", + "s_8_input": "cosmological_parameters", + "logt_agn": "halo_model_parameters", + "a": "intrinsic_alignment_parameters", + "m1": "shear_calibration_parameters", + "bias_1": "nofz_shifts", + "alpha": "psf_leakage_parameters", + "beta": "psf_leakage_parameters", + "m": "supernova_params", + "a_planck": "planck", +} + +best_fit["SP_v1.4.6.3_B_no_ia_config"]["a"] = 0 + + +env = os.environ.copy() +env["LD_LIBRARY_PATH"] = ( + "/home/guerrini/.conda/envs/sp_validation/lib/python3.9/site-packages/cosmosis/datablock:" + + env.get("LD_LIBRARY_PATH", "") +) + +for i, root in enumerate(roots): + print(root) + config = configparser.ConfigParser() + config.optionxform = str # Preserve case sensitivity of option names + + for param, section in section_map.items(): + # Check if this parameter exists for the current root + if param in best_fit[root]: + value = best_fit[root][param] + + if section not in config: + config.add_section(section) + + config[section][param] = str(value) + + with open(path_ini_files + "/values_empty.ini", "w") as configfile: + config.write(configfile) + + # Modify the ini file to run in test mode at the best fit + config = configparser.ConfigParser() + config.optionxform = str # Preserve case sensitivity of option names + + ini_file = path_ini_files + "config_space_v1.4.6.3_fiducial/pipeline/{}.ini".format( + ini_roots[i] + ) + config.read(ini_file) + + sampler = config["runtime"]["sampler"] + config["runtime"]["sampler"] = "test" + values = config["pipeline"]["values"] + config["pipeline"]["values"] = path_ini_files + "/values_empty.ini" + config["DEFAULT"]["FITS_FILE"] = ( + f"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_versions[0]}/cosmosis_{catalog_sub_versions[i]}.fits" + ) + config["test"]["save_dir"] = root_dir + "{}/best_fit".format(root) + + with open(ini_file, "w") as configfile: + config.write(configfile) + + # Run cosmosis + result = subprocess.run( + ["cosmosis", ini_file], env=env, capture_output=True, text=True + ) + print(f"STDOUT:\n{result.stdout}") + print(f"STDERR:\n{result.stderr}") + + # Modify the ini file to the previous one + config["pipeline"]["values"] = values + config["runtime"]["sampler"] = sampler + + with open(ini_file, "w") as configfile: + config.write(configfile) + + +# ## Compute the $\chi^2$ + + +metrics = {} + +for idx, root in enumerate(roots): + print(root) + match = re.search(r"corr_([A-Za-z])", root) + if match: + blind = match.group(1) + + add_xi_sys = properties[root]["add_xi_sys"] + print(f"add_xi_sys: {add_xi_sys}") + lower_bound_xi_plus = properties[root]["lower_bound_xi_plus"] + upper_bound_xi_plus = properties[root]["upper_bound_xi_plus"] + lower_bound_xi_minus = properties[root]["lower_bound_xi_minus"] + upper_bound_xi_minus = properties[root]["upper_bound_xi_minus"] + + # Read the results + theta = np.loadtxt( + output_folder + "{}/best_fit/shear_xi_plus/theta.txt".format(root) + ) + theta_arcmin = theta * 180 * 60 / np.pi + shear_xi_plus = np.loadtxt( + output_folder + "{}/best_fit/shear_xi_plus/bin_1_1.txt".format(root) + ) + shear_xi_minus = np.loadtxt( + output_folder + "{}/best_fit/shear_xi_minus/bin_1_1.txt".format(root) + ) + + if add_xi_sys == "T": + xi_sys_plus = np.loadtxt( + output_folder + "{}/best_fit/xi_sys/shear_xi_plus.txt".format(root) + ) + xi_sys_minus = np.loadtxt( + output_folder + "{}/best_fit/xi_sys/shear_xi_minus.txt".format(root) + ) + + theta_tau = np.loadtxt( + output_folder + "{}/best_fit/tau_0_plus/theta.txt".format(root) + ) + theta_tau_arcmin = theta_tau * 180 * 60 / np.pi + tau_0_model = np.loadtxt( + output_folder + "{}/best_fit/tau_0_plus/bin_1_1.txt".format(root) + ) + tau_2_model = np.loadtxt( + output_folder + "{}/best_fit/tau_2_plus/bin_1_1.txt".format(root) + ) + + data = fits.open( + f"/home/guerrini/sp_validation/cosmo_inference/data/{catalog_versions[0]}/cosmosis_{catalog_sub_versions[idx]}.fits" + ) + + tau_0_data = data["TAU_0_PLUS"].data["VALUE"] + tau_2_data = data["TAU_2_PLUS"].data["VALUE"] + + theta_data = data["XI_PLUS"].data["ANG"] + xi_plus_data = data["XI_PLUS"].data["VALUE"] + xi_minus_data = data["XI_MINUS"].data["VALUE"] + + # Load the covariance + cov = data["COVMAT"].data + cov_xi = cov[0 : 2 * len(xi_plus_data), 0 : 2 * len(xi_plus_data)] + cov_tau = cov[2 * len(xi_plus_data) :, 2 * len(xi_plus_data) :] + + # interpolate the model + interp_xi_plus = interp1d( + theta_arcmin, shear_xi_plus, kind="cubic", fill_value="extrapolate" + ) + interp_xi_minus = interp1d( + theta_arcmin, shear_xi_minus, kind="cubic", fill_value="extrapolate" + ) + + xi_plus_model = interp_xi_plus(theta_data) + if add_xi_sys: + xi_plus_model += xi_sys_plus + xi_minus_model = interp_xi_minus(theta_data) + if add_xi_sys: + xi_minus_model += xi_sys_minus + + # Concatenate the data vector + xi_data = np.concatenate((xi_plus_data, xi_minus_data)) + xi_model = np.concatenate((xi_plus_model, xi_minus_model)) + + tau_data = np.concatenate((tau_0_data, tau_2_data)) + tau_model = np.concatenate((tau_0_model, tau_2_model)) + + # Apply scale cuts + mask_xi_plus = (theta_data > lower_bound_xi_plus) & ( + theta_data < upper_bound_xi_plus + ) + mask_xi_minus = (theta_data > lower_bound_xi_minus) & ( + theta_data < upper_bound_xi_minus + ) + mask = np.concatenate((mask_xi_plus, mask_xi_minus)) + + xi_data = xi_data[mask] + xi_model = xi_model[mask] + cov_xi = cov_xi[mask][:, mask] + + cov_xi_plus = cov[0 : len(xi_plus_data), 0 : len(xi_plus_data)] + cov_xi_plus = cov_xi_plus[mask_xi_plus][:, mask_xi_plus] + cov_xi_minus = cov[ + len(xi_plus_data) : 2 * len(xi_minus_data), + len(xi_plus_data) : 2 * len(xi_minus_data), + ] + cov_xi_minus = cov_xi_minus[mask_xi_minus][:, mask_xi_minus] + + xi_plus_chi2 = np.dot( + (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]), + np.dot( + np.linalg.inv(cov_xi_plus), + (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]), + ), + ) + xi_minus_chi2 = np.dot( + (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]), + np.dot( + np.linalg.inv(cov_xi_minus), + (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]), + ), + ) + xi_chi2 = np.dot( + (xi_model - xi_data), np.dot(np.linalg.inv(cov_xi), (xi_model - xi_data)) + ) + tau_chi2 = np.dot( + (tau_model - tau_data), np.dot(np.linalg.inv(cov_tau), (tau_model - tau_data)) + ) + n_dof_xi_plus = np.sum(mask_xi_plus) + n_dof_xi_minus = np.sum(mask_xi_minus) + n_dof_tau = len(tau_0_data) + len(tau_2_data) + p_value_xi_plus = 1 - stats.chi2.cdf(xi_plus_chi2, n_dof_xi_plus) + p_value_xi_minus = 1 - stats.chi2.cdf(xi_minus_chi2, n_dof_xi_minus) + p_value_xi = 1 - stats.chi2.cdf(xi_chi2, n_dof_xi_plus + n_dof_xi_minus) + p_value_tau = 1 - stats.chi2.cdf(tau_chi2, n_dof_tau) + chi2_tot = xi_plus_chi2 + xi_minus_chi2 + tau_chi2 + n_dof_tot = n_dof_xi_plus + n_dof_xi_minus + n_dof_tau + p_value_tot = 1 - stats.chi2.cdf(chi2_tot, n_dof_tot) + + metrics[root] = { + "chi2_xi_plus": xi_plus_chi2, + "n_dof_xi_plus": n_dof_xi_plus, + "p_value_xi_plus": p_value_xi_plus, + "chi2_xi_minus": xi_minus_chi2, + "n_dof_xi_minus": n_dof_xi_minus, + "p_value_xi_minus": p_value_xi_minus, + "chi2_xi": xi_chi2, + "p_value_xi": p_value_xi, + "chi2_tau": tau_chi2, + "n_dof_tau": n_dof_tau, + "p_value_tau": p_value_tau, + "chi2_tot": chi2_tot, + "n_dof_tot": n_dof_tot, + "p_value_tot": p_value_tot, + } + print("Done!") + + +def get_latex_table(metrics): + latex_lines = [ + r"\begin{tabular}{lccc|ccc|ccc}", + r"\hline", + r"Root & $\chi^2_{\xi^+}$/dof & $p_{\xi^+}$ & $\chi^2_{\xi^-}$/dof & $p_{\xi^+}$ & $\chi^2_{\xi}$/dof & $p_{\xi}$ &" + r"$\chi^2_\tau$/dof & $p_\tau$ & $\chi^2_{\text{tot}}$/dof & $p_{\text{tot}}$ \\", + r"\hline", + ] + + for root, vals in metrics.items(): + escaped = root.replace("_", r"\_") + line = ( + f"{escaped} & " + f"{vals['chi2_xi_plus']:.2f}/{vals['n_dof_xi_plus']} & {vals['p_value_xi_plus']:.3g} & " + f"{vals['chi2_xi_minus']:.2f}/{vals['n_dof_xi_minus']} & {vals['p_value_xi_minus']:.3g} & " + f"{vals['chi2_xi']:.2f}/{vals['n_dof_xi_plus'] + vals['n_dof_xi_minus']} & {vals['p_value_xi']:.3g} &" + f"{vals['chi2_tau']:.2f}/{vals['n_dof_tau']} & {vals['p_value_tau']:.3g} & " + f"{vals['chi2_tot']:.2f}/{vals['n_dof_tot']} & {vals['p_value_tot']:.3g} \\\\" + ) + latex_lines.append(line) + + latex_lines.append(r"\hline") + latex_lines.append(r"\end{tabular}") + + # Print LaTeX table + print("\n".join(latex_lines)) + + +get_latex_table(metrics) + + +def display_markdown(metrics): + # Build Markdown table + header = ( + "| Root | $\\chi^2$ (ξ⁺) / dof | p-val (ξ⁺) |$\\chi^2$ (ξ-) / dof | p-val (ξ-) | $\\chi^2$ (ξ) / dof | p-val (ξ) | $\\chi^2$ (τ) / dof | p-val (τ) | $\\chi^2$ (tot) / dof | p-val (tot) |\n" + "|------|----------------|------------|----------------|------------|------------|---------------|------------|------------|------------------|--------------|\n" + ) + + rows = [] + for root, vals in metrics.items(): + row = f"| `{root}` " + row += f"| {vals['chi2_xi_plus']:.2f} / {vals['n_dof_xi_plus']} " + row += f"| {vals['p_value_xi_plus']:.5f} " + row += f"| {vals['chi2_xi_minus']:.2f} / {vals['n_dof_xi_minus']} " + row += f"| {vals['p_value_xi_minus']:.5f} " + row += f"| {vals['chi2_xi']:.2f} / {vals['n_dof_xi_minus'] + vals['n_dof_xi_plus']} " + row += f"| {vals['p_value_xi']:.5f} " + row += f"| {vals['chi2_tau']:.2f} / {vals['n_dof_tau']} " + row += f"| {vals['p_value_tau']:.5f} " + row += f"| {vals['chi2_tot']:.2f} / {vals['n_dof_tot']} " + row += f"| {vals['p_value_tot']:.5f} |" + rows.append(row) + + # Display in Jupyter + return header + "\n".join(rows) + + +markdown_source = display_markdown(metrics) diff --git a/papers/realspace/get_chi2_glass_mock.py b/papers/realspace/get_chi2_glass_mock.py new file mode 100644 index 00000000..596da161 --- /dev/null +++ b/papers/realspace/get_chi2_glass_mock.py @@ -0,0 +1,468 @@ +import configparser +import os +import subprocess +import sys + +import matplotlib.pyplot as plt +import numpy as np + +# Make the plot +import seaborn as sns +from astropy.io import fits +from getdist import plots +from scipy.interpolate import interp1d +from scipy.stats import chi2 + +sys.path.append("/home/guerrini/sp_validation/cosmo_inference/scripts") + +import chain_postprocessing + +plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") + +plt.rcParams["axes.labelsize"] = 18 +plt.rcParams["xtick.labelsize"] = 18 +plt.rcParams["ytick.labelsize"] = 18 + +plt.rcParams["text.usetex"] = True + +g = plots.get_subplot_plotter(width_inch=30) +g.settings.axes_fontsize = 30 +g.settings.axes_labelsize = 30 +g.settings.alpha_filled_add = 0.7 +g.settings.legend_fontsize = 40 + +# #SPECIFY DATA DIRECTORY AND DESIRED CHAINS TO ANALYSE + +root_dir = "/n09data/guerrini/glass_mock_chains/" + +# Version of the glass mock chain run +chain_version = "v6" + +# Path to the glass mock data vectors +root_glass_dv = ( + f"/home/guerrini/sp_validation/cosmo_inference/data/glass_mocks/{chain_version}/" +) + +# Choose the best-fit method +best_fit_method = "2Dkde" + +# Create the list of mocks +max_sim = 350 +failed_simulations = [82, 83, 281, 282, 283, 284, 285, 286, 287] +roots = [f"glass_mock_{chain_version}_{str(i).zfill(5)}" for i in range(1, max_sim + 1)] +roots = [root for root in roots if int(root.split("_")[-1]) not in failed_simulations] + +catalog_versions = [ + "SP_v1.4.6.3_config/SP_v1.4.6.3_A", +] + +output_folder_chains = "/n23data1/n06data/lgoh/scratch/temp/" +path_ini_files = "/home/xguerrini/sp_validation/cosmo_inference/cosmosis_config/" + +ini_root = "blind_A/fiducial" + +lower_bound_xi = 12 +upper_bound_xi = 83 + +# ## Retrieve the chains + + +# READ CHAIN + +chains = [] +best_fit = {} + +for i, root in enumerate(roots): + burnin = 0 + + if os.path.isfile(f"{root_dir}/{root}/{root}/getdist_{root}.txt") == True: + chain = g.samples_for_root( + f"{root_dir}/{root}/{root}/getdist_{root}", + cache=False, + settings={ + "ignore_rows": burnin, + "smooth_scale_2D": 0.5, + "smooth_scale_1D": 0.5, + }, + ) + p = chain.getParams() + + best_fit[root] = chain_postprocessing.extract_best_fit_params( + chain, best_fit_method="2Dkde" + ) + + +param_list = [ + "OMEGA_M", + "ombh2", + "h0", + "n_s", + "SIGMA_8", + "s_8_input", + "logt_agn", + "a", + "m1", + "bias_1", + "alpha", + "beta", + "omch2", + "m", + "a_planck", +] +label_list = [ + r"\Omega_m", + r"\omega_b", + "h_0", + "n_s", + r"\sigma_8", + "S_8", + "log T_{AGN}", + "A_{IA}", + "m_1", + r"\Delta z_1", + "\\alpha_{PSF}", + "\\beta_{PSF}", + r"\omega_c", + "M", + "A_{\rm Planck}", +] + + +# ## Run `Cosmosis` in test mode to get the data vectors + + +if not os.path.exists(path_ini_files + "/values_empty.ini"): + content = """[cosmological_parameters] + +tau = 0.0544 +w = -1.0 +mnu = 0.06 +omega_k = 0.0 +wa = 0.0 + +[halo_model_parameters] + +[intrinsic_alignment_parameters] + +[shear_calibration_parameters] + +[nofz_shifts] + +[psf_leakage_parameters] +""" + + with open(path_ini_files + "/values_empty.ini", "w") as f: + f.write(content) + f.close() + + print("File created successfully") + + +section_map = { + "omch2": "cosmological_parameters", + "ombh2": "cosmological_parameters", + "h0": "cosmological_parameters", + "n_s": "cosmological_parameters", + "s_8_input": "cosmological_parameters", + "logt_agn": "halo_model_parameters", + "a": "intrinsic_alignment_parameters", + "m1": "shear_calibration_parameters", + "bias_1": "nofz_shifts", + "alpha": "psf_leakage_parameters", + "beta": "psf_leakage_parameters", +} + + +env = os.environ.copy() +env["LD_LIBRARY_PATH"] = ( + "/home/guerrini/.conda/envs/sp_validation/lib/python3.9/site-packages/cosmosis/datablock:" + + env.get("LD_LIBRARY_PATH", "") +) +for i, root in enumerate(roots): + print(root) + config = configparser.ConfigParser() + config.optionxform = str # Preserve case sensitivity of option names + + for param, section in section_map.items(): + # Check if this parameter exists for the current root + if param in best_fit[root]: + value = best_fit[root][param] + + if section not in config: + config.add_section(section) + + config[section][param] = str(value) + + with open(path_ini_files + "/values_empty.ini", "w") as configfile: + config.write(configfile) + + # Modify the ini file to run in test mode at the best fit + config = configparser.ConfigParser() + config.optionxform = str # Preserve case sensitivity of option names + + ini_file = ( + path_ini_files + f"config_space_v1.4.6.3_fiducial/pipeline/{ini_root}.ini" + ) + config.read(ini_file) + + sampler = config["runtime"]["sampler"] + config["runtime"]["sampler"] = "test" + values = config["pipeline"]["values"] + config["pipeline"]["values"] = path_ini_files + "/values_empty.ini" + config["DEFAULT"]["FITS_FILE"] = ( + f"{root_glass_dv}/glass_mock_{root[-5:]}/cosmosis_glass_mock_v6_{root[-5:]}.fits" + ) + config["test"]["save_dir"] = output_folder_chains + f"{root}/best_fit_config" + + with open(ini_file, "w") as configfile: + config.write(configfile) + + # Run cosmosis + result = subprocess.run( + ["cosmosis", ini_file], env=env, capture_output=True, text=True + ) + # print(f"STDOUT:\n{result.stdout}") + # print(f"STDERR:\n{result.stderr}") + + # Modify the ini file to the previous one + config["pipeline"]["values"] = values + config["runtime"]["sampler"] = sampler + + with open(ini_file, "w") as configfile: + config.write(configfile) + + +xi_plus_chi2s = np.array([]) +xi_minus_chi2s = np.array([]) +xi_chi2s = np.array([]) +tau_chi2s = np.array([]) +chi2_tots = np.array([]) + + +for idx, root in enumerate(roots): + print(root) + + data = fits.open( + f"{root_glass_dv}/glass_mock_{root[-5:]}/cosmosis_glass_mock_v6_{root[-5:]}.fits" + ) + + tau_0_data = data["TAU_0_PLUS"].data["VALUE"] + tau_2_data = data["TAU_2_PLUS"].data["VALUE"] + + theta_data = data["XI_PLUS"].data["ANG"] + xi_plus_data = data["XI_PLUS"].data["VALUE"] + xi_minus_data = data["XI_MINUS"].data["VALUE"] + xi_data = np.concatenate((xi_plus_data, xi_minus_data)) + + tau_data = np.concatenate((tau_0_data, tau_2_data)) + + # Apply scale cuts + mask_xi_plus = (theta_data > lower_bound_xi) & (theta_data < upper_bound_xi) + mask_xi_minus = (theta_data > lower_bound_xi) & (theta_data < upper_bound_xi) + mask = np.concatenate((mask_xi_plus, mask_xi_minus)) + # Load the covariance + cov = data["COVMAT"].data + cov_xi = cov[0 : 2 * len(xi_plus_data), 0 : 2 * len(xi_plus_data)] + cov_tau = cov[ + 2 * len(xi_plus_data) : 4 * len(xi_plus_data), + 2 * len(xi_plus_data) : 4 * len(xi_plus_data), + ] + xi_data = xi_data[mask] + cov_xi = cov_xi[mask][:, mask] + + cov_xi_plus = cov[0 : len(xi_plus_data), 0 : len(xi_plus_data)] + cov_xi_plus = cov_xi_plus[mask_xi_plus][:, mask_xi_plus] + cov_xi_minus = cov[ + len(xi_plus_data) : 2 * len(xi_minus_data), + len(xi_plus_data) : 2 * len(xi_minus_data), + ] + cov_xi_minus = cov_xi_minus[mask_xi_minus][:, mask_xi_minus] + + # Read the results + theta = np.loadtxt( + output_folder_chains + f"{root}/best_fit_config/shear_xi_plus/theta.txt" + ) + theta_arcmin = theta * 180 * 60 / np.pi + shear_xi_plus = np.loadtxt( + output_folder_chains + f"{root}/best_fit_config/shear_xi_plus/bin_1_1.txt" + ) + shear_xi_minus = np.loadtxt( + output_folder_chains + f"{root}/best_fit_config/shear_xi_minus/bin_1_1.txt" + ) + + xi_sys_plus = np.loadtxt( + output_folder_chains + f"{root}/best_fit_config/xi_sys/shear_xi_plus.txt" + ) + xi_sys_minus = np.loadtxt( + output_folder_chains + f"{root}/best_fit_config/xi_sys/shear_xi_minus.txt" + ) + + theta_tau = np.loadtxt( + output_folder_chains + f"{root}/best_fit_config/tau_0_plus/theta.txt" + ) + theta_tau_arcmin = theta_tau * 180 * 60 / np.pi + tau_0_model = np.loadtxt( + output_folder_chains + f"{root}/best_fit_config/tau_0_plus/bin_1_1.txt" + ) + tau_2_model = np.loadtxt( + output_folder_chains + f"{root}/best_fit_config/tau_2_plus/bin_1_1.txt" + ) + + # interpolate the model + interp_xi_plus = interp1d( + theta_arcmin, shear_xi_plus, kind="cubic", fill_value="extrapolate" + ) + interp_xi_minus = interp1d( + theta_arcmin, shear_xi_minus, kind="cubic", fill_value="extrapolate" + ) + + xi_plus_model = interp_xi_plus(theta_data) + xi_plus_model += xi_sys_plus + xi_minus_model = interp_xi_minus(theta_data) + xi_minus_model += xi_sys_minus + + xi_model = np.concatenate((xi_plus_model, xi_minus_model)) + tau_model = np.concatenate((tau_0_model, tau_2_model)) + xi_model = xi_model[mask] + + xi_plus_chi2 = np.dot( + (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]), + np.dot( + np.linalg.inv(cov_xi_plus), + (xi_plus_model[mask_xi_plus] - xi_plus_data[mask_xi_plus]), + ), + ) + xi_minus_chi2 = np.dot( + (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]), + np.dot( + np.linalg.inv(cov_xi_minus), + (xi_minus_model[mask_xi_minus] - xi_minus_data[mask_xi_minus]), + ), + ) + xi_chi2 = np.dot( + (xi_model - xi_data), np.dot(np.linalg.inv(cov_xi), (xi_model - xi_data)) + ) + tau_chi2 = np.dot( + (tau_model - tau_data), np.dot(np.linalg.inv(cov_tau), (tau_model - tau_data)) + ) + chi2_tot = xi_plus_chi2 + xi_minus_chi2 + tau_chi2 + + xi_plus_chi2s = np.append(xi_plus_chi2s, xi_plus_chi2) + xi_minus_chi2s = np.append(xi_minus_chi2s, xi_minus_chi2) + xi_chi2s = np.append(xi_chi2s, xi_chi2) + tau_chi2s = np.append(tau_chi2s, tau_chi2) + chi2_tots = np.append(chi2_tots, chi2_tot) + + +fig, [ax1, ax2] = plt.subplots(2, 1, figsize=(7, 10)) +chi2_fiducial = -2 * -37.560916821678894 +dof, loc, scale = chi2.fit(chi2_tots, floc=0) + +print(f"Best-fit dof: {dof:.3e}") +counts, bin_edges = np.histogram(chi2_tots, bins=25, density=True) + +sns.histplot( + chi2_tots, + ax=ax1, + kde=False, + bins=bin_edges, + stat="density", + label=r"$\chi^2$ for \texttt{GLASS} mocks best-fits", + color="green", + alpha=0.3, +) + +# Compute the p-value + +# 1. Get in which bin the chi2 of the fiducial falls +bin_index = np.digitize(chi2_fiducial, bin_edges) + +# 2. Compute the p-value as the integral of the tail of the histogram +p_value = np.sum(counts[bin_index:]) * np.diff(bin_edges)[0] + +print(f"P-value: {p_value}") + +ax1.axvline(chi2_fiducial, color="red", label=r"$\chi^2$ of the fiducial", lw=2) + +mantissa, exponent = np.frexp(p_value) +pte_string = rf"${{\rm PTE}} = {p_value:.4f}$" +print(f"mantissa: {mantissa}, exponent: {exponent}") +x_text = 78 +y_text = max(counts) * 0.95 +ax1.text( + x_text, + y_text, + pte_string, + fontsize=15, + bbox=dict(facecolor="wheat", alpha=0.8, edgecolor="black"), +) + +chi2_string = rf"${{\rm Eff. dof}}= {dof:.1f}$" +y_text = max(counts) * 0.85 +ax1.text( + x_text, + y_text, + chi2_string, + fontsize=15, + bbox=dict(facecolor="wheat", alpha=0.8, edgecolor="black"), +) + +ax1.set_xlabel(r"$\chi^2_{\rm tot}$") +ax1.set_ylabel("Density") + +chi2_fiducial = 9.5 +dof, loc, scale = chi2.fit(xi_chi2s, floc=0) + +print(f"Best-fit dof: {dof:.3e}") +counts, bin_edges = np.histogram(xi_chi2s, bins=25, density=True) + +sns.histplot( + xi_chi2s, + ax=ax2, + kde=False, + bins=bin_edges, + stat="density", + label=r"$\chi^2$ for \texttt{GLASS} mocks best-fits", + color="pink", + alpha=0.5, +) + +# Compute the p-value + +# 1. Get in which bin the chi2 of the fiducial falls +bin_index = np.digitize(chi2_fiducial, bin_edges) + +# 2. Compute the p-value as the integral of the tail of the histogram +p_value = np.sum(counts[bin_index:]) * np.diff(bin_edges)[0] + +print(f"P-value: {p_value}") + +ax2.axvline(chi2_fiducial, color="red", label=r"$\chi^2$ of the fiducial", lw=2) + +mantissa, exponent = np.frexp(p_value) +print(f"mantissa: {mantissa}, exponent: {exponent}") +pte_string = rf"${{\rm PTE}} = {p_value:.4f}$" +# rf"${{\rm PTE}} = {mantissa:.2f} \times 10^{{{exponent}}}$" if exponent != 0 else +x_text = 17.5 +y_text = max(counts) * 0.95 +ax2.text( + x_text, + y_text, + pte_string, + fontsize=15, + bbox=dict(facecolor="wheat", alpha=0.8, edgecolor="black"), +) + +chi2_string = rf"${{\rm Eff. dof}}= {dof:.1f}$" +y_text = max(counts) * 0.85 +ax2.text( + x_text, + y_text, + chi2_string, + fontsize=15, + bbox=dict(facecolor="wheat", alpha=0.8, edgecolor="black"), +) + +ax2.set_xlabel(r"$\chi^2 (\xi_\pm)$") +ax2.set_ylabel("Density") +fig.savefig("./../../results/chi2_glass_mocks_p_value_xi_tau.pdf") diff --git a/papers/realspace/get_prior_psf_leakage.py b/papers/realspace/get_prior_psf_leakage.py new file mode 100644 index 00000000..4d335085 --- /dev/null +++ b/papers/realspace/get_prior_psf_leakage.py @@ -0,0 +1,163 @@ +# # Covariance matrix and PSF leakage +# +# This notebook plots the combined covariance matrix, and samples and plots the 2D marginalised posteriors of the PSF leakage parameters $\alpha$ and $\beta$. + + +import matplotlib.pyplot as plt +import numpy as np +import seaborn as sns +from astropy.io import fits +from getdist import MCSamples, plots +from shear_psf_leakage.rho_tau_stat import PSFErrorFit, RhoStat, TauStat + +# Use paper style and seaborn with husl palette +plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") +# Set default palette - will be updated per plot as needed +sns.set_palette("husl") + +g = plots.get_subplot_plotter(width_inch=30) +g.settings.axes_fontsize = 30 +g.settings.axes_labelsize = 30 +g.settings.alpha_filled_add = 0.7 +g.settings.legend_fontsize = 25 + +ver = "v1.4.6.3" +blind = "B" + + +data_path = f"/home/guerrini/sp_validation/cosmo_inference/data/SP_{ver}_config/" + +path_cosmo_val = "/home/guerrini/sp_validation/cosmo_val/output/" + +roots = [f"SP_{ver}_{blind}", f"SP_{ver}_leak_corr_{blind}"] + +labels = [f"SP_{ver}_{blind}", f"SP_{ver}_leak_corr_{blind}"] + + +data_vectors = [] + +for root in roots: + data_vectors.append( + fits.open(data_path + f"SP_{ver}_{blind}/cosmosis_{root}_masked.fits") + ) + + +def cov_to_corr(cov): + """Convert a covariance matrix to a correlation matrix.""" + d = np.sqrt(np.diag(cov)) + corr = cov / np.outer(d, d) + corr[cov == 0] = 0 + return corr + + +# Print the covariance matrix for each root +for i, root in enumerate(roots): + print(f"Covariance matrix for {labels[i]}:") + cov = data_vectors[i]["COVMAT"].data + + n_bins = cov.shape[0] // 4 + + fig, ax = plt.subplots(figsize=(10, 8)) + + im = ax.imshow(cov_to_corr(cov), vmin=-1, vmax=1, cmap="seismic") + ax.set_aspect("equal") + ax.set_yticks(np.array([10, 30, 50, 70])) + ax.set_yticklabels( + [ + r"$\xi_+(\vartheta)$", + r"$\xi_-(\vartheta)$", + r"$\tau_0(\vartheta)$", + r"$\tau_2(\vartheta)$", + ] + ) + ax.set_xticks(np.array([10, 30, 50, 70])) + ax.set_xticklabels( + [ + r"$\xi_+(\vartheta)$", + r"$\xi_-(\vartheta)$", + r"$\tau_0(\vartheta)$", + r"$\tau_2(\vartheta)$", + ], + rotation=45, + ) + fig.colorbar(im, ax=ax) + + plt.savefig(f"./../../results/cov_matrix_{root}.png", bbox_inches="tight", dpi=300) + + +# Create dummy rho and tau stat handler. + +# Inference of the xi_sys parameters +sep_units = "arcmin" +coord_units = "degrees" +theta_min = 1.0 +theta_max = 250 +nbins = 20 + + +TreeCorrConfig_xi = { + "ra_units": coord_units, + "dec_units": coord_units, + "min_sep": theta_min, + "max_sep": theta_max, + "sep_units": sep_units, + "nbins": nbins, + "var_method": "jackknife", +} + +rho_stats_handler = RhoStat(output=".", treecorr_config=TreeCorrConfig_xi, verbose=True) + +tau_stats_handler = TauStat( + catalogs=rho_stats_handler.catalogs, + output=".", + treecorr_config=TreeCorrConfig_xi, + verbose=True, +) + + +# Create a PSFErrorFit instance +psf_fitter = PSFErrorFit( + rho_stats_handler, + tau_stats_handler, + path_cosmo_val + "rho_tau_stats/", + use_eta=False, +) + +g = plots.get_subplot_plotter(width_inch=30) + +g.settings.axes_fontsize = 30 +g.settings.axes_labelsize = 30 +g.settings.alpha_filled_add = 0.7 +g.settings.legend_fontsize = 40 + +chains = [] + +# Load rho-, tau-statistics, and cov_tau from the data_vector +for i, root in enumerate(roots): + print("Sampling PSF parameters for ", labels[i]) + path_rho = f"rho_stats_{root}.fits" + path_tau = f"tau_stats_{root}.fits" + path_cov_rho = f"cov_rho_{root}.npy" + path_cov_tau = f"cov_tau_{root}_th.npy" + psf_fitter.load_rho_stat(path_rho) + psf_fitter.load_tau_stat(path_tau) + psf_fitter.load_covariance(path_cov_rho, cov_type="rho") + psf_fitter.load_covariance(path_cov_tau, cov_type="tau") + samples_lq, _, _ = psf_fitter.get_least_squares_params_samples( + npatch=None, apply_debias=False + ) + + samples_gd = MCSamples( + samples=samples_lq, names=[r"\alpha", r"\beta"], labels=[r"\alpha", r"\beta"] + ) + + chains.append(samples_gd) + +g.triangle_plot( + chains, + filled=True, + legend_labels=labels, + legend_loc="upper right", +) + +plt.savefig("./../../results/psf_leakage_params.png", bbox_inches="tight", dpi=300) diff --git a/papers/realspace/glass_mock_hist.py b/papers/realspace/glass_mock_hist.py new file mode 100644 index 00000000..97b34f1d --- /dev/null +++ b/papers/realspace/glass_mock_hist.py @@ -0,0 +1,458 @@ +import os + +import matplotlib.pyplot as plt +import numpy as np +import seaborn as sns +from getdist import plots +from tqdm import tqdm + +g = plots.get_subplot_plotter(width_inch=7) +g.settings.axes_fontsize = 15 +g.settings.axes_labelsize = 15 +g.settings.alpha_filled_add = 0.7 +g.settings.legend_fontsize = 15 + +if os.path.exists("/home/guerrini/matplotlib_config/paper.mplstyle"): + plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") + +# Set default palette - will be updated per plot as needed +sns.set_palette("husl") + +root_dir = "/n09data/guerrini/glass_mock_chains/" +chain_version = "v6" +num_sims = 350 + +roots = [f"glass_mock_{chain_version}_{i + 1:05d}" for i in range(num_sims)] + + +# +def load_samples_and_write_paramames(root_dir, root, chain_type="configuration"): + assert chain_type in ["configuration", "harmonic"], ( + "chain_type must be 'configuration' or 'harmonic'" + ) + + if chain_type == "configuration": + path_samples = root_dir + "{}/{}/samples_{}.txt".format("/" + root, root, root) + path_paramnames = root_dir + "{}/{}/getdist_{}.paramnames".format( + "/" + root, root, root + ) + else: + path_samples = root_dir + "{}/{}/samples_{}_cell.txt".format( + "/" + root, root, root + ) + path_paramnames = root_dir + "{}/{}/getdist_{}_cell.paramnames".format( + "/" + root, root, root + ) + + with open(path_samples, "r") as file: + params = file.readline()[1:].split("\t")[:-4] + file.close() + + with open(path_paramnames, "w") as file: + for i in range(len(params)): + if len(params[i].split("--")) > 1: + file.write(params[i].split("--")[1] + "\n") + else: + file.write(params[i].split("--")[0] + "\n") + file.close() + + +def write_samples_getdist_format(root_dir, root, chain_type="configuration"): + assert chain_type in ["configuration", "harmonic"], ( + "chain_type must be 'configuration' or 'harmonic'" + ) + + if chain_type == "configuration": + path_samples = root_dir + "{}/{}/samples_{}.txt".format("/" + root, root, root) + path_gd_samples = root_dir + "{}/{}/getdist_{}.txt".format( + "/" + root, root, root + ) + path_gd = root_dir + "{}/{}/getdist_{}".format(root, root, root) + else: + path_samples = root_dir + "{}/{}/samples_{}_cell.txt".format( + "/" + root, root, root + ) + path_gd_samples = root_dir + "{}/{}/getdist_{}_cell.txt".format( + "/" + root, root, root + ) + path_gd = root_dir + "{}/{}/getdist_{}_cell".format(root, root, root) + + samples = np.loadtxt( + path_samples, + ) + if "nautilus" in root: + samples = np.column_stack( + (np.exp(samples[:, -3]), samples[:, -1] - samples[:, -2], samples[:, 0:-3]) + ) + else: + samples = np.column_stack((samples[:, -1], samples[:, -2], samples[:, 0:-4])) + np.savetxt(path_gd_samples, samples) + + chain = g.samples_for_root( + path_gd, + cache=False, + settings={"ignore_rows": 0.0, "smooth_scale_2D": 0.5, "smooth_scale_1D": 0.5}, + ) + + return chain + + +def extract_param_chain(chain, param_names): + margestats = chain.getMargeStats() + likestats = chain.getLikeStats() + + param_values = {} + for param_name in param_names: + if param_name not in chain.getParamNames().list(): + raise ValueError(f"Parameter {param_name} not found in chain.") + + param_stats = margestats.parWithName(param_name) + param_values[param_name] = { + "mean": param_stats.mean, + "1sigma_minus": param_stats.mean - param_stats.limits[0].lower, + "1sigma_plus": param_stats.limits[0].upper - param_stats.mean, + "2sigma_minus": param_stats.mean - param_stats.limits[1].lower, + "2sigma_plus": param_stats.limits[1].upper - param_stats.mean, + } + + param_stats = likestats.parWithName(param_name) + param_names_getdist = chain.getParamNames() + par = param_names_getdist.parWithName(param_name) + kde = chain.get1DDensity(par, num_bins=1000) + kde_map = kde.x[np.argmax(kde.P)] + param_values[param_name].update( + { + "MAP": kde_map, + } + ) + + par = chain.getParamNames().parWithName("S_8") + par_om = chain.getParamNames().parWithName("OMEGA_M") + kde = chain.get2DDensity(par, par_om, fine_bins_2D=1000) + s8_kde_map = kde.x[np.unravel_index(np.argmax(kde.P), kde.P.shape)[1]] + om_kde_map = kde.y[np.unravel_index(np.argmax(kde.P), kde.P.shape)[0]] + param_values["S_8"].update( + { + "MAP_2D": s8_kde_map, + } + ) + param_values["OMEGA_M"].update( + { + "MAP_2D": om_kde_map, + } + ) + + return param_values + + +def concatenate_param_stats(name, param_values, verbose=False): + output = [name] + for key in param_values.keys(): + param_stat = param_values[key] + if verbose: + print( + f"{name} - {key}: {param_stat['mean']:.4f} +{param_stat['1sigma_plus']:.4f}/-{param_stat['1sigma_minus']:.4f} (1σ), +{param_stat['2sigma_plus']:.4f}/-{param_stat['2sigma_minus']:.4f} (2σ)" + ) + + param_list = [ + param_stat["mean"], + param_stat["1sigma_minus"], + param_stat["1sigma_plus"], + param_stat["2sigma_minus"], + param_stat["2sigma_plus"], + param_stat["MAP"], + ] + + if key == "S_8": + param_list.append(param_stat["MAP_2D"]) + + if key == "OMEGA_M": + param_list.append(param_stat["MAP_2D"]) + + output += param_list + + return output + + +def merge_param_stats(params_configuration, params_harmonic): + merged_params = {} + for key in params_configuration.keys(): + if key in params_harmonic: + merged_params[key] = { + "configuration": params_configuration[key], + "harmonic": params_harmonic[key], + } + return merged_params + + +def concatenate_merge_params(name, merged_params, verbose=False): + output = [name] + for key in merged_params.keys(): + param_config = merged_params[key]["configuration"] + param_harm = merged_params[key]["harmonic"] + + if verbose: + print( + f"{name} - {key} (Configuration): {param_config['mean']:.4f} +{param_config['1sigma_plus']:.4f}/-{param_config['1sigma_minus']:.4f} (1σ), +{param_config['2sigma_plus']:.4f}/-{param_config['2sigma_minus']:.4f} (2σ)" + ) + print( + f"{name} - {key} (Harmonic): {param_harm['mean']:.4f} +{param_harm['1sigma_plus']:.4f}/-{param_harm['1sigma_minus']:.4f} (1σ), +{param_harm['2sigma_plus']:.4f}/-{param_harm['2sigma_minus']:.4f} (2σ)" + ) + + param_list = [ + param_config["mean"], + param_config["1sigma_minus"], + param_config["1sigma_plus"], + param_config["2sigma_minus"], + param_config["2sigma_plus"], + param_config["MAP"], + param_harm["mean"], + param_harm["1sigma_minus"], + param_harm["1sigma_plus"], + param_harm["2sigma_minus"], + param_harm["2sigma_plus"], + param_harm["MAP"], + ] + + output += param_list + + return output + + +chain_harmonic = [] +chain_config = [] + +for i, root in enumerate(tqdm(roots)): + if os.path.isfile(f"{root_dir}/{root}/{root}/getdist_{root}.txt"): + # Load samples and write paramnames for harmonic space + load_samples_and_write_paramames(root_dir, root, chain_type="harmonic") + write_samples_getdist_format(root_dir, root, chain_type="harmonic") + chain_harm = g.samples_for_root( + root_dir + f"/{root}/{root}/getdist_{root}_cell", + cache=False, + settings={ + "ignore_rows": 0.0, + "smooth_scale_2D": 0.5, + "smooth_scale_1D": 0.5, + }, + ) + chain_harmonic.append(chain_harm) + + # Load samples and write paramnames for harmonic space + load_samples_and_write_paramames(root_dir, root, chain_type="configuration") + write_samples_getdist_format(root_dir, root, chain_type="configuration") + chain_conf = g.samples_for_root( + root_dir + f"/{root}/{root}/getdist_{root}", + cache=False, + settings={ + "ignore_rows": 0.0, + "smooth_scale_2D": 0.5, + "smooth_scale_1D": 0.5, + }, + ) + chain_config.append(chain_conf) +# +param_names = ["S_8", "OMEGA_M", "SIGMA_8", "a"] + +output_mocks_harm = np.array( + [ + "Name", + "S8_mean", + "S8_1sigma_minus", + "S8_1sigma_plus", + "S8_2sigma_minus", + "S8_2sigma_plus", + "S8_MAP", + "S8_MAP_2D", + "OMEGA_M_mean", + "OMEGA_M_1sigma_minus", + "OMEGA_M_1sigma_plus", + "OMEGA_M_2sigma_minus", + "OMEGA_M_2sigma_plus", + "OMEGA_M_MAP", + "OMEGA_M_MAP_2D", + "SIGMA_8_mean", + "SIGMA_8_1sigma_minus", + "SIGMA_8_1sigma_plus", + "SIGMA_8_2sigma_minus", + "SIGMA_8_2sigma_plus", + "SIGMA_8_MAP", + "a_mean", + "a_1sigma_minus", + "a_1sigma_plus", + "a_2sigma_minus", + "a_2sigma_plus", + "a_MAP", + ] +) + +output_mocks_config = np.array( + [ + "Name", + "S8_mean", + "S8_1sigma_minus", + "S8_1sigma_plus", + "S8_2sigma_minus", + "S8_2sigma_plus", + "S8_MAP", + "S8_MAP_2D", + "OMEGA_M_mean", + "OMEGA_M_1sigma_minus", + "OMEGA_M_1sigma_plus", + "OMEGA_M_2sigma_minus", + "OMEGA_M_2sigma_plus", + "OMEGA_M_MAP", + "OMEGA_M_MAP_2D", + "SIGMA_8_mean", + "SIGMA_8_1sigma_minus", + "SIGMA_8_1sigma_plus", + "SIGMA_8_2sigma_minus", + "SIGMA_8_2sigma_plus", + "SIGMA_8_MAP", + "a_mean", + "a_1sigma_minus", + "a_1sigma_plus", + "a_2sigma_minus", + "a_2sigma_plus", + "a_MAP", + ] +) + +for i, root in enumerate(tqdm(roots[:-1])): + param_values_harm = extract_param_chain(chain_harmonic[i], param_names) + + param_harm = concatenate_param_stats(root, param_values_harm, verbose=False) + + output_mocks_harm = np.vstack((output_mocks_harm, param_harm)) + + param_values_config = extract_param_chain(chain_config[i], param_names) + + param_config = concatenate_param_stats(root, param_values_config, verbose=False) + + output_mocks_config = np.vstack((output_mocks_config, param_config)) + +np.savetxt( + f"summary_parameter_constraints_harmonic_space_{chain_version}.txt", + output_mocks_harm, + fmt="%s", + delimiter=";", +) +np.savetxt( + f"summary_parameter_constraints_configuration_space_{chain_version}.txt", + output_mocks_config, + fmt="%s", + delimiter=";", +) +print( + f"Saved summary of parameter constraints for harmonic space in summary_parameter_constraints_harmonic_space_{chain_version}.txt" +) +print( + f"Saved summary of parameter constraints for configuration space in summary_parameter_constraints_configuration_space_{chain_version}.txt" +) + + +import pandas as pd + +output_df_harm = pd.read_csv( + f"summary_parameter_constraints_harmonic_space_{chain_version}.txt", + delimiter=";", + skiprows=1, + names=output_mocks_harm[0], +) + +output_df_config = pd.read_csv( + f"summary_parameter_constraints_configuration_space_{chain_version}.txt", + delimiter=";", + skiprows=1, + names=output_mocks_config[0], +) + + +# Define the true value of the parameters +from astropy.cosmology import Planck18 as planck + +Omega_m_fid = planck.Om0 +sigma_8_fid = 0.8102 +s8_fid = sigma_8_fid * (Omega_m_fid / 0.3) ** 0.5 +h = planck.h +Omega_b_fig = planck.Ob0 +n_s_fid = 0.9665 +print( + f"Fiducial values: Omega_m = {Omega_m_fid}, sigma_8 = {sigma_8_fid}, S_8 = {s8_fid}" +) + + +sns.histplot( + output_df_harm["S8_mean"] - output_df_config["S8_mean"], + kde=True, + bins=30, + label="Mean", +) +# sns.histplot( +# output_df_harm["S8_MAP"]-output_df_config["S8_MAP"], +# kde=True, +# bins=20, +# label="MAP", +# ) +sns.histplot( + output_df_harm["S8_MAP_2D"] - output_df_config["S8_MAP_2D"], + kde=True, + bins=30, + label="2D Mode", + alpha=0.5, +) +plt.axvline(0, color="black", linestyle="--") +plt.legend(fontsize=12) + +plt.xlabel(r"$\Delta S_8$") +plt.savefig( + "./../../results/S8_comparison_harmonic_vs_configuration.pdf", + bbox_inches="tight", +) + + +output_df_config["S8_MAP_2D"].shape +output_df_harm["S8_MAP_2D"].shape + + +# Create JointGrid +g = sns.JointGrid( + x=output_df_config["OMEGA_M_MAP_2D"], + y=output_df_config["S8_MAP_2D"], + height=7, + ratio=5, + space=0, +) + +# Main 2D histogram +sns.histplot( + x=output_df_config["OMEGA_M_MAP_2D"], + y=output_df_config["S8_MAP_2D"], + bins=25, + cmap="Greens", + cbar=False, + ax=g.ax_joint, +) + +# Marginal histograms +sns.histplot( + x=output_df_config["OMEGA_M_MAP_2D"], bins=25, color="#2ca25f", ax=g.ax_marg_x +) +sns.histplot(y=output_df_config["S8_MAP_2D"], bins=25, color="#2ca25f", ax=g.ax_marg_y) + +# Add dashed reference lines +g.ax_joint.axvline(Omega_m_fid, color="k", linestyle="--") +g.ax_joint.axhline(s8_fid, color="k", linestyle="--") + +# Labels +g.set_axis_labels( + r"$\Omega_m$ estimated from mocks (Configuration space)", + r"$S_8$ estimated from mocks (Configuration space)", +) + +# Optional styling tweaks +g.ax_joint.tick_params(labelsize=12) +plt.savefig( + "./../../results/S8_vs_OmegaM_configuration_space_mocks.pdf", + bbox_inches="tight", +) diff --git a/papers/realspace/nonlin_k_analysis.py b/papers/realspace/nonlin_k_analysis.py new file mode 100644 index 00000000..a44002a3 --- /dev/null +++ b/papers/realspace/nonlin_k_analysis.py @@ -0,0 +1,104 @@ +# # Nonlinear $k$ contributions +# +# This notebook plots the 2D heatmap of ratio of scale contributions to the $\xi_\pm$ 2PCF given angular scale $\theta$ and wavenumber $k$. + + +import matplotlib.pylab as plt +import numpy as np +import seaborn as sns + +plt.style.use("/home/guerrini/matplotlib_config/paper.mplstyle") + +plt.rcParams["text.usetex"] = True + +plt.rcParams.update( + { + "font.size": 20, + "axes.titlesize": 21, + "axes.labelsize": 20, + "xtick.labelsize": 20, + "ytick.labelsize": 20, + "legend.fontsize": 20, + "figure.titlesize": 21, + } +) +sns.set_palette("husl") + +blind = "B" +ver = "v1.4.6.3" + + +data_dir = "/n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/data/" + +# Read the 2D array from the text file + +file_headers = ["xip_%s_%s" % (ver, blind), "xim_%s_%s" % (ver, blind)] + +for f in file_headers: + xis = np.loadtxt(data_dir + f"theta_k_{f}.txt") + xis_reshaped = xis.reshape(-1, 201) + sorted_xis = xis_reshaped[np.argsort(xis_reshaped[:, 0])] + + np.savetxt(data_dir + f"theta_k_{f}_sorted.txt", sorted_xis) + + +fig, axs = plt.subplots(2, 1, figsize=(8, 10)) + +# --- k grid --- +h = 0.6766 +k_plot = np.logspace(-4, 2, 200) + +file_header = "%s_%s" % (ver, blind) + +xi_thetas = np.loadtxt(data_dir + f"theta_k_xip_{file_header}_sorted.txt") +thetas = xi_thetas[:, 0] +xis = xi_thetas[:, 1:] + +# normalise +xi_plot = xis / np.max(xis, axis=1, keepdims=True) + +T, K = np.meshgrid(thetas, k_plot) + +axs[0].contour(T, K, xi_plot.T, levels=[0.9], colors="red", linewidths=1.7) +pcm = axs[0].pcolormesh(T, K, xi_plot.T, shading="auto", cmap="viridis") +pcm.set_rasterized(True) + +axs[0].axvline(5, color="k", ls="dashed", lw=1.2) +axs[0].axvline(12, color="white", ls="dashed", lw=1.6) +axs[0].axhline(1, color="k", ls="dashed", lw=1.2) # converted to h/Mpc space if needed +axs[0].axhline(0.425, color="white", ls="dashed", lw=1.6) + +axs[0].set_yscale("log") +axs[0].set_xlabel(r"$\theta\ \mathrm{(arcmin)}$") +axs[0].set_ylabel(r"$k\ (h$ Mpc$^{-1})$") + +axs[0].set_title(r"$\xi_+$") + +xi_thetas = np.loadtxt(data_dir + f"theta_k_xim_{file_header}_sorted.txt") +thetas = xi_thetas[:, 0] +xis = xi_thetas[:, 1:] + +xi_plot = xis / np.max(xis, axis=1, keepdims=True) + +T, K = np.meshgrid(thetas, k_plot) + +axs[1].contour(T, K, xi_plot.T, levels=[0.9], colors="red", linewidths=1.7) +pcm = axs[1].pcolormesh(T, K, xi_plot.T, shading="nearest", cmap="viridis") +pcm.set_rasterized(True) + +axs[1].axvline(12, color="white", ls="dashed", lw=1.6) +axs[1].axhline(2.85, color="white", ls="dashed", lw=1.6) + + +axs[1].set_yscale("log") +axs[1].set_xlabel(r"$\theta\ \mathrm{(arcmin)}$") +axs[1].set_ylabel(r"$k\ (h$ Mpc$^{-1})$") +axs[1].set_title(r"$\xi_-$") + + +fig.tight_layout() + +cbar_ax = fig.add_axes([0.99, 0.15, 0.02, 0.7]) +cbar = fig.colorbar(pcm, cax=cbar_ax) + +fig.savefig("./../../results/theta_k_xip_xim_{ver}_{blind}.pdf", bbox_inches="tight") diff --git a/scripts/compute_m_bias_image_sims.py b/scripts/compute_m_bias_image_sims.py new file mode 100644 index 00000000..04ca49b7 --- /dev/null +++ b/scripts/compute_m_bias_image_sims.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python +"""Compute multiplicative and additive shear bias from image simulations. + +Usage: + compute_m_bias_image_sims.py -c config.yaml [-v] [--cumulative] [--n_tiles N] +""" + +import argparse +import os +import sys + +# Configure matplotlib for non-interactive backend +import matplotlib +import numpy as np +import yaml + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt + +from sp_validation.image_sims import ImageSimMBias + + +def to_python(obj): + """Recursively cast numpy scalars/arrays to plain Python types. + + ``ImageSimMBias.run`` returns a nested dict of numpy floats; dumping those + to YAML with ``yaml.dump`` writes opaque ``!!python/object`` binary tags. A + recursive pass down the results tree (dicts, lists, arrays, scalars) leaves + a clean, human-readable, ``safe_load``-able document. + """ + if isinstance(obj, dict): + return {key: to_python(val) for key, val in obj.items()} + if isinstance(obj, (list, tuple)): + return [to_python(val) for val in obj] + if isinstance(obj, np.ndarray): + return float(obj.item()) if obj.size == 1 else obj.tolist() + if isinstance(obj, (np.integer, np.floating)): + return float(obj) + return obj + + +def parse_args(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("-c", "--config", required=True, help="config YAML file") + p.add_argument("-v", "--verbose", action="store_true", help="verbose output") + p.add_argument( + "--cumulative", + action="store_true", + default=True, + help="track convergence as tiles accumulate (default: True)", + ) + p.add_argument( + "--n_tiles", type=int, help="number of tiles (auto-detected if not given)" + ) + return p.parse_args() + + +def get_n_tiles(grids_dir, num): + """Detect number of tiles from final_cat HDF5 files.""" + try: + import h5py + + # Count tiles in first sim's final_cat + for sim in ["1z2z_grid", "1m2z_grid", "1p2z_grid", "1z2m_grid", "1z2p_grid"]: + sim_name = f"{sim}_{num}" + final_cat = os.path.join(grids_dir, sim_name, f"final_cat_{sim_name}.hdf5") + if os.path.isfile(final_cat): + with h5py.File(final_cat, "r") as hf: + if "patches" in hf: + n_tiles = sum( + 1 for patch in hf["patches"] for _ in hf[f"patches/{patch}"] + ) + return n_tiles + except Exception: + pass + return None + + +def update_cumulative_file(cumulative_path, n_tiles, results): + """Update the cumulative m/c bias tracking file. + + Writes ``results`` under the ``n_tiles`` key, *overwriting* an existing + entry for that count. The earlier behaviour silently skipped when the key + was already present, which meant a re-run against fixed catalogues left the + old (possibly wrong) number in place -- a stale value masquerading as + current. A fresh run is the authority for its tile count, so it overwrites. + + Returns ``True`` when a new key was added, ``False`` when an existing entry + was overwritten (the file is written either way). + """ + if os.path.isfile(cumulative_path): + with open(cumulative_path) as f: + try: + cumulative = yaml.safe_load(f) or {} + except yaml.YAMLError: + # Legacy file written before the to_python cleanup: it carries + # numpy python-object tags that safe_load rejects. Load it + # unsafely, then the to_python pass on write heals it in place. + f.seek(0) + cumulative = yaml.unsafe_load(f) or {} + else: + cumulative = {} + + is_new = str(n_tiles) not in cumulative + cumulative[str(n_tiles)] = results + with open(cumulative_path, "w") as f: + yaml.dump(to_python(cumulative), f, default_flow_style=False) + return is_new + + +def plot_convergence(cumulative_path, diagnostics_dir): + """Create convergence plots: m/c vs n_tiles and errors vs n_tiles.""" + os.makedirs(diagnostics_dir, exist_ok=True) + + try: + with open(cumulative_path) as f: + cumulative = yaml.safe_load(f) + except Exception as e: + print(f"Warning: could not read cumulative file {cumulative_path}: {e}") + return + + if not cumulative: + print("No cumulative data yet, skipping plots") + return + + # Sort by n_tiles + n_tiles_list = sorted([int(k) for k in cumulative.keys()]) + m1_vals = [] + m1_err_vals = [] + c1_vals = [] + c1_err_vals = [] + m2_vals = [] + m2_err_vals = [] + c2_vals = [] + c2_err_vals = [] + + for n in n_tiles_list: + res = cumulative[str(n)] + m1_vals.append(res["m1"]) + m1_err_vals.append(res["m1_err"]) + c1_vals.append(res["c1"]) + c1_err_vals.append(res["c1_err"]) + m2_vals.append(res["m2"]) + m2_err_vals.append(res["m2_err"]) + c2_vals.append(res["c2"]) + c2_err_vals.append(res["c2_err"]) + + n_tiles_str = ( + f"n_tiles = {n_tiles_list}" + if len(n_tiles_list) > 1 + else f"n_tiles = {n_tiles_list[0]}" + ) + + # Plot 1: m and c with error bars + fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5)) + fig.suptitle(f"m and c convergence ({n_tiles_str})", fontsize=12) + + ax1.errorbar( + n_tiles_list, m1_vals, yerr=m1_err_vals, fmt="o-", label="m1", capsize=5 + ) + ax1.errorbar( + n_tiles_list, m2_vals, yerr=m2_err_vals, fmt="s-", label="m2", capsize=5 + ) + ax1.axhline(0, color="k", linestyle="--", alpha=0.3) + ax1.set_xlabel("Number of tiles") + ax1.set_ylabel("Multiplicative bias m") + ax1.legend() + ax1.grid(True, alpha=0.3) + + ax2.errorbar( + n_tiles_list, c1_vals, yerr=c1_err_vals, fmt="o-", label="c1", capsize=5 + ) + ax2.errorbar( + n_tiles_list, c2_vals, yerr=c2_err_vals, fmt="s-", label="c2", capsize=5 + ) + ax2.axhline(0, color="k", linestyle="--", alpha=0.3) + ax2.set_xlabel("Number of tiles") + ax2.set_ylabel("Additive bias c") + ax2.legend() + ax2.grid(True, alpha=0.3) + + plt.tight_layout() + plot1_path = os.path.join(diagnostics_dir, "mbias_convergence.png") + plt.savefig(plot1_path, dpi=150) + plt.close() + print(f"Saved convergence plot to {plot1_path}") + + # Plot 2: error bars only + fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5)) + fig.suptitle(f"Error convergence ({n_tiles_str})", fontsize=12) + + ax1.errorbar( + n_tiles_list, + [0] * len(n_tiles_list), + yerr=m1_err_vals, + fmt="o-", + label="m1 error", + capsize=5, + alpha=0.7, + ) + ax1.errorbar( + n_tiles_list, + [0] * len(n_tiles_list), + yerr=m2_err_vals, + fmt="s-", + label="m2 error", + capsize=5, + alpha=0.7, + ) + ax1.set_xlabel("Number of tiles") + ax1.set_ylabel("Multiplicative bias error") + ax1.legend() + ax1.grid(True, alpha=0.3) + ax1.set_ylim(bottom=0) + + ax2.errorbar( + n_tiles_list, + [0] * len(n_tiles_list), + yerr=c1_err_vals, + fmt="o-", + label="c1 error", + capsize=5, + alpha=0.7, + ) + ax2.errorbar( + n_tiles_list, + [0] * len(n_tiles_list), + yerr=c2_err_vals, + fmt="s-", + label="c2 error", + capsize=5, + alpha=0.7, + ) + ax2.set_xlabel("Number of tiles") + ax2.set_ylabel("Additive bias error") + ax2.legend() + ax2.grid(True, alpha=0.3) + ax2.set_ylim(bottom=0) + + plt.tight_layout() + plot2_path = os.path.join(diagnostics_dir, "mbias_errors.png") + plt.savefig(plot2_path, dpi=150) + plt.close() + print(f"Saved errors plot to {plot2_path}") + + +def main(): + args = parse_args() + + with open(args.config) as f: + config = yaml.safe_load(f) + + print(f"Config: {args.config}") + print(f"Grids : {config['grids_dir']}") + print(f"Run : grid_{config['num']}") + print(f"g_in : ±{config['shear_amplitude']}") + print() + + # Auto-detect n_tiles if --cumulative + if args.cumulative and not args.n_tiles: + n_tiles = get_n_tiles(config["grids_dir"], config["num"]) + if n_tiles: + args.n_tiles = n_tiles + print(f"Auto-detected {n_tiles} tiles") + + mb = ImageSimMBias(config) + + print("Loading catalogues...") + mb.load_catalogs(verbose=args.verbose) + + # ``run`` returns a document with the primary scheme's m/c mirrored at the + # top level plus a per-scheme ``weights`` block. Cast the whole tree to + # plain Python floats so the YAML/text output is human-readable (raw numpy + # scalars serialise as !!python/object binary). + results = to_python(mb.run(verbose=True)) + + print() + print("=" * 40) + print(" Results") + print("=" * 40) + for scheme, res in results["weights"].items(): + print(f" weights: {scheme}") + print(f" m1 = {res['m1']:+.4f} +-{res['m1_err']:.4f}") + print(f" c1 = {res['c1']:+.4f} +-{res['c1_err']:.4f}") + print(f" m2 = {res['m2']:+.4f} +-{res['m2_err']:.4f}") + print(f" c2 = {res['c2']:+.4f} +-{res['c2_err']:.4f}") + print("=" * 40) + + # Cumulative tracking + if args.cumulative: + results_dir = config.get( + "diagnostics_dir", config.get("results_dir", "results") + ) + os.makedirs(results_dir, exist_ok=True) + else: + results_dir = None + + # Output path: in results dir if cumulative, else from config or current dir + if results_dir: + out_path = os.path.join(results_dir, "m_bias_results.yaml") + else: + out_path = config.get("output_path", "m_bias_results.yaml") + + # A result file describes itself: the provenance block the rule assembled + # (manifest hash, both repos' branch+commit, container sif + GHCR revision) + # rides verbatim from the config into the output yaml. It is appended as a + # separate top-level key, so the numeric m/c fields serialise byte-for-byte + # as before -- the reproduction gate sees only the added `provenance:` block. + output = dict(results) + if "provenance" in config: + output["provenance"] = config["provenance"] + + with open(out_path, "w") as f: + yaml.dump(output, f, default_flow_style=False) + print(f"Results written to {out_path}") + + # Also write to text file for readability + if results_dir: + txt_path = os.path.join(results_dir, "m_bias_results.txt") + else: + txt_path = config.get("output_path", "m_bias_results.yaml").replace( + ".yaml", ".txt" + ) + + with open(txt_path, "w") as f: + f.write("Multiplicative and additive shear bias from image simulations\n") + f.write("=" * 60 + "\n") + for scheme, res in results["weights"].items(): + f.write(f"\nweights: {scheme}\n") + f.write(f" m1 = {res['m1']:+.6f} ± {res['m1_err']:.6f}\n") + f.write(f" c1 = {res['c1']:+.6f} ± {res['c1_err']:.6f}\n") + f.write(f" m2 = {res['m2']:+.6f} ± {res['m2_err']:.6f}\n") + f.write(f" c2 = {res['c2']:+.6f} ± {res['c2_err']:.6f}\n") + f.write( + "\nErrors computed via bootstrap resampling " + f"(n={config['n_bootstrap']} resamples)\n" + ) + print(f"Results written to {txt_path}") + + if args.cumulative: + cumulative_path = os.path.join(results_dir, "mbias_cumulative.yaml") + if args.n_tiles: + added = update_cumulative_file(cumulative_path, args.n_tiles, results) + verb = "Added" if added else "Overwrote" + print(f"\n{verb} n_tiles={args.n_tiles} in {cumulative_path}") + # Regenerate plots after every update (an overwrite can shift the + # curve, so the plots must track it -- not just fresh additions). + try: + plot_convergence(cumulative_path, results_dir) + except Exception as e: + print( + f"Warning: could not generate convergence plots: {e}", + file=sys.stderr, + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/diagnostics_image_sims.py b/scripts/diagnostics_image_sims.py new file mode 100644 index 00000000..699d5c49 --- /dev/null +++ b/scripts/diagnostics_image_sims.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python +"""Per-sim diagnostics for image simulation catalogues. + +For each requested grid catalogue, produces: + - footprint (RA/Dec scatter) + - ellipticity histograms (e1, e2) + - weight histogram + - response matrix element histograms (R_g11, R_g22, R_g12, R_g21) + - PSF leakage scatter (e1 vs e1_PSF, e2 vs e2_PSF) + - additive bias (weighted mean e1, e2) + +Shares the estimator's config schema (``sp_validation.image_sims``): the same +``grids_dir`` / ``num`` / ``catalog_name`` keys, the ``branches`` list (the sim +map, not a hard-coded five), and the same ``w_col`` weight semantics -- a column +name, or ``null`` for unit weights. Reading ``w_col`` (rather than hard-coding +``w_des``) means the diagnostics never KeyError on a catalogue that lacks the +weight column, and they weight exactly as the m-bias run they accompany. + +Usage: + diagnostics_image_sims.py -c config.yaml [-v] +""" + +import argparse +import os +import sys + +import matplotlib +import numpy as np +import yaml + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from astropy.io import fits + +# Conventional campaign layout, used only when the config carries no branch map +# -- the same fallback the estimator uses. +_DEFAULT_BRANCHES = ["1z2z", "1m2z", "1p2z", "1z2m", "1z2p"] + + +def load(path): + with fits.open(path) as hdul: + return {col.name: hdul[1].data[col.name].copy() for col in hdul[1].columns} + + +def weights(cat, w_col): + """Per-object weights: the ``w_col`` column, or unit weights when null. + + Mirrors the estimator's ``w_col`` contract (image_sims._load_cat): ``None`` + -> every object unit weight (the no-weighting mode, #227). Reading it here + means the diagnostics never KeyError when the weight column is absent. + """ + return cat[w_col].copy() if w_col else np.ones(len(cat["RA"])) + + +def parse_args(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("-c", "--config", required=True) + p.add_argument("-v", "--verbose", action="store_true") + return p.parse_args() + + +def savefig(fig, out_dir, name): + path = f"{out_dir}/{name}.png" + fig.savefig(path, dpi=150, bbox_inches="tight") + plt.close(fig) + return path + + +def plot_footprints(cats, colors, out_dir): + fig, ax = plt.subplots(figsize=(8, 6)) + for name, d in cats.items(): + ax.scatter(d["RA"], d["Dec"], s=1, alpha=0.4, label=name, color=colors[name]) + ax.set_xlabel("RA [deg]") + ax.set_ylabel("Dec [deg]") + ax.legend(markerscale=5) + ax.set_title("Footprint") + return savefig(fig, out_dir, "footprint") + + +def plot_ellipticity(cats, colors, w_col, out_dir, nbins=100): + fig, axs = plt.subplots(1, 2, figsize=(14, 5)) + bins = np.linspace(-1.0, 1.0, nbins + 1) + for name, d in cats.items(): + w = weights(d, w_col) + for ax, col, label in zip(axs, ["e1", "e2"], [r"$e_1$", r"$e_2$"]): + ax.hist( + d[col], + bins=bins, + density=True, + weights=w, + histtype="step", + label=name, + color=colors[name], + ) + for ax, label in zip(axs, [r"$e_1$", r"$e_2$"]): + ax.set_xlabel(label) + ax.set_ylabel("normalised count") + ax.legend(fontsize=7) + wlabel = w_col if w_col else "unit" + fig.suptitle(f"Ellipticity histograms ({wlabel} weighted)") + return savefig(fig, out_dir, "ellipticity_hist") + + +def plot_weights(cats, colors, w_col, out_dir, nbins=50): + fig, ax = plt.subplots(figsize=(8, 5)) + for name, d in cats.items(): + ax.hist( + weights(d, w_col), + bins=nbins, + density=True, + histtype="step", + label=name, + color=colors[name], + ) + wlabel = w_col if w_col else "unit" + ax.set_xlabel(wlabel) + ax.set_ylabel("normalised count") + ax.legend() + ax.set_title("Weight distribution") + return savefig(fig, out_dir, "weight_hist") + + +def plot_response(cats, colors, out_dir, nbins=50): + cols = ["R_g11", "R_g22", "R_g12", "R_g21"] + fig, axs = plt.subplots(2, 2, figsize=(12, 10)) + for ax, col in zip(axs.flat, cols): + for name, d in cats.items(): + ax.hist( + d[col], + bins=nbins, + range=(-1, 2), + density=True, + histtype="step", + label=name, + color=colors[name], + ) + ax.set_xlim(-1, 2) + ax.set_xlabel(col) + ax.set_ylabel("normalised count") + ax.legend(fontsize=7) + fig.suptitle("Response matrix elements") + fig.tight_layout() + return savefig(fig, out_dir, "response_hist") + + +def plot_psf_leakage(cats, colors, out_dir): + fig, axs = plt.subplots(1, 2, figsize=(14, 5)) + for name, d in cats.items(): + for ax, eg, ep, label in zip( + axs, + ["e1", "e2"], + ["e1_PSF", "e2_PSF"], + [r"$e_1$", r"$e_2$"], + ): + ax.scatter(d[ep], d[eg], s=1, alpha=0.3, label=name, color=colors[name]) + for ax, xlab, ylab in zip( + axs, [r"$e_1^{\rm PSF}$", r"$e_2^{\rm PSF}$"], [r"$e_1$", r"$e_2$"] + ): + ax.set_xlabel(xlab) + ax.set_ylabel(ylab) + ax.legend(markerscale=5, fontsize=7) + fig.suptitle("Object-wise PSF leakage") + return savefig(fig, out_dir, "psf_leakage") + + +def calculate_additive_bias(cats, w_col, verbose=True): + print("\n--- Additive bias (weighted mean ellipticity) ---") + results = {} + for name, d in cats.items(): + w = weights(d, w_col) + c1 = np.average(d["e1"], weights=w) + c2 = np.average(d["e2"], weights=w) + results[name] = (c1, c2) + if verbose: + print(f" {name}: c1 = {c1:+.5f} c2 = {c2:+.5f}") + return results + + +def main(): + args = parse_args() + with open(args.config) as f: + config = yaml.safe_load(f) + + # Same config schema as the estimator: grids_dir (not base), num, + # catalog_name, the branch map, and the w_col weight contract. + grids_dir = config["grids_dir"] + num = config["num"] + cat_name = config.get("catalog_name", "shape_catalog_cut_ngmix.fits") + branches = list(config.get("branches", _DEFAULT_BRANCHES)) + w_col = config["w_col"] # required, like the estimator; null -> unit weights + out_dir = config.get("diagnostics_dir", f"{grids_dir}/diagnostics") + + # Colour per branch from a palette, so any branch list plots (no hard-coded + # five-branch colour map). + palette = plt.get_cmap("tab10") + colors = {name: palette(i % 10) for i, name in enumerate(branches)} + + os.makedirs(out_dir, exist_ok=True) + + print(f"Loading catalogues from {grids_dir}...") + cats = {} + for name in branches: + path = f"{grids_dir}/{name}_grid_{num}/{cat_name}" + if not os.path.exists(path): + print(f" WARNING: {path} not found, skipping") + continue + cats[name] = load(path) + if args.verbose: + print(f" {name}: {len(cats[name]['RA'])} objects") + + if not cats: + print("No catalogues found, exiting.") + return 1 + + print(f"\nSaving plots to {out_dir}/") + print(f" footprint -> {plot_footprints(cats, colors, out_dir)}") + print(f" ellipticity -> {plot_ellipticity(cats, colors, w_col, out_dir)}") + print(f" weights -> {plot_weights(cats, colors, w_col, out_dir)}") + print(f" response -> {plot_response(cats, colors, out_dir)}") + print(f" PSF leakage -> {plot_psf_leakage(cats, colors, out_dir)}") + calculate_additive_bias(cats, w_col, verbose=True) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/sp_validation/image_sims.py b/src/sp_validation/image_sims.py new file mode 100644 index 00000000..e259109d --- /dev/null +++ b/src/sp_validation/image_sims.py @@ -0,0 +1,324 @@ +"""IMAGE_SIMS. + +:Description: Multiplicative and additive shear bias from image simulations. + +:Author: Martin Kilbinger + +""" + +import numpy as np +from astropy.io import fits + +from sp_validation.catalog import match_catalogs_radec + +# Conventional campaign layout, used only when the config carries no branch map +# (e.g. the synthetic-recovery tests). In a workflow run the branches and pairs +# come from manifest.yaml via the m_bias config; nothing about the injected +# shear is hard-coded on the estimator's side. +_DEFAULT_BRANCHES = ["1z2z", "1p2z", "1m2z", "1z2p", "1z2m"] +_DEFAULT_PAIRS = [ + ("1p2z", "1m2z", 0), # g1 component, index 0 → e1 + ("1z2p", "1z2m", 1), # g2 component, index 1 → e2 +] + + +# Weight-scheme name that means "no weighting": every object gets unit weight. +# ``None`` (from a YAML ``null``) is accepted as an alias, so the fiducial +# unweighted primary scheme can be written either ``none`` or ``null``. +_UNWEIGHTED = "none" + + +def _is_unweighted(scheme): + """True for the unit-weight scheme (``"none"`` or ``None``).""" + return scheme is None or scheme == _UNWEIGHTED + + +def _load_cat(path, w_cols): + """Load RA, Dec, ellipticities and per-scheme weights from a FITS catalogue. + + Reads the ``e1``/``e2`` columns, which the calibration stage writes as the + *calibrated* shear estimate ``g = R^-1 g_uncal - c`` (metacal response and + additive-bias corrected) -- not the raw ``e1_uncal``/``e2_uncal`` columns + that sit alongside them in the same catalogue. The bias this estimator + measures is therefore the *residual* m/c left after the chain's own metacal + calibration, not the raw pre-calibration bias. + + ``w_cols`` is the list of weight schemes to load. The scheme ``"none"`` + (equivalently a ``None``/``null`` entry) gives every object unit weight -- + the no-weighting mode for m-bias runs (#227: shape weights are excluded from + sim calibration); any other entry is read as a FITS column name. The weights + come back as a dict keyed by scheme so one catalogue load serves every + scheme in a multi-weight run. + """ + with fits.open(path) as hdul: + data = hdul[1].data + cat = { + "ra": data["RA"].copy(), + "dec": data["Dec"].copy(), + "e1": data["e1"].copy(), + "e2": data["e2"].copy(), + "w": {}, + } + for scheme in w_cols: + cat["w"][scheme] = ( + np.ones(len(cat["ra"])) + if _is_unweighted(scheme) + else data[scheme].copy() + ) + return cat + + +class ImageSimMBias: + """Compute multiplicative and additive shear bias from image simulations. + + The estimator consumes the *calibrated* ``e1``/``e2`` columns (the metacal + response- and additive-bias-corrected shear ``g = R^-1 g_uncal - c``), so + the headline m/c is the **residual** bias remaining after the chain's own + metacal calibration, not the raw pre-calibration bias. + + Parameters + ---------- + config : dict + Configuration dictionary with keys: + - grids_dir : str, path to the grids directory + - num : int, run number (e.g. 2 for *_grid_2) + - catalog_name : str, filename of the cut catalogue + (default 'shape_catalog_cut_ngmix.fits') + - shear_amplitude : float, input shear |g| (from manifest.yaml) + - branches : list of str, branch names in load order (incl. the + unsheared reference); defaults to the conventional 5-branch layout + - pairs : list of dicts {plus, minus, component}, the +/- sheared + branch pairing per component; defaults to the conventional pairs + - match_radius_deg : float, matching radius in degrees (required) + - pair_match : bool, match objects between the +g and -g sheared + catalogues (required); if False, use all objects of each + catalogue (the paired per-object cancellation is then unavailable) + - w_cols : list of str, weight schemes to compute in one run + (required); ``"none"`` (or a ``null`` entry) means unit weights, + any other entry is a FITS column name. The **first** entry is the + primary result surfaced at the top level of ``run()``'s output. Our + fiducial run leads with the unweighted scheme (``["none", ...]``), + per the #227 verdict that shape weights are excluded from sim + calibration; the unweighted m also avoids the ``cov(w, e)`` residual + weighted estimators carry on constant-shear sims. + - w_col : str or None, *deprecated* single weight scheme; accepted for + back-compat and used as ``[w_col]`` only when ``w_cols`` is absent. + - n_bootstrap : int, number of bootstrap resamples for errors (required) + - bootstrap_seed : int, seed for the per-pair bootstrap RNG (required); + makes the bootstrap errors bit-reproducible. The resample indices are + drawn once per pair and shared across every weight scheme, so the + schemes differ only in their weighting, never in their draws. + + The science knobs (``match_radius_deg``, ``pair_match``, ``w_cols``, + ``n_bootstrap``, ``bootstrap_seed``) are read with no in-code default: a + missing one is a config bug and raises ``KeyError`` at construction, per the + fail-fast contract (the workflow emits every one into the m_bias config). + The lone exception is the deprecated ``w_col``, which is honoured as a + fallback so pre-``w_cols`` configs still run. + """ + + def __init__(self, config): + self.cfg = config + self.g_in = config["shear_amplitude"] + self.thresh = config["match_radius_deg"] + self.pair_match = config["pair_match"] + # ``w_cols`` is the required science key. A pre-``w_cols`` config that + # still carries the deprecated scalar ``w_col`` is honoured as a + # single-scheme run; only a config with neither raises (fail-fast). + if "w_cols" in config: + w_cols = config["w_cols"] + else: + w_cols = [config["w_col"]] + # Normalise a ``None``/``null`` entry to the canonical "none" name so + # results key off a string; downstream still treats it as unit weights. + self.w_cols = [_UNWEIGHTED if _is_unweighted(w) else str(w) for w in w_cols] + self.n_boot = config["n_bootstrap"] + self.boot_seed = config["bootstrap_seed"] + # Branch list and pairing come from the manifest-derived config + # (``branches`` / ``pairs``); fall back to the conventional layout only + # when neither is given. ``branches`` fixes the catalogue load order; + # ``pairs`` fixes which sims difference into which component. + self.sim_names = list(config.get("branches", _DEFAULT_BRANCHES)) + if config.get("pairs"): + self.pairs = [ + (p["plus"], p["minus"], p["component"]) for p in config["pairs"] + ] + else: + self.pairs = list(_DEFAULT_PAIRS) + self.cats = {} + + def load_catalogs(self, verbose=True): + """Load the 5 sheared and reference catalogues.""" + grids_dir = self.cfg["grids_dir"] + num = self.cfg["num"] + cat_name = self.cfg.get("catalog_name", "shape_catalog_cut_ngmix.fits") + # ``sim_names`` (incl. the unsheared reference) comes from the config's + # branch map. The +g/-g pool estimator pairs the sheared sims directly; + # the reference is loaded for completeness and null-test diagnostics. + for name in self.sim_names: + path = f"{grids_dir}/{name}_grid_{num}/{cat_name}" + if verbose: + print(f" Loading {path}") + self.cats[name] = _load_cat(path, self.w_cols) + if verbose: + print(f" {len(self.cats[name]['ra'])} objects") + + def print_mean_ellipticities(self): + """Print the mean e1, e2 for each catalogue and weight scheme, as a check. + + The unweighted scheme (``"none"``) gives the plain unweighted means. + """ + for scheme in self.w_cols: + print(f"\nMean ellipticities (all objects, weights: {scheme}):") + for name, cat in self.cats.items(): + mean_e1 = np.average(cat["e1"], weights=cat["w"][scheme]) + mean_e2 = np.average(cat["e2"], weights=cat["w"][scheme]) + print(f" {name}: = {mean_e1:+.5f} = {mean_e2:+.5f}") + + def _m_c_pair(self, name_p, name_m, comp, verbose=True): + """Compute m and c for one shear pair and component (0=g1, 1=g2). + + Paired ("pool") estimator. The +g and -g simulations inject opposite + input shear on the *same* galaxies, so matching them directly by + RA/Dec yields a one-to-one correspondence. Differencing the two + ellipticities per object, + + m = <(e_+ - e_-) / (2 g_in) - 1> , c = <(e_+ + e_-) / 2> , + + cancels the intrinsic shape (sigma_e ~ 0.3) object-by-object in the + multiplicative term, leaving only measurement noise -- so sigma(m) + shrinks by ~sigma_e/sigma_meas relative to differencing two + independent means. (The additive term c is a *sum*, so intrinsic + shape does not cancel there and its error stays shape-noise limited.) + + With ``pair_match=False`` the +g and -g sims are *not* matched: every + object of each catalogue is used, so the per-object cancellation is + lost and m, c fall back to differencing/summing the two independent + weighted means. The paired bootstrap likewise cannot be applied (the + two arrays generally have different lengths), so each side is resampled + independently per replicate. + """ + e_key = f"e{comp + 1}" + + if self.pair_match: + # Match the +g and -g sims to each other: same galaxies, opposite + # shear. This is a nearest-neighbour match within `thresh`, not a + # strict bijection -- on grid sims galaxies are well separated so + # pairs are effectively 1:1 (verified ~99% co-located to <0.05" on + # SKiLLS grid_1); on denser fields a small fraction could share a + # +g partner and dilute the cancellation. + idx_p, idx_m = match_catalogs_radec( + self.cats[name_p]["ra"], + self.cats[name_p]["dec"], + self.cats[name_m]["ra"], + self.cats[name_m]["dec"], + thresh_deg=self.thresh, + ) + if verbose: + print(f" {name_p} <-> {name_m}: {len(idx_p)} paired objects") + else: + idx_p = slice(None) + idx_m = slice(None) + if verbose: + print( + f" no pair-matching: {name_p}: {len(self.cats[name_p][e_key])}" + f" | {name_m}: {len(self.cats[name_m][e_key])} objects" + ) + + e_p = self.cats[name_p][e_key][idx_p] + e_m = self.cats[name_m][e_key][idx_m] + + # Draw the bootstrap resample indices *once*, before the weight-scheme + # loop, and reuse them for every scheme -- the schemes then differ only + # in their weighting, never in their draws (so a scheme comparison is a + # clean weighting comparison). Pre-drawing the full ``(n_boot, n)`` block + # in one call is bit-identical to drawing ``rng.integers(0, n, n)`` once + # per replicate (numpy fills the block row-major), so the numbers match a + # single-scheme, per-iteration bootstrap to the last bit. + rng = np.random.default_rng(seed=self.boot_seed) + if self.pair_match: + n = len(e_p) + ib = rng.integers(0, n, (self.n_boot, n)) + else: + n_p, n_m = len(e_p), len(e_m) + ib_p = rng.integers(0, n_p, (self.n_boot, n_p)) + ib_m = rng.integers(0, n_m, (self.n_boot, n_m)) + + res = {} + for scheme in self.w_cols: + w_p = self.cats[name_p]["w"][scheme][idx_p] + w_m = self.cats[name_m]["w"][scheme][idx_m] + m_boot = np.empty(self.n_boot) + c_boot = np.empty(self.n_boot) + + if self.pair_match: + # Per-object shear-differenced (-> m) and summed (-> c) + # ellipticity, with a symmetric per-pair weight. + w = 0.5 * (w_p + w_m) + d = (e_p - e_m) / (2 * self.g_in) - 1 + s = (e_p + e_m) / 2 + + m = np.average(d, weights=w) + c = np.average(s, weights=w) + + # Paired bootstrap: the same object draw is applied to both + # sims, so the per-object cancellation in `d` is preserved in + # the error estimate. + for i in range(self.n_boot): + m_boot[i] = np.average(d[ib[i]], weights=w[ib[i]]) + c_boot[i] = np.average(s[ib[i]], weights=w[ib[i]]) + else: + # No matching: difference/sum the two independent weighted means. + mean_ep = np.average(e_p, weights=w_p) + mean_em = np.average(e_m, weights=w_m) + + m = (mean_ep - mean_em) / (2 * self.g_in) - 1 + c = (mean_ep + mean_em) / 2 + + # Unpaired bootstrap: the +g and -g arrays generally differ in + # length, so each side is resampled independently per replicate. + for i in range(self.n_boot): + ep_b = np.average(e_p[ib_p[i]], weights=w_p[ib_p[i]]) + em_b = np.average(e_m[ib_m[i]], weights=w_m[ib_m[i]]) + m_boot[i] = (ep_b - em_b) / (2 * self.g_in) - 1 + c_boot[i] = (ep_b + em_b) / 2 + + res[scheme] = (m, np.std(m_boot), c, np.std(c_boot)) + + return res + + def run(self, verbose=True): + """Compute m and c for both shear components and every weight scheme. + + Returns + ------- + dict + ``results["weights"][scheme]`` holds ``m1, m1_err, c1, c1_err, + m2, m2_err, c2, c2_err`` for each weight scheme. The primary + (first) scheme's keys are also mirrored at the top level, so a + reader that wants the headline m/c never has to know the scheme + name. + """ + results = {"weights": {scheme: {} for scheme in self.w_cols}} + for name_p, name_m, comp in self.pairs: + label = f"g{comp + 1}" + if verbose: + print(f"\n--- {label}: {name_p} / {name_m} ---") + res = self._m_c_pair(name_p, name_m, comp, verbose=verbose) + for scheme, (m, m_err, c, c_err) in res.items(): + w = results["weights"][scheme] + w[f"m{comp + 1}"] = m + w[f"m{comp + 1}_err"] = m_err + w[f"c{comp + 1}"] = c + w[f"c{comp + 1}_err"] = c_err + if verbose: + print( + f" [{scheme}] m{comp + 1} = {m:.4f} ± {m_err:.4f}" + f" c{comp + 1} = {c:.4f} ± {c_err:.4f}" + ) + + # Mirror the primary (first) scheme's m/c at the top level: the headline + # result reads out without knowing the scheme name, and a downstream + # gate keyed on the old flat keys still finds them. + results.update(results["weights"][self.w_cols[0]]) + return results diff --git a/src/sp_validation/tests/test_image_sims.py b/src/sp_validation/tests/test_image_sims.py new file mode 100644 index 00000000..ef3e3b14 --- /dev/null +++ b/src/sp_validation/tests/test_image_sims.py @@ -0,0 +1,253 @@ +"""UNIT TESTS FOR THE IMAGE-SIMULATION m/c ESTIMATOR. + +Exercise ``sp_validation.image_sims.ImageSimMBias`` -- the multiplicative and +additive shear-bias estimator used by the image-simulation workflow -- and the +``sp_validation.catalog.match_catalogs_radec`` helper it relies on. + +The estimator recovers ``m`` and ``c`` from five calibrated catalogues named +``1z2z`` (reference, no input shear), ``1p2z``/``1m2z`` (input shear +``g1 = +-|g|``) and ``1z2p``/``1z2m`` (``g2 = +-|g|``). The +g and -g sims are +matched to *each other* by RA/Dec -- same galaxies, opposite input shear -- and +the bias is the object-paired ("pool") average + + m = <(e_+ - e_-) / (2 |g|) - 1> , c = <(e_+ + e_-) / 2> , + +so the intrinsic shape cancels object-by-object in ``m``. + +We build synthetic catalogues in which the measured ellipticity is exactly +``e = (1 + m_true) g_in + c_true`` at shared positions, so the recovered m/c +must equal the injected values to machine precision -- an analytic check of +the estimator maths that needs no pipeline run. + +:Author: cdaley + +""" + +import numpy as np +import numpy.testing as npt +from astropy.io import fits + +from sp_validation.catalog import match_catalogs_radec +from sp_validation.image_sims import ImageSimMBias + +# Injected truth, shared across the synthetic-recovery test. +A = 0.02 # input shear amplitude |g| +M_TRUE = 0.05 # multiplicative bias (same for both components) +C1_TRUE = 0.001 # additive bias, component 1 +C2_TRUE = -0.002 # additive bias, component 2 +N_GAL = 2000 + + +def _write_cat(path, ra, dec, e1, e2, w): + """Write a minimal calibrated shape catalogue (RA, Dec, e1, e2, w_des).""" + cols = [ + fits.Column(name=name, array=arr, format="D") + for name, arr in ( + ("RA", ra), + ("Dec", dec), + ("e1", e1), + ("e2", e2), + ("w_des", w), + ) + ] + fits.HDUList([fits.PrimaryHDU(), fits.BinTableHDU.from_columns(cols)]).writeto( + path, overwrite=True + ) + + +def _make_grid(grids_dir, num): + """Create the five sheared/reference catalogues with a known m/c.""" + rng = np.random.default_rng(0) + ra = 30.0 + rng.uniform(0, 0.1, N_GAL) + dec = rng.uniform(0, 0.1, N_GAL) + w = np.ones(N_GAL) + zero = np.zeros(N_GAL) + + def e(g_in): + return (1 + M_TRUE) * g_in + zero + + sims = { + "1z2z": (C1_TRUE + zero, C2_TRUE + zero), + "1p2z": (e(+A) + C1_TRUE, C2_TRUE + zero), + "1m2z": (e(-A) + C1_TRUE, C2_TRUE + zero), + "1z2p": (C1_TRUE + zero, e(+A) + C2_TRUE), + "1z2m": (C1_TRUE + zero, e(-A) + C2_TRUE), + } + for name, (e1, e2) in sims.items(): + sim_dir = grids_dir / f"{name}_grid_{num}" + sim_dir.mkdir(parents=True, exist_ok=True) + _write_cat(sim_dir / "cat.fits", ra, dec, e1, e2, w) + + +def test_match_catalogs_radec_identity(): + """Identical positions match one-to-one; a shifted object drops out.""" + ra = np.array([30.0, 30.01, 30.02]) + dec = np.array([10.0, 10.01, 10.02]) + # Second catalogue = first, but the last object nudged well past threshold. + ra2, dec2 = ra.copy(), dec.copy() + ra2[2] += 1.0 + idx1, idx2 = match_catalogs_radec(ra, dec, ra2, dec2, thresh_deg=0.0002) + npt.assert_array_equal(idx2, [0, 1]) + npt.assert_array_equal(idx1, [0, 1]) + + +def test_mbias_recovers_injected_values(tmp_path): + """ImageSimMBias recovers the injected m/c to machine precision.""" + num = 7 + _make_grid(tmp_path, num) + config = { + "grids_dir": str(tmp_path), + "num": num, + "catalog_name": "cat.fits", + "shear_amplitude": A, + "match_radius_deg": 0.0002, + "w_cols": ["w_des"], + "n_bootstrap": 50, + "pair_match": True, + "bootstrap_seed": 42, + } + mb = ImageSimMBias(config) + mb.load_catalogs(verbose=False) + res = mb.run(verbose=False) + + npt.assert_allclose(res["m1"], M_TRUE, atol=1e-9) + npt.assert_allclose(res["m2"], M_TRUE, atol=1e-9) + npt.assert_allclose(res["c1"], C1_TRUE, atol=1e-9) + npt.assert_allclose(res["c2"], C2_TRUE, atol=1e-9) + # Bootstrap errors are non-negative and finite. + for key in ("m1_err", "m2_err", "c1_err", "c2_err"): + assert np.isfinite(res[key]) and res[key] >= 0 + # Self-describing results: the primary scheme is mirrored at the top level + # *and* lives under ``weights[scheme]``, and the two agree exactly. + assert list(res["weights"]) == ["w_des"] + for key in ("m1", "m1_err", "c1", "c1_err", "m2", "m2_err", "c2", "c2_err"): + assert res[key] == res["weights"]["w_des"][key] + + +def test_mbias_multiple_weight_schemes_share_draws(tmp_path): + """Multi-scheme runs key results per scheme; the primary mirrors the first. + + ``none`` (unit weights) and a real weight column are computed in one run. + With uniform per-object weights in the synthetic grid the two schemes give + the *same* m/c (the weighting is a no-op), and the shared bootstrap indices + make even the errors identical -- the property that lets a scheme + comparison be a clean weighting comparison. The first entry (``none``) is + the primary result surfaced at the top level. + """ + num = 8 + _make_grid(tmp_path, num) + config = { + "grids_dir": str(tmp_path), + "num": num, + "catalog_name": "cat.fits", + "shear_amplitude": A, + "match_radius_deg": 0.0002, + "w_cols": ["none", "w_des"], + "n_bootstrap": 50, + "pair_match": True, + "bootstrap_seed": 42, + } + mb = ImageSimMBias(config) + mb.load_catalogs(verbose=False) + res = mb.run(verbose=False) + + assert list(res["weights"]) == ["none", "w_des"] + # Primary (first) scheme mirrored at the top level. + for key in ("m1", "m1_err", "c1", "c1_err", "m2", "m2_err", "c2", "c2_err"): + assert res[key] == res["weights"]["none"][key] + # Uniform grid weights make the schemes agree bit-for-bit, errors included + # (shared bootstrap draws). + assert res["weights"]["none"] == res["weights"]["w_des"] + + +def test_mbias_deprecated_w_col_still_runs(tmp_path): + """A pre-``w_cols`` config with the scalar ``w_col`` still runs. + + The deprecated single-scheme key is honoured as ``[w_col]`` when ``w_cols`` + is absent, so a legacy run config keeps working and produces the same + single-scheme result as the ``w_cols=[w_col]`` spelling. + """ + num = 9 + _make_grid(tmp_path, num) + base = { + "grids_dir": str(tmp_path), + "num": num, + "catalog_name": "cat.fits", + "shear_amplitude": A, + "match_radius_deg": 0.0002, + "n_bootstrap": 50, + "pair_match": True, + "bootstrap_seed": 42, + } + res_dep = ImageSimMBias({**base, "w_col": "w_des"}) + res_dep.load_catalogs(verbose=False) + out_dep = res_dep.run(verbose=False) + + res_new = ImageSimMBias({**base, "w_cols": ["w_des"]}) + res_new.load_catalogs(verbose=False) + out_new = res_new.run(verbose=False) + + assert list(out_dep["weights"]) == ["w_des"] + assert out_dep["weights"] == out_new["weights"] + + +def test_mbias_pool_cancels_shape_noise(tmp_path): + """The paired estimator cancels intrinsic shape noise in m. + + With realistic per-galaxy intrinsic ellipticity (sigma_e ~ 0.3) shared + between the +g and -g sims plus small independent measurement noise, the + object-paired difference cancels the intrinsic shape, so sigma(m) is set by + the measurement noise (~1e-2), not the shape noise. An *unpaired* estimator + (differencing two independently-drawn means) would instead return + sigma(m) ~ sigma_e / (2 |g| sqrt(N)) -- an order of magnitude larger. We + assert the recovered error sits well below that shape-noise floor, which is + the property the pooling exists to deliver. + """ + num = 3 + rng = np.random.default_rng(1) + ra = 30.0 + rng.uniform(0, 0.1, N_GAL) + dec = rng.uniform(0, 0.1, N_GAL) + w = np.ones(N_GAL) + sigma_e, sigma_meas = 0.3, 0.01 + e1_int = rng.normal(0, sigma_e, N_GAL) # intrinsic shape, shared across sims + e2_int = rng.normal(0, sigma_e, N_GAL) + + def measured(g1_in, g2_in): + """Measured ellipticity = intrinsic + (1 + m) * input shear + noise.""" + e1 = e1_int + (1 + M_TRUE) * g1_in + rng.normal(0, sigma_meas, N_GAL) + e2 = e2_int + (1 + M_TRUE) * g2_in + rng.normal(0, sigma_meas, N_GAL) + return e1, e2 + + sims = { + "1z2z": measured(0, 0), + "1p2z": measured(+A, 0), + "1m2z": measured(-A, 0), + "1z2p": measured(0, +A), + "1z2m": measured(0, -A), + } + for name, (e1, e2) in sims.items(): + sim_dir = tmp_path / f"{name}_grid_{num}" + sim_dir.mkdir(parents=True, exist_ok=True) + _write_cat(sim_dir / "cat.fits", ra, dec, e1, e2, w) + + config = { + "grids_dir": str(tmp_path), + "num": num, + "catalog_name": "cat.fits", + "shear_amplitude": A, + "match_radius_deg": 0.0002, + "w_cols": ["w_des"], + "n_bootstrap": 200, + "pair_match": True, + "bootstrap_seed": 42, + } + mb = ImageSimMBias(config) + mb.load_catalogs(verbose=False) + res = mb.run(verbose=False) + + shape_noise_floor = sigma_e / (2 * A * np.sqrt(N_GAL)) # the unpaired error + for comp in (1, 2): + # m recovered within a few sigma of truth... + assert abs(res[f"m{comp}"] - M_TRUE) < 5 * res[f"m{comp}_err"] + # ...and its error is far below what an unpaired estimator would give. + assert res[f"m{comp}_err"] < 0.1 * shape_noise_floor diff --git a/src/sp_validation/tests/test_mask_overlay.py b/src/sp_validation/tests/test_mask_overlay.py new file mode 100644 index 00000000..26244053 --- /dev/null +++ b/src/sp_validation/tests/test_mask_overlay.py @@ -0,0 +1,85 @@ +"""The image-sim mask config is a declared overlay on the data mask config. + +The image-sim calibration does not keep an independent copy of the mask / +calibration config: it keeps the *data* config (``mask_v1.X.9.yaml``) as the one +home for the shared cuts, and declares the sim-specific delta in an overlay +(``mask_v1.X.9_im_sim.overlay.yaml``). ``im_compose_mask.py`` applies the +overlay to the base and must reproduce the committed runtime file +(``mask_v1.X.9_im_sim.yaml``) **byte-for-byte**. + +This guard locks that equality, so the two artefacts cannot drift: + +* if someone edits the runtime file without updating the overlay (or vice + versa), :func:`test_compose_reproduces_runtime_byte_identical` goes red; +* if the base config changes such that an overlay anchor no longer matches, + the compose fails loudly rather than emitting a wrong file -- + :func:`test_compose_fails_loud_on_stale_anchor` locks that fail-fast. + +The runtime file is a tracked input to ``im_init``; keeping it byte-stable is +what keeps the reproduction gate bit-exact, so this test's unit is bytes, not +parsed YAML. +""" + +import importlib.util +from pathlib import Path + +import pytest + + +def _repo_root() -> Path: + """Locate the repo root by walking up to the ``pyproject.toml`` marker.""" + for parent in Path(__file__).resolve().parents: + if (parent / "pyproject.toml").exists(): + return parent + raise RuntimeError("could not locate repo root (no pyproject.toml above test)") + + +_CALIB_DIR = _repo_root() / "config" / "calibration" +_BASE = _CALIB_DIR / "mask_v1.X.9.yaml" +_OVERLAY = _CALIB_DIR / "mask_v1.X.9_im_sim.overlay.yaml" +_RUNTIME = _CALIB_DIR / "mask_v1.X.9_im_sim.yaml" + + +def _compose_module(): + """Import ``workflow/scripts/im_compose_mask.py`` (lives outside the package).""" + path = _repo_root() / "workflow" / "scripts" / "im_compose_mask.py" + spec = importlib.util.spec_from_file_location("im_compose_mask", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_compose_reproduces_runtime_byte_identical(): + """compose(base, overlay) == the committed runtime file, byte-for-byte.""" + import yaml + + compose = _compose_module().compose + overlay = yaml.safe_load(_OVERLAY.read_text()) + base_text = _BASE.read_text() + + composed = compose(base_text, overlay) + + assert composed == _RUNTIME.read_text(), ( + "compose(mask_v1.X.9.yaml, overlay) no longer reproduces " + "mask_v1.X.9_im_sim.yaml byte-for-byte -- the runtime file and its " + "declared overlay have drifted; reconcile one against the other." + ) + + +def test_compose_fails_loud_on_stale_anchor(): + """A base whose text no longer carries an overlay anchor aborts, not composes. + + This is the drift-proofing: the overlay anchors to verbatim base text, so if + the base config is edited such that an anchor vanishes, the compose must die + with a clear message rather than silently emit a file missing that delta. + """ + import yaml + + module = _compose_module() + overlay = yaml.safe_load(_OVERLAY.read_text()) + # Drop the IMAFLAGS_ISO cut from the base text so its overlay anchor no + # longer matches; compose must abort (SystemExit from die()). + mangled = _BASE.read_text().replace("IMAFLAGS_ISO", "SOMETHING_ELSE") + + with pytest.raises(SystemExit, match="out of sync with the base"): + module.compose(mangled, overlay) diff --git a/workflow/image_sims/Snakefile b/workflow/image_sims/Snakefile new file mode 100644 index 00000000..d7a89065 --- /dev/null +++ b/workflow/image_sims/Snakefile @@ -0,0 +1,46 @@ +"""Standalone entry point for the image-simulation m-bias workflow. + +Run the sp_validation-side chain (merge -> extract -> calibrate -> m-bias), +optionally including the ShapePipe pipeline stage, without pulling in the +cosmology-validation config the top-level ``workflow/Snakefile`` requires. + +Layer a run config over the operational defaults -- the workflow config.yaml +carries operational defaults but *no* science keys, so it is incomplete on its +own (by design); the run config supplies the science knobs. The one drive +command on candide, with the committed SLURM profile owning all scheduling: + + snakemake --profile workflow/profiles/candide \\ + -s workflow/image_sims/Snakefile \\ + im_mbias --configfile my_run.yaml + +``configfile: "workflow/image_sims/config.yaml"`` below loads the operational +defaults automatically, so only ``my_run.yaml`` (the science knobs, and any +operational override that run wants) is passed on the command line; Snakemake +deep-merges the two. The profile supplies the executor, account, partition, +node excludes and job floor -- no ``-j`` needed (the slurm executor sets the +job cap). Off-cluster, drop ``--profile`` and add ``-j N`` to run locally. + +The target (``im_mbias``) is given *before* ``--configfile``: Snakemake's +``--configfile`` takes one-or-more paths, so a target placed after it is +swallowed as a config path ("No such file: im_mbias"). Put targets ahead of +``--configfile`` (or make ``--configfile`` the last flag on the line). Always +dry-run first with ``-n``. + +The same rules are also available inside the main workflow: they are included +there under ``if "image_sims" in config``. +""" + +configfile: "workflow/image_sims/config.yaml" + + +# The image-sims rules own their container invocation explicitly, so no +# top-level container is needed here. +container: None + + +include: "../rules/image_sims.smk" + + +rule all: + input: + f"{GRIDS_BASE}/results/m_bias_results.yaml", diff --git a/workflow/image_sims/config.yaml b/workflow/image_sims/config.yaml new file mode 100644 index 00000000..8f719c68 --- /dev/null +++ b/workflow/image_sims/config.yaml @@ -0,0 +1,89 @@ +# Image-simulation m-bias workflow configuration. +# +# Two kinds of keys live under `image_sims:`, and the split is the point: +# +# * OPERATIONAL keys default here (active lines below) and *nowhere else* -- +# the .smk reads them bare, so this file is their single home. Override in +# a run config only when a run genuinely differs from the shared setup. +# +# * SCIENCE keys have NO default -- not here, not in code. They fix the +# estimator's scientific behaviour and must be stated per run, so they +# appear below only as commented template lines. Supply them in a run +# config layered on top: +# +# snakemake -s workflow/image_sims/Snakefile \ +# --configfile workflow/image_sims/config.yaml \ +# --configfile my_run.yaml \ +# -j 4 im_mbias +# +# A run config that omits a science key fails at DAG parse, naming the key; an +# unknown key under `image_sims:` fails as a typo. The structural keys below +# (sif, repos, data roots, num, tile_ids) also have no default and must be set. + +image_sims: + + # --- containers ------------------------------------------------------- + # Two images, one per half of the chain (the split gate766 ran). One image + # is the eventual target -- the sp_validation image is FROM the ShapePipe + # image -- but until sp_validation is uv-locked with cosmo_numba declared, + # its published image can drift NumPy past numba's window (seen 2026-07-11: + # "Numba needs NumPy 2.4 or less. Got NumPy 2.5" at ngmix). PYTHONPATH + # shadows pure-Python code only, never binary deps. + sif: /n17data/cdaley/containers/sp_validation_im_sims.sif # extract/calibrate/m-bias + sif_pipeline: /n17data/cdaley/containers/shapepipe_im_sims-runtime.sif # pipeline/merge + # Apptainer bind mounts. /automnt is required when repos/data are + # automounted (candide gotcha); harmless otherwise. [operational] + binds: /n17data,/n09data,/home,/automnt + + # --- repositories ----------------------------------------------------- + # Bound into the image; both repos' src go on PYTHONPATH so this branch's + # code wins over the baked copies: ShapePipe's #766 build, and sp_validation's + # image_sims.py / catalog.match_catalogs_radec. + shapepipe_repo: /n17data/cdaley/unions/code/shapepipe + sp_validation_repo: /n17data/cdaley/unions/code/sp_validation + + # --- data and run directories ---------------------------------------- + # grids_base is the run/output root: one sub-directory per simulation. + grids_base: /n17data/cdaley/unions/scratch-wf/imsims-run/grids + input_sims_base: /n09data/hervas/skills_out + psf_dict: /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle + + # --- simulation grid -------------------------------------------------- + sims_type: grid # 'grid' -> *_grid_{num}; anything else -> *_{num} [operational] + num: 1 + # Branches this run requests: the unsheared reference plus the four +/- + # sheared branches. Injected shear is NOT set here -- it is parsed from each + # branch's basic_info.txt by im_manifest. [operational] + branches: ["1z2z", "1p2z", "1m2z", "1z2p", "1z2m"] + # Tiles to process: an explicit list, the one tile-input mechanism. + tile_ids: ["233.293", "237.292", "238.292"] + + # --- calibration ------------------------------------------------------ + shape: ngmix # [operational] + # ShapePipe cfis configs (final_cat.param etc.); default is + # {shapepipe_repo}/example/cfis_image_sims once #766 lands. [operational] + config_dir: /n17data/cdaley/unions/scratch-wf/imsims-run/grids/_cfis_image_sims + psf_model: psfex # [operational] + n_smp: -1 # [operational] + # Extract/calibrate scripts run from the sp_validation repo checkout (branch + # code, not the baked copies). Point elsewhere for a different checkout. + # [operational] + extract_script: /n17data/cdaley/unions/code/sp_validation/scripts/calibration/extract_info.py + calibrate_script: /n17data/cdaley/unions/code/sp_validation/scripts/calibration/calibrate_comprehensive_cat.py + + # --- science knobs (REQUIRED in the run config; no default) ----------- + # Copy these into your run config and set them. There is deliberately no + # default: each fixes the estimator's scientific behaviour, so a run must + # state it. The injected |g| is separate -- parsed from basic_info.txt by + # im_manifest, never set here. + # + # mask_config: config/calibration/mask_v1.X.9_im_sim.yaml # relative to sp_validation_repo + # match_radius_deg: 0.0002 # RA/Dec pair-match radius, degrees + # w_cols: [none, w_iv] # weight schemes to compute in one run; "none" + # # (or null) = unit weights (#227); first entry is + # # the primary/headline result -- lead with "none" + # # for the fiducial unweighted estimator. The + # # deprecated scalar `w_col` is still accepted. + # pair_match: true # match +g/-g object-by-object (per-object cancellation) + # n_bootstrap: 500 # bootstrap resamples for the errors + # bootstrap_seed: 42 # seed for the bootstrap RNG (makes errors reproducible) diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml new file mode 100644 index 00000000..0316b909 --- /dev/null +++ b/workflow/profiles/candide/config.yaml @@ -0,0 +1,85 @@ +# Committed SLURM profile for the candide cluster (IAP). +# +# This is the "one run command" half of the workflow: drive any target with +# +# snakemake --profile workflow/profiles/candide \ +# -s workflow/image_sims/Snakefile \ +# --configfile +# +# and Snakemake owns all scheduling -- it fans out one SLURM job per branch x +# tile and drives them against the cluster, MPI-free. Everything here is +# cluster policy (executor, account, partition, node excludes, per-job +# defaults); it carries no science and no workflow logic. +# +# What is deliberately NOT here: +# +# * Container / apptainer settings. The image-sims rules set +# ``container: None`` and own their ``apptainer exec`` call through the +# shared ``EXEC`` prefix (one image for every stage, with PYTHONPATH / +# PSF_DICT / OMP_NUM_THREADS injected there). So no +# ``software-deployment-method: apptainer`` / ``apptainer-args`` -- those +# would wrap a *second*, redundant container around jobs that already run +# inside one. +# +# * OMP_NUM_THREADS. It is pinned to 1 on the ``apptainer exec`` line in +# workflow/rules/image_sims.smk, not here. The slurm executor submits with +# ``--export=ALL``, which propagates the *driver's* ambient environment; a +# profile only sets CLI flags, never the driver's own env, so an +# ``OMP_NUM_THREADS`` set here would silently depend on the operator having +# exported it by hand. Injecting it at the container boundary puts it where +# the compute runs, committed and independent of the launching shell. +# +# * Per-rule resources (mem_mb, runtime). Those live on each rule in the +# .smk; the ``default-resources`` below are only the floor for rules that +# set none. + +executor: slurm + +# Cluster policy applied to every job unless a rule overrides it. The excludes +# are the flaky/no-internet candide nodes (n17 mount issues, n09 no internet, +# n36); ``slurm_extra`` is passed verbatim onto the sbatch line by the executor +# plugin, so the quoting is what sbatch must see. +# +# Three candide-specific SLURM lessons are baked into the values below (learned +# the hard way on the earlier hand-driven im-sims runs; see shapepipe's retired +# image_sims_pipeline/Snakefile docstring): +# +# * ``runtime`` MUST carry a unit (``60m``, ``6h``, ``2d``). Snakemake's +# resource parser reads a *bare* number as SECONDS, so ``runtime: 60`` would +# silently give every job a 60-second wall clock and kill it on start. The +# quoted-with-unit form here is deliberate; keep it that way, and prefer the +# same in any ``--default-resources`` passed on the command line. (A bare +# integer in a *rule's* ``resources: runtime=720`` is fine -- snakemake +# reads rule-level numeric runtime as minutes -- the seconds trap is only +# the CLI/default-resources parser.) +# +# * ``cpus_per_task`` is pinned to 12 to CAP JOBS PER NODE, not because a job +# needs 12 cores (the chain is MPI-free and pins ``OMP_NUM_THREADS=1`` at +# the container). candide's per-user process limit is ``ulimit -u 1200`` +# *per node*, and apptainer crashes ("can't start new thread") beyond ~4 +# concurrent jobs on a 48-core node. Requesting 12 CPUs/job holds SLURM to +# ~4 jobs per 48-core node, under the ceiling. Dropping this to 1 would let +# SLURM pack ~48 jobs onto a node and crash the compute-heavy im_pipeline +# stage (which inherits this default -- it sets mem/runtime but not cpus). +# +# * After launching a real fan-out, VERIFY the request actually landed: +# ``squeue -u $USER -o "%C %l"`` must show 12 (CPUs) and the wall clock you +# intended (e.g. 12:00:00 for im_pipeline). A silently-misparsed runtime or +# cpus shows up here before it wastes a queue slot. +default-resources: + slurm_account: "cusers" + slurm_partition: "comp,pscomp" + runtime: "60m" + cpus_per_task: 12 + slurm_extra: "'--exclude=n17,n09,n36'" + +# Give an appearing output file a moment on candide's automounted filesystems +# before Snakemake calls a job failed for a missing output, and retry a job +# once on transient node failure. +latency-wait: 5 +retries: 1 + +# Keep the SLURM logs of successful jobs (candide debugging), and rerun a job +# when its code / params / inputs change, not only on mtime. +slurm-keep-successful-logs: true +rerun-triggers: ["mtime", "params", "input", "code"] diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk new file mode 100644 index 00000000..a4eb92d0 --- /dev/null +++ b/workflow/rules/image_sims.smk @@ -0,0 +1,568 @@ +"""Image-simulation orchestration: raw SKiLLS sim images -> shear m/c bias. + +This rule set drives the image-simulation validation chain end to end and is +the sp_validation-side half of the split described in +``UNIONS-WL/MultiBand_ImSim#1``: ShapePipe turns the simulated tiles into +per-tile shape catalogues, then sp_validation merges, extracts, calibrates and +finally measures the multiplicative/additive shear bias. + +Two images, one prefix shape. Architecturally one image could run every +stage -- the sp_validation image is built ``FROM`` the ShapePipe image, so it +carries both stacks -- but the *published* sp_validation image's environment +is not yet trustworthy for the ShapePipe half: sp_validation has no lockfile +and does not declare its numba-bearing dependency (``cosmo_numba``), so +unpinned install layers can drift NumPy past numba's window (a 2026-07-11 +gate run hit exactly this: ``Numba needs NumPy 2.4 or less. Got NumPy 2.5`` +at the ngmix stage). PYTHONPATH shadowing covers pure-Python *code*, never +binary deps, so until sp_validation is uv-locked with its deps declared +(spun off as its own task), each half runs in its own repo's image -- the +same split the gate766 baseline ran: + +* ShapePipe stages -> ``pipeline`` (raw images -> per-tile cats) and ``merge`` + (``create_final_cat`` -> ``final_cat_{sim}.hdf5``) run in ``sif_pipeline`` + (the ShapePipe image). +* sp_validation stages -> ``manifest``, ``extract`` (-> comprehensive cat), + ``calibrate`` (-> cut cat) and ``m_bias`` (-> ``m_bias_results.yaml``) run + in ``sif`` (the sp_validation image). + +Every rule sets ``container: None`` and calls ``apptainer exec`` explicitly +through a shared prefix template (``EXEC_PIPELINE`` / ``EXEC`` -- identical +env injections, different image), because the images are not the workflow's +top-level container. Everything is parameterised under +``config["image_sims"]`` -- the two ``sif`` keys, repository roots, data roots, +the PSF dictionary, the explicit ``tile_ids`` list and the sim/calibration +knobs -- so a fresh user drives it from config alone, with no hard-coded clone +layout. Configuration is fail-fast: a schema check at load rejects an unknown +key (typo) and a missing science key (see ``workflow/image_sims/config.yaml`` +for the operational/science split). The ``PYTHONPATH`` override injects both +repos' ``src`` so the *branch* source (ShapePipe's ``#766`` build; +sp_validation's ``image_sims.py``, ``catalog.match_catalogs_radec``) wins over +whatever is baked into the image. + +The five simulations per grid are the reference ``1z2z`` (no input shear) plus +the ``+/-`` shear pairs ``1p2z``/``1m2z`` (g1) and ``1z2p``/``1z2m`` (g2); the +m-bias estimator matches each to the reference by RA/Dec. +""" + +import os +from pathlib import Path + +IMSIM = config["image_sims"] + +# --- fail-fast schema check ---------------------------------------------- +# One home for every fact: the run config carries the science knobs, the +# workflow config.yaml carries the operational defaults, and *this* block is +# where a typo or a missing knob dies -- at DAG parse, before any compute. +# +# Every key must be declared below. An unknown key under ``image_sims:`` is a +# hard error (typo protection); a missing *science* key is a hard error naming +# the key (no silent code default anywhere). Operational keys default in the +# workflow config.yaml and nowhere else: the .smk reads them as bare +# ``IMSIM[key]`` (never ``.get`` with a second literal), so their value comes +# from config.yaml alone -- the single home for an operational default. +# +# Science keys: required from the *run* config; no default in config.yaml (only +# a commented template line) and no default in code. These fix the estimator's +# scientific behaviour, so they must be stated per run, never inherited. +_SCIENCE_KEYS = { + "w_cols", + "pair_match", + "match_radius_deg", + "n_bootstrap", + "bootstrap_seed", + "mask_config", +} +# Deprecated science keys: accepted (so a pre-``w_cols`` run config still parses +# and the estimator's back-compat path runs) but not *required* -- our configs +# state ``w_cols``. Listed here only to keep them out of the unknown-key error. +_DEPRECATED_KEYS = { + "w_col", +} +# Operational keys: default (visibly) in the workflow config.yaml; the .smk +# reads them bare, so config.yaml is their one home. +_OPERATIONAL_KEYS = { + "binds", + "sims_type", + "branches", + "shape", + "config_dir", + "psf_model", + "n_smp", + "extract_script", + "calibrate_script", +} +# Structural keys: paths/identifiers the run must supply (no sensible default). +_STRUCTURAL_KEYS = { + "sif", + "sif_pipeline", + "shapepipe_repo", + "sp_validation_repo", + "grids_base", + "input_sims_base", + "psf_dict", + "num", + "tile_ids", +} +_ALLOWED_KEYS = ( + _SCIENCE_KEYS | _DEPRECATED_KEYS | _OPERATIONAL_KEYS | _STRUCTURAL_KEYS +) + +_unknown = set(IMSIM) - _ALLOWED_KEYS +if _unknown: + raise ValueError( + "image_sims: unknown config key(s) " + f"{sorted(_unknown)} -- check for a typo (allowed keys: " + f"{sorted(_ALLOWED_KEYS)})" + ) +_missing_science = sorted(_SCIENCE_KEYS - set(IMSIM)) +if _missing_science: + raise ValueError( + "image_sims: missing required science key(s) " + f"{_missing_science} -- these have no default and must be set in the " + "run config (see the commented template in workflow/image_sims/config.yaml)" + ) +_missing_structural = sorted(_STRUCTURAL_KEYS - set(IMSIM)) +if _missing_structural: + raise ValueError( + "image_sims: missing required key(s) " + f"{_missing_structural} -- set them in the run config" + ) + +# --- containers ----------------------------------------------------------- +# Two images (see module docstring): the ShapePipe image for the pipeline and +# merge stages, the sp_validation image for everything downstream. Collapse +# back to one image once sp_validation's env is lock-managed. +SIF = IMSIM["sif"] # sp_validation stages +SIF_PIPELINE = IMSIM["sif_pipeline"] # ShapePipe stages +BINDS = IMSIM["binds"] + +# --- repositories (bound into the image; branch code overrides) ----------- +SHAPEPIPE_REPO = IMSIM["shapepipe_repo"] +SPV_REPO = IMSIM["sp_validation_repo"] + +# --- data and run directories -------------------------------------------- +GRIDS_BASE = IMSIM["grids_base"] # run/output root; one sub-dir per sim +INPUT_SIMS_BASE = IMSIM["input_sims_base"] # SKiLLS sim images +PSF_DICT = IMSIM["psf_dict"] # Herve's Full_psf_dict.pickle + +# --- simulation grid ------------------------------------------------------ +# SIM_BASES is the set of branches *this run requests* -- the reference plus the +# four +/- sheared branches. Their injected shear (amplitude, per-branch +# (g1,g2), pairing) is NOT a literal here: it lives only in manifest.yaml, built +# by im_manifest from each branch's basic_info.txt and read back by im_mbias. +NUM = IMSIM["num"] +SIMS_TYPE = IMSIM["sims_type"] +_SUFFIX = f"_{SIMS_TYPE}_{NUM}" if SIMS_TYPE == "grid" else f"_{NUM}" +SIM_BASES = list(IMSIM["branches"]) +SIMS = [f"{base}{_SUFFIX}" for base in SIM_BASES] +MANIFEST = f"{GRIDS_BASE}/manifest.yaml" +BUILD_MANIFEST = f"{SPV_REPO}/workflow/scripts/im_build_manifest.py" + +# --- tiles ---------------------------------------------------------------- +# tile_ids is the one tile-input mechanism: an explicit list in the run config. +TILE_IDS = list(IMSIM["tile_ids"]) + +# --- calibration / m-bias knobs ------------------------------------------ +SHAPE = IMSIM["shape"] +MASK_CONFIG = IMSIM["mask_config"] # e.g. config/calibration/mask_v1.X.9_im_sim.yaml +PARAMS_TEMPLATE = f"{SPV_REPO}/workflow/image_sims/params_im_sim.py" +# ShapePipe cfis_image_sims config dir (per-tile/exposure configs + final_cat.param). +CONFIG_DIR = IMSIM["config_dir"] + +# ShapePipe scripts live in the ShapePipe repo (also baked into its image). +CREATE_FINAL_CAT = f"{SHAPEPIPE_REPO}/scripts/python/create_final_cat.py" +RUN_JOB = f"{SHAPEPIPE_REPO}/scripts/sh/run_job_sp_canfar_v2.0.bash" +# Extract/calibrate run from the sp_validation *repo* checkout (bind-mounted), +# not the baked copies: the container tracks the branch but lags it, and the +# image-sims path needs branch-only fixes (star-catalogue-optional extract, +# FITS-aware CalibrateCat.read_cat). Overridable for a different checkout. +EXTRACT_INFO = IMSIM["extract_script"] +CALIBRATE = IMSIM["calibrate_script"] +# m-bias is *this branch's* extracted core, injected on PYTHONPATH. +COMPUTE_M_BIAS = f"{SPV_REPO}/scripts/compute_m_bias_image_sims.py" + +# --- container exec prefixes ---------------------------------------------- +# One prefix *shape* for every stage -- two instances, one per image. Three +# env injections make the on-disk branch +# code and the sim PSF win over the image's baked copies: +# +# * PYTHONPATH prepends BOTH repos' ``src`` (ShapePipe first, then +# sp_validation), so Python resolves the worktree build before +# ``/app``/``/sp_validation`` -- the local-testing counterpart of the +# git-ref deps, letting the branch code run without an image rebuild. This +# covers the Python *packages* only: the bash entry points (run_job) and +# the ShapePipe/sp_validation *scripts* are still invoked at the repo paths +# resolved from config (RUN_JOB, CREATE_FINAL_CAT, EXTRACT_INFO, ...), not +# shadowed by PYTHONPATH. +# * PSF_DICT points the fake_psf module (PSF_DICT_PATH = $PSF_DICT, expanded +# via getexpanded) at this run's PSF dictionary. +# +# The SLURM env vars are stripped (``env -u ...``) so that when the ShapePipe +# pipeline stage's OpenMPI initialises inside the image it does not try to +# attach to the host SLURM launcher (cf. apptainer_noslurm.sh). The strip is +# harmless for the pure-Python sp_validation stages, so one prefix serves all. +# +# ``OMP_NUM_THREADS=1`` is injected here, at the ``apptainer exec`` call, and +# not left to the SLURM profile. The chain is MPI-free: Snakemake fans out one +# job per branch x tile and each job's parallelism is ShapePipe's own internal +# multiprocessing (``-N n_smp``), so the OpenMP/BLAS thread pool inside the +# container must be pinned to 1 to avoid oversubscription. The SLURM profile +# cannot pin it reliably: the slurm executor submits with ``--export=ALL``, +# which propagates the *driver's* ambient environment -- but a Snakemake +# profile only sets CLI flags, never the driver's own env, so an +# ``OMP_NUM_THREADS`` there would depend on the operator having exported it by +# hand (the implicit, uncommitted state the "one run command" is meant to +# retire). Injecting it on the ``apptainer exec`` line puts it where the +# compute actually runs -- inside the container, independent of the driver's +# env -- the same lever this prefix already uses for PYTHONPATH/PSF_DICT. +_EXEC_PREFIX = ( + "env -u SLURM_JOBID -u SLURM_JOB_ID -u SLURM_PROCID " + f"apptainer exec --bind {BINDS} " + f"--env PYTHONPATH={SHAPEPIPE_REPO}/src:{SPV_REPO}/src " + f"--env PSF_DICT={PSF_DICT} --env OMP_NUM_THREADS=1 " +) +EXEC = _EXEC_PREFIX + SIF # sp_validation stages +EXEC_PIPELINE = _EXEC_PREFIX + SIF_PIPELINE # ShapePipe stages + +JOB_MASK = sum([1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]) + + +wildcard_constraints: + sim="|".join(SIMS), + tile="|".join(t.replace(".", r"\.") for t in TILE_IDS), + + +# ========================================================================== +# Convenience targets (run in order) +# ========================================================================== +rule im_manifest_only: + input: + MANIFEST, + + +rule im_init_all: + input: + expand(f"{GRIDS_BASE}/{{sim}}/params.py", sim=SIMS), + + +rule im_pipeline_all: + input: + expand( + f"{GRIDS_BASE}/{{sim}}/logs/pipeline_{{tile}}.done", + sim=SIMS, + tile=TILE_IDS, + ), + + +rule im_merge_all: + input: + expand(f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", sim=SIMS), + + +rule im_extract_all: + input: + expand( + f"{GRIDS_BASE}/{{sim}}/shape_catalog_comprehensive_{SHAPE}.fits", + sim=SIMS, + ), + + +rule im_calibrate_all: + input: + expand( + f"{GRIDS_BASE}/{{sim}}/shape_catalog_cut_{SHAPE}.fits", sim=SIMS + ), + + +# ========================================================================== +# Rules +# ========================================================================== +rule im_manifest: + """Build the campaign manifest at the head of the DAG. + + Parses ``g_cosmic`` from every requested branch's ``basic_info.txt``, + cross-checks each against its ``1{X}2{Y}`` name and the (0,0) reference, + derives the single injected amplitude, and writes ``manifest.yaml`` into the + run root. This is the one home for the injected-shear facts; im_mbias reads + the amplitude and branch map from here, nowhere else. Pure sp_validation + stage (stdlib parse of basic_info; PyYAML to write). + """ + input: + # basic_info.txt for each requested branch, so editing a sim's record + # rebuilds the manifest (and re-validates) rather than reusing a stale one. + basic_info=expand( + f"{INPUT_SIMS_BASE}/{{sim}}/basic_info.txt", sim=SIMS + ), + output: + manifest=MANIFEST, + params: + branch_args=lambda wc: " ".join(f"--branch {b}" for b in SIM_BASES), + input_sims_base=INPUT_SIMS_BASE, + sims_type=SIMS_TYPE, + num=NUM, + shell: + "{EXEC} python {BUILD_MANIFEST} " + "--input-sims-base {params.input_sims_base} " + "--sims-type {params.sims_type} --num {params.num} " + "{params.branch_args} -o {output.manifest}" + + +rule im_init: + """Stage per-sim run directory: params.py, mask config, ShapePipe configs, + and the raw SKiLLS image inputs. + + ``params_im_sim.py`` derives the field name from the directory basename, so + the same template serves every sim; ``config_mask.yaml`` and ``cfis`` are + symlinks the downstream calibration and merge steps read from cwd. + + ``input_tiles``/``input_exp`` are top-level symlinks to the raw SKiLLS tile + and exposure images; ShapePipe's ``get_images_runner`` resolves them via + ``$SP_DIR/input_{tiles,exp}`` (``$SP_DIR`` is the run dir). ``run_job`` does + not stage these, so ``im_init`` must -- this is what makes ``im_pipeline`` + runnable from raw images, not just from pre-staged intermediates. + """ + input: + # Tracked so that editing the params template or mask config re-stages + # them into every run dir (a plain params: value would not retrigger, + # silently leaving stale params.py behind after a grammar change). + template=PARAMS_TEMPLATE, + mask_src=os.path.join(SPV_REPO, MASK_CONFIG), + output: + params=f"{GRIDS_BASE}/{{sim}}/params.py", + mask=f"{GRIDS_BASE}/{{sim}}/config_mask.yaml", + params: + config_dir=CONFIG_DIR, + run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + cfis=lambda wc: f"{GRIDS_BASE}/{wc.sim}/cfis", + sim_tiles=lambda wc: f"{INPUT_SIMS_BASE}/{wc.sim}/images/SP_tiles", + sim_exp=lambda wc: f"{INPUT_SIMS_BASE}/{wc.sim}/images/SP_exp", + shell: + # cfis / input_tiles / input_exp are stable read-only symlinks (used by + # get_images, merge, extract); created here but not tracked as outputs, + # which snakemake will not accept for a symlink/directory. + "mkdir -p $(dirname {output.params}) && " + "cp {input.template} {output.params} && " + "ln -sf {input.mask_src} {output.mask} && " + "ln -sfT {params.config_dir} {params.cfis} && " + "ln -sfT {params.sim_tiles} {params.run_dir}/input_tiles && " + "ln -sfT {params.sim_exp} {params.run_dir}/input_exp" + + +rule im_pipeline: + """Run ShapePipe on one simulated tile (ShapePipe stage). + + Delegates the module DAG to ShapePipe's own job runner; the sentinel log + marks tile completion for the merge step. This is the compute-heavy, + MPI-bearing stage. + """ + input: + # ``params.py`` is a *tracked* output of ``im_init``, so this one input + # supplies the im_init -> im_pipeline edge. The ``cfis`` symlink the + # shell reads (via {RUN_JOB}) is created by that same im_init shell block + # as an *untracked* side effect -- no rule declares it as an output + # (snakemake will not track a symlink/directory output). Declaring it an + # input here therefore asked the DAG for a file no rule produces: on a + # fresh grids_base it aborted the build with MissingInputException before + # any job ran. It is safe to drop -- cfis exists whenever params does, + # since im_init stages both together. + params=f"{GRIDS_BASE}/{{sim}}/params.py", + output: + done=touch(f"{GRIDS_BASE}/{{sim}}/logs/pipeline_{{tile}}.done"), + params: + run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + psf=IMSIM["psf_model"], + n_smp=IMSIM["n_smp"], + resources: + mem_mb=16000, + runtime=720, + shell: + "cd {params.run_dir} && " + "{EXEC_PIPELINE} bash {RUN_JOB} " + "-e {wildcards.tile} -t image_sims -j {JOB_MASK} " + "-p {params.psf} -N {params.n_smp}" + + +rule im_merge: + """Merge per-tile ShapePipe catalogues into final_cat_{sim}.hdf5. + + ``create_final_cat.py`` lives in the ShapePipe repo/image; run in image_sims + mode (``-I``) it walks the per-tile output under the run directory. + """ + input: + tiles=expand( + f"{GRIDS_BASE}/{{{{sim}}}}/logs/pipeline_{{tile}}.done", + tile=TILE_IDS, + ), + output: + cat=f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", + params: + run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + shell: + "cd {params.run_dir} && " + "{EXEC_PIPELINE} python {CREATE_FINAL_CAT} " + "-I -m final_cat_{wildcards.sim}.hdf5 -i .. " + "-p cfis/final_cat.param -P {wildcards.sim} " + "-o n_tiles_final.txt -v" + + +rule im_extract: + """Extract the comprehensive ngmix catalogue (sp_validation stage). + + ``extract_info.py`` reads ``params.py`` from cwd and the merged catalogue, + writing ``shape_catalog_comprehensive_{shape}``. + """ + input: + cat=f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", + params=f"{GRIDS_BASE}/{{sim}}/params.py", + output: + cat=f"{GRIDS_BASE}/{{sim}}/shape_catalog_comprehensive_{SHAPE}.fits", + params: + run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + shell: + "cd {params.run_dir} && {EXEC} python {EXTRACT_INFO}" + + +rule im_calibrate: + """Calibrate and cut the comprehensive catalogue (sp_validation stage). + + ``calibrate_comprehensive_cat.py`` reads ``config_mask.yaml`` from cwd, + applies the metacal calibration and selection, and writes + ``shape_catalog_cut_{shape}.fits``. + """ + input: + cat=f"{GRIDS_BASE}/{{sim}}/shape_catalog_comprehensive_{SHAPE}.fits", + mask=f"{GRIDS_BASE}/{{sim}}/config_mask.yaml", + output: + cat=f"{GRIDS_BASE}/{{sim}}/shape_catalog_cut_{SHAPE}.fits", + params: + run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + shell: + "cd {params.run_dir} && " + "{EXEC} python {CALIBRATE} -s calibrate" + + +rule im_mbias: + """Multiplicative/additive shear bias from the calibrated grids. + + Produces the workflow's headline artifact, ``m_bias_results.yaml``. The + injected shear (``shear_amplitude`` and the branch map) comes from + ``manifest.yaml`` alone -- no literal amplitude here or in config.yaml. The + generated ``m_bias_config.yaml`` carries the manifest's ``branches`` and + ``pairs``, so the estimator's sim list and pairing are the campaign's, not a + hard-coded default. + """ + input: + manifest=MANIFEST, + cats=expand( + f"{GRIDS_BASE}/{{sim}}/shape_catalog_cut_{SHAPE}.fits", sim=SIMS + ), + output: + results=f"{GRIDS_BASE}/results/m_bias_results.yaml", + params: + cfg=f"{GRIDS_BASE}/results/m_bias_config.yaml", + grids_base=GRIDS_BASE, + num=NUM, + cat_name=f"shape_catalog_cut_{SHAPE}.fits", + sif=SIF, + sif_pipeline=SIF_PIPELINE, + shapepipe_repo=SHAPEPIPE_REPO, + sp_validation_repo=SPV_REPO, + # Science knobs, read bare from the run config (no default here). + match_radius_deg=IMSIM["match_radius_deg"], + w_cols=IMSIM["w_cols"], + n_bootstrap=IMSIM["n_bootstrap"], + pair_match=IMSIM["pair_match"], + bootstrap_seed=IMSIM["bootstrap_seed"], + run: + import hashlib + import re + import subprocess + + import yaml + + with open(input.manifest) as fh: + manifest = yaml.safe_load(fh) + + def _git(repo, *args): + """Read a git fact from ``repo``; ``None`` if it is not a checkout.""" + try: + return subprocess.run( + ["git", "-C", repo, *args], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError): + return None + + def _sif_revision(sif_path): + """GHCR revision baked into the SIF's OCI labels. + + A plain-text scan of the image file (login-safe: no exec, no + container start), reading org.opencontainers.image.revision -- the + source commit GHCR built the image from. ``None`` if absent. + """ + try: + with open(sif_path, "rb") as fh: + blob = fh.read() + except OSError: + return None + m = re.search( + rb'org\.opencontainers\.image\.revision"?[:=]"?([0-9a-f]{7,40})', + blob, + ) + return m.group(1).decode() if m else None + + # Manifest hash: sha256 of the exact bytes im_manifest wrote, so the + # result records which injected-shear facts it was computed against. + with open(input.manifest, "rb") as fh: + manifest_sha256 = hashlib.sha256(fh.read()).hexdigest() + + provenance = { + "manifest_sha256": manifest_sha256, + "sp_validation": { + "branch": _git(params.sp_validation_repo, "rev-parse", "--abbrev-ref", "HEAD"), + "commit": _git(params.sp_validation_repo, "rev-parse", "HEAD"), + }, + "shapepipe": { + "branch": _git(params.shapepipe_repo, "rev-parse", "--abbrev-ref", "HEAD"), + "commit": _git(params.shapepipe_repo, "rev-parse", "HEAD"), + }, + "containers": { + "sif": params.sif, + "ghcr_revision": _sif_revision(params.sif), + "sif_pipeline": params.sif_pipeline, + "ghcr_revision_pipeline": _sif_revision(params.sif_pipeline), + }, + } + + os.makedirs(os.path.dirname(output.results), exist_ok=True) + # Emit *every* key the estimator requires -- pair_match and + # bootstrap_seed included. Requiring a key without emitting it would + # be a KeyError at run time, so the generated config is the complete + # contract between rule and estimator. ``provenance`` rides along as a + # top-level block: the compute script copies it verbatim into the output + # results yaml, so a result file is self-describing (which manifest, + # which repo commits, which container built the number). + mbias_cfg = { + "grids_dir": params.grids_base, + "num": params.num, + "catalog_name": params.cat_name, + # Injected shear: from the manifest, the single source of truth. + "shear_amplitude": manifest["shear_amplitude"], + "branches": list(manifest["branches"]), + "pairs": manifest["pairs"], + "match_radius_deg": params.match_radius_deg, + "w_cols": list(params.w_cols), + "pair_match": params.pair_match, + "n_bootstrap": params.n_bootstrap, + "bootstrap_seed": params.bootstrap_seed, + "results_dir": os.path.dirname(output.results), + "output_path": output.results, + "provenance": provenance, + } + with open(params.cfg, "w") as fh: + yaml.safe_dump(mbias_cfg, fh) + shell( + "{EXEC} python {COMPUTE_M_BIAS} -c {params.cfg} -v" + ) diff --git a/workflow/scripts/im_build_manifest.py b/workflow/scripts/im_build_manifest.py new file mode 100644 index 00000000..6f0a0faf --- /dev/null +++ b/workflow/scripts/im_build_manifest.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python +"""Build the image-simulation campaign manifest from the sims' own records. + +This is the head of the image-simulation DAG: it reads the injected shear that +the sim campaign recorded for each requested branch, cross-checks it against the +branch's *name*, and writes a single ``manifest.yaml`` that every downstream +stage reads. The point is one home for the injected-shear facts -- amplitude, +per-branch ``(g1, g2)``, the reference branch, and the ``+/-`` pairing -- so no +literal amplitude or hard-coded branch list survives anywhere else. + +The source of truth is each branch's ``basic_info.txt``, written by the sim +campaign. The one line this parser needs looks like:: + + g_cosmic = 0.025 0.0 + +i.e. the literal key ``g_cosmic``, run-together whitespace, an ``=``, then the +two injected shear components ``g1 g2`` as space-separated floats (sign as a +leading ``-``; ``0.0`` for an un-sheared component). We parse *only* that line, +by stdlib string ops -- no YAML/regex dependency -- so the script runs inside +the container with nothing but the standard library. + +Branch names follow the ``1{X}2{Y}`` convention: the character after ``1`` is +the g1 sign, the character after ``2`` is the g2 sign, each one of ``p`` (+), +``m`` (-), ``z`` (0). So ``1p2z`` injects ``(+|g|, 0)``, ``1z2m`` injects +``(0, -|g|)``, ``1z2z`` is the un-sheared reference. The suffix +(``_grid_1`` etc.) is appended by the workflow and is not part of the sign code. + +Validation (all fail-loud, each message naming the offending file and field): + +* every branch's parsed ``(g1, g2)`` sign/axis matches its name's sign code; +* the reference branch parses to exactly ``(0, 0)``; +* the derived ``|g|`` (the single nonzero magnitude of a sheared branch) is + equal across all four sheared branches -- one injected amplitude for the + whole campaign. + +Only the branches this run requests are read and validated. +""" + +import argparse +import os +import sys + +import yaml + +# Branch-name sign code: the character after "1" (g1) and after "2" (g2). +_SIGN = {"p": +1, "m": -1, "z": 0} + + +def die(msg): + """Abort with a clear, prefixed message on stderr.""" + sys.exit(f"im_build_manifest: {msg}") + + +def basic_info_path(input_sims_base, branch): + """Path to a branch's basic_info.txt (the sim campaign's own record).""" + return os.path.join(input_sims_base, branch, "basic_info.txt") + + +def parse_g_cosmic(path): + """Parse the ``g_cosmic = g1 g2`` line from a basic_info.txt file. + + Returns ``(g1, g2)`` as floats. Fails loud, naming the file, if the line + is absent, malformed, or does not carry exactly two float components. + """ + if not os.path.isfile(path): + die(f"basic_info.txt not found: {path}") + + with open(path) as fh: + lines = fh.readlines() + + matches = [ln for ln in lines if ln.split("=", 1)[0].strip() == "g_cosmic"] + if not matches: + die(f"no 'g_cosmic' line in {path}") + if len(matches) > 1: + die(f"multiple 'g_cosmic' lines in {path}") + + rhs = matches[0].split("=", 1)[1].split() + if len(rhs) != 2: + die( + f"'g_cosmic' in {path}: expected two components 'g1 g2', " + f"got {len(rhs)}: {matches[0].strip()!r}" + ) + try: + return float(rhs[0]), float(rhs[1]) + except ValueError: + die(f"'g_cosmic' in {path}: components not floats: {matches[0].strip()!r}") + + +def sign_code(branch): + """Extract the ``(g1_sign, g2_sign)`` code from a ``1{X}2{Y}`` branch name. + + ``branch`` is the bare sign code (e.g. ``1p2z``), suffix already stripped. + Fails loud if the name does not match the convention. + """ + if ( + len(branch) != 4 + or branch[0] != "1" + or branch[2] != "2" + or branch[1] not in _SIGN + or branch[3] not in _SIGN + ): + die( + f"branch name {branch!r} does not match the 1{{X}}2{{Y}} convention " + f"(X, Y each one of p/m/z)" + ) + return _SIGN[branch[1]], _SIGN[branch[3]] + + +def build_manifest(input_sims_base, sims_type, num, branches): + """Parse + validate every requested branch, return the manifest dict. + + ``branches`` are bare sign codes (``1z2z``, ``1p2z``, ...); the on-disk + directory name is ``{branch}{suffix}`` with ``suffix`` derived from + ``sims_type``/``num`` exactly as the workflow builds it. + """ + suffix = f"_{sims_type}_{num}" if sims_type == "grid" else f"_{num}" + + parsed = {} # branch -> (g1, g2) from basic_info.txt + for branch in branches: + dirname = f"{branch}{suffix}" + g1, g2 = parse_g_cosmic(basic_info_path(input_sims_base, dirname)) + s1, s2 = sign_code(branch) + + # Sign/axis must agree with the name: a component is nonzero iff its + # sign code is nonzero, and its sign matches. + for comp, (g, s) in enumerate(((g1, s1), (g2, s2)), start=1): + path = basic_info_path(input_sims_base, dirname) + if s == 0 and g != 0.0: + die( + f"{path}: branch {branch!r} names g{comp} un-sheared (z) but " + f"g_cosmic gives g{comp} = {g}" + ) + if s != 0 and (g == 0.0 or (g > 0) != (s > 0)): + die( + f"{path}: branch {branch!r} names g{comp} sign {'+' if s > 0 else '-'} " + f"but g_cosmic gives g{comp} = {g}" + ) + parsed[branch] = (g1, g2) + + # Reference branch: the one whose name codes (0, 0). Must exist and be (0,0). + refs = [b for b in branches if sign_code(b) == (0, 0)] + if len(refs) != 1: + die( + f"expected exactly one reference branch (name code 1z2z) among " + f"{branches}, found {refs}" + ) + reference = refs[0] + if parsed[reference] != (0.0, 0.0): + die( + f"{basic_info_path(input_sims_base, f'{reference}{suffix}')}: reference " + f"branch {reference!r} must inject (0, 0), got {parsed[reference]}" + ) + + # Derived amplitude: the single nonzero magnitude of each sheared branch, + # cross-checked equal across all four. + amplitudes = {} # branch -> |g| + for branch in branches: + if branch == reference: + continue + g1, g2 = parsed[branch] + amplitudes[branch] = abs(g1) if g1 != 0.0 else abs(g2) + distinct = sorted(set(amplitudes.values())) + if len(distinct) != 1: + die( + "injected |g| differs across sheared branches (must be one campaign " + f"amplitude): {amplitudes} " + f"[files under {input_sims_base}/{suffix}/basic_info.txt]" + ) + shear_amplitude = distinct[0] + + # Pairs: (+component, -component) for each sheared axis, in branch order so + # the estimator's per-pair processing order is stable. + plus = {} # component (0/1) -> branch with +|g| on that component + minus = {} + for branch in branches: + if branch == reference: + continue + s1, s2 = sign_code(branch) + comp = 0 if s1 != 0 else 1 + (plus if (s1 or s2) > 0 else minus)[comp] = branch + pairs = [ + {"plus": plus[comp], "minus": minus[comp], "component": comp} + for comp in sorted(set(plus) & set(minus)) + ] + + return { + "input_sims_base": input_sims_base, + "sims_type": sims_type, + "num": num, + "shear_amplitude": shear_amplitude, + "reference": reference, + # Branch order preserved (dict insertion order round-trips through + # yaml.safe_dump with sort_keys=False) so downstream load order is fixed. + "branches": { + branch: {"g1": parsed[branch][0], "g2": parsed[branch][1]} + for branch in branches + }, + "pairs": pairs, + } + + +def parse_args(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "--input-sims-base", + required=True, + help="root under which each branch dir holds basic_info.txt", + ) + p.add_argument( + "--sims-type", required=True, help="'grid' -> _grid_{num} suffix, else _{num}" + ) + p.add_argument("--num", required=True, type=int, help="run number") + p.add_argument( + "--branch", + required=True, + action="append", + dest="branches", + help="bare branch sign code (1z2z, 1p2z, ...); repeatable", + ) + p.add_argument("-o", "--output", required=True, help="manifest.yaml output path") + return p.parse_args() + + +def main(): + args = parse_args() + manifest = build_manifest( + args.input_sims_base, args.sims_type, args.num, args.branches + ) + os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) + with open(args.output, "w") as fh: + yaml.safe_dump(manifest, fh, sort_keys=False) + print(f"im_build_manifest: wrote {args.output}") + print(f" shear_amplitude = {manifest['shear_amplitude']}") + print(f" reference = {manifest['reference']}") + print(f" branches = {list(manifest['branches'])}") + for pair in manifest["pairs"]: + print( + f" pair g{pair['component'] + 1}: {pair['plus']} (+) / {pair['minus']} (-)" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workflow/scripts/im_compose_mask.py b/workflow/scripts/im_compose_mask.py new file mode 100644 index 00000000..fb602a11 --- /dev/null +++ b/workflow/scripts/im_compose_mask.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +"""Compose the image-sim mask/calibration config from base + declared overlay. + +The image-sim calibration differs from the data calibration in only a handful +of places (input path, dropped coverage cuts, unweighted global response, +additive-bias off). Instead of maintaining a second full copy of the config -- +which can silently drift from the base it was branched from -- we keep the +*data* config (``mask_v1.X.9.yaml``) as the single home for the shared cuts and +declare the sim-specific delta in an overlay +(``mask_v1.X.9_im_sim.overlay.yaml``). This script applies the overlay to the +base and emits the resolved config, which is byte-for-byte the committed runtime +file ``mask_v1.X.9_im_sim.yaml``. A test locks that equality, so the declaration +and the runtime file cannot diverge. + +The overlay is a list of block operations on the base *text* (not on parsed +YAML), so the resolved file preserves the base's exact formatting and comments +-- the property that makes byte-identity with a hand-maintained runtime file +achievable, and the delta legible as a plain diff. Each op: + +* ``drop:`` remove a verbatim block of base text; +* ``replace:`` / ``with:`` swap a verbatim block for new text. + +Every anchor (the ``drop`` block, or a ``replace`` block) must occur **exactly +once** in the base -- zero or multiple matches is a hard error, so an overlay +that has fallen out of sync with the base fails loudly instead of composing +something wrong. ``why`` is prose for the human reader and is ignored here. + +Stdlib + PyYAML only, so it runs inside the sp_validation container with nothing +extra. +""" + +import argparse +import os +import sys + +import yaml + + +def die(msg): + """Abort with a clear, prefixed message on stderr.""" + sys.exit(f"im_compose_mask: {msg}") + + +def _apply_once(text, anchor, replacement, *, kind, i): + """Replace the single occurrence of ``anchor`` in ``text`` with ``replacement``. + + ``anchor`` must occur exactly once; anything else (missing, or ambiguous) + means the overlay no longer matches the base and is a hard error naming the + offending op. + """ + n = text.count(anchor) + if n != 1: + die( + f"op {i} ({kind}): anchor block occurs {n} time(s) in the base, " + "expected exactly 1 -- the overlay is out of sync with the base.\n" + f"--- anchor ---\n{anchor}\n--------------" + ) + return text.replace(anchor, replacement) + + +def compose(base_text, overlay): + """Apply ``overlay['ops']`` to ``base_text`` and return the resolved text.""" + text = base_text + for i, op in enumerate(overlay["ops"]): + if "drop" in op: + text = _apply_once(text, op["drop"], "", kind="drop", i=i) + elif "replace" in op: + if "with" not in op: + die(f"op {i} (replace): missing 'with:' block") + text = _apply_once(text, op["replace"], op["with"], kind="replace", i=i) + else: + die(f"op {i}: needs a 'drop:' or 'replace:'/'with:' block") + return text + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument( + "overlay", + help="overlay yaml declaring base + block ops " + "(e.g. mask_v1.X.9_im_sim.overlay.yaml)", + ) + ap.add_argument( + "-o", + "--output", + help="write resolved config here; default: stdout", + ) + args = ap.parse_args(argv) + + with open(args.overlay) as fh: + overlay = yaml.safe_load(fh) + + # The base path is stated in the overlay, relative to the overlay's own dir, + # so the pair travels together (both live in config/calibration/). + base_path = os.path.join(os.path.dirname(args.overlay), overlay["base"]) + with open(base_path) as fh: + base_text = fh.read() + + resolved = compose(base_text, overlay) + + if args.output: + with open(args.output, "w") as fh: + fh.write(resolved) + else: + sys.stdout.write(resolved) + + +if __name__ == "__main__": + main() From f14774b2df479990d362cc6d2f8c7d143323ffd0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 03:06:10 +0200 Subject: [PATCH 039/160] Restore develop content wrongly reverted by stale base (.gitignore uv.lock, CONTRIBUTING lint gate, calibration/catalog modules) --- .gitignore | 13 +- CONTRIBUTING.md | 25 +- .../calibrate_comprehensive_cat.py | 7 +- scripts/calibration/extract_info.py | 266 +++++++++--------- scripts/calibration/params.py | 3 + src/sp_validation/calibration.py | 16 +- src/sp_validation/catalog.py | 144 +++++++--- src/sp_validation/catalog_builders.py | 16 ++ src/sp_validation/pseudo_cl.py | 29 -- 9 files changed, 303 insertions(+), 216 deletions(-) diff --git a/.gitignore b/.gitignore index f8eec899..82fb2697 100644 --- a/.gitignore +++ b/.gitignore @@ -122,7 +122,7 @@ venv.bak/ # and the lockfile has never been tracked. Ignore it rather than commit a # pinned-dep reproducibility promise the project hasn't made. Flip to tracked # if we decide to pin deps via uv. -uv.lock +# uv.lock is committed — it is the reproducible pin the container installs from. # Spyder project settings .spyderproject @@ -177,13 +177,10 @@ cosmo_inference/cosmosis_config/glass_mocks_v* # repo, so these are script/notebook outputs, not LaTeX-tracked figures. papers/catalog/plots/*.pdf -# felt — track the fiber records (engineering decisions & findings); skip only -# the regenerable index and runtime locks. -.felt/*.db -.felt/*.db-shm -.felt/*.db-wal -.felt/*.lock -.felt/index-sync.* +# felt fiber store: canonical copy lives in ~/loom (git-synced privately); +# .felt here is a machine-local symlink into it. Never track it in this repo. +/.felt/ +/.felt # Claude Code agent worktrees — transient isolated checkouts for background # agents; never tracked. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 35d5e659..00a8d907 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,17 +70,26 @@ undefined names, unused variables, other judgement calls — is printed as a **warning** and never blocks the commit. Judgement-call lint stays out of your way locally; the gate below is where it's enforced. -**`develop` is the gate.** On every push to `develop` and every PR into it, CI -runs the full ruff policy. If it fails, the check goes **red and blocks the -merge**, and the bot tells you what to fix where you already are: - -- **On a PR** → it posts (and keeps updating) a **comment on the PR** listing the - violations (also surfaced as annotations in the CI run). Push a fix and the - comment turns green. +**`develop` is the gate — and on a PR it fixes for you.** On every push to +`develop` and every PR into it, CI runs the full ruff policy. + +- **On a PR from a branch in this repo** → the gate doesn't just report, it + **fixes**: it runs `ruff format` + `ruff check --fix` and **pushes the result + back to your branch as the `github-actions` bot**, then re-checks. If that + cleaned everything, the PR comment goes green (`🤖 autofix pushed …, ruff is + clean`) and there's nothing to do — just `git pull` to pick up the commit. If + anything ruff *won't* safely fix survives (undefined names, unused variables, + other judgement calls), the check stays **red** and the comment lists **only + the residual** — the mechanical stuff is already handled. So you rarely touch + ruff by hand; when you do, it's the real judgement calls. +- **On a PR from a fork** (where CI can't push to your branch) → it posts (and + keeps updating) a **comment on the PR** with the full violation list. Push a + fix and the comment turns green. - **On a direct push to `develop`** (no PR) → it opens (or updates) a single **lint-debt issue assigned to you**, which auto-closes when CI is green. -So: warn while you work, clean before it lands. +So: warn while you work, and for same-repo PRs the gate mostly cleans up after +you before it lands. ## Commit hygiene (notebooks & large files) diff --git a/scripts/calibration/calibrate_comprehensive_cat.py b/scripts/calibration/calibrate_comprehensive_cat.py index fb3ef3ca..2605338f 100644 --- a/scripts/calibration/calibrate_comprehensive_cat.py +++ b/scripts/calibration/calibrate_comprehensive_cat.py @@ -96,8 +96,13 @@ ) # %% +additive_correction = cm.get("additive_correction", True) +if not additive_correction: + print("Additive bias correction disabled (additive_correction: False)") + g_corr_mc, g_uncorr, w, mask_metacal, c, c_err = calibration.get_calibrated_m_c( - gal_metacal + gal_metacal, + additive_correction=additive_correction, ) num_ok = len(g_corr_mc[0]) diff --git a/scripts/calibration/extract_info.py b/scripts/calibration/extract_info.py index 1cbf836f..123c7437 100644 --- a/scripts/calibration/extract_info.py +++ b/scripts/calibration/extract_info.py @@ -32,6 +32,7 @@ import os import sys +import h5py import numpy as np from astropy.io import fits @@ -51,8 +52,11 @@ # ### Create and open output files and directories -make_out_dirs(output_dir, plot_dir, [], verbose=verbose) -stats_file = open_stats_file(plot_dir, stats_file_name) +os.makedirs(output_dir, exist_ok=True) +stats_file = open_stats_file(output_dir, stats_file_name) + +output_shape_cat_stem = output_shape_cat_base +output_ext = output_format # ## 2. Load data # @@ -107,6 +111,8 @@ tile_IDs, path_tile_ID, path_found_ID, path_missing_ID, verbose=verbose ) +print_stats(f"Tiles in input catalogue: {n_found}", stats_file, verbose=verbose) + # ### Load star catalogue if star_cat_path: @@ -146,42 +152,40 @@ verbose=verbose, ) -# #### Refine: Match to valid, unflagged galaxy sample - -# + -# Flags to indicate valid star sample - -m_star = ( - (dd["FLAGS"][ind_star] == 0) - & (dd["IMAFLAGS_ISO"][ind_star] == 0) - & (dd["NGMIX_MCAL_FLAGS"][ind_star] == 0) - & (dd["NGMIX_G1_PSF_ORIG_NOSHEAR"][ind_star] != -10) -) - -ra_star, dec_star, g_star_psf = spv_cat.match_subsample( - dd, - ind_star, - m_star, - [col_name_ra, col_name_dec], - key_PSF_g1, - key_PSF_g2, - n_star_tot, - stats_file, - verbose=verbose, -) -# - + # Flags to indicate valid star sample. + # Star matching, PSF-catalogue output and star metacalibration are all + # diagnostics that require an input star catalogue; the image-simulation + # pipeline has none (star_cat_path is None), so every star-dependent block + # below is guarded and simply skipped for the sims. + + m_star = ( + (dd["FLAGS"][ind_star] == 0) + & (dd["IMAFLAGS_ISO"][ind_star] == 0) + & (dd["NGMIX_MCAL_FLAGS"][ind_star] == 0) + & (dd["NGMIX_G1_PSF_ORIG_NOSHEAR"][ind_star] != -10) + ) -# MKDEBUG: Moved from end of this script + ra_star, dec_star, g_star_psf = spv_cat.match_subsample( + dd, + ind_star, + m_star, + [col_name_ra, col_name_dec], + key_PSF_g1, + key_PSF_g2, + n_star_tot, + stats_file, + verbose=verbose, + ) -# ### Write PSF catalogue with multi-epoch shapes from shape measurement methods + # ### Write PSF catalogue with multi-epoch shapes from shape measurement methods -spv_cat.write_PSF_cat( - f"{output_PSF_cat_base}_{shape}.fits", - ra_star, - dec_star, - g_star_psf[0], - g_star_psf[1], -) + spv_cat.write_PSF_cat( + f"{output_PSF_cat_base}_{shape}.fits", + ra_star, + dec_star, + g_star_psf[0], + g_star_psf[1], + ) # ## Check for objects with invalid PSF @@ -253,8 +257,9 @@ if verbose: print("Writing comprehensive catalogue...") +comprehensive_cat_path = f"{output_shape_cat_stem}_comprehensive_{shape}{output_ext}" spv_cat.write_shape_catalog( - f"{output_shape_cat_base}_comprehensive_{shape}.fits", + comprehensive_cat_path, ra_all, dec_all, iv_w, @@ -265,6 +270,17 @@ add_cols=ext_cols_pre_cal, add_cols_format=add_cols_pre_cal_format, ) + +# Write tile count to HDF5 attributes +if output_ext == ".hdf5": + try: + with h5py.File(comprehensive_cat_path, "a") as hf: + hf.attrs["n_tiles"] = n_found + if verbose: + print(f" Added n_tiles={n_found} to HDF5 attributes") + except Exception as e: + if verbose: + print(f" Warning: could not add n_tiles attribute: {e}") # - do_selection_calibration = False @@ -275,6 +291,7 @@ else: if verbose: print("Continuing with selection and calibration") + os.makedirs(os.path.join(output_dir, plot_dir), exist_ok=True) # ## 4. Select galaxies @@ -628,23 +645,24 @@ # ## Metacalibration for stars -star_metacal = metacal(dd[ind_star], m_star, masking_type="star", verbose=verbose) +if star_cat_path: + star_metacal = metacal(dd[ind_star], m_star, masking_type="star", verbose=verbose) -# #### Number density + # #### Number density -# + -# mask for 'no shear' images + # + + # mask for 'no shear' images -mask_ns_stars = star_metacal.mask_dict["ns"] -n_star = len(star_metacal.ns["g1"][mask_ns_stars]) + mask_ns_stars = star_metacal.mask_dict["ns"] + n_star = len(star_metacal.ns["g1"][mask_ns_stars]) -print_stats(f"Number of stars = {n_star}", stats_file, verbose=verbose) -print_stats( - "Star density = {:.2f} stars/deg2".format(n_star / area_deg2), - stats_file, - verbose=verbose, -) -# - + print_stats(f"Number of stars = {n_star}", stats_file, verbose=verbose) + print_stats( + "Star density = {:.2f} stars/deg2".format(n_star / area_deg2), + stats_file, + verbose=verbose, + ) + # - # ## Additive bias # Use raw, uncorrected ellipticities. @@ -730,20 +748,21 @@ print_stats(rs, stats_file, verbose=verbose) # + -print_stats("stars:", stats_file, verbose=verbose) +if star_cat_path: + print_stats("stars:", stats_file, verbose=verbose) -print_stats("total response matrix:", stats_file, verbose=verbose) -rs = np.array2string(star_metacal.R) -print_stats(rs, stats_file, verbose=verbose) + print_stats("total response matrix:", stats_file, verbose=verbose) + rs = np.array2string(star_metacal.R) + print_stats(rs, stats_file, verbose=verbose) -print_stats("shear response matrix:", stats_file, verbose=verbose) -R_shear_stars = np.mean(star_metacal.R_shear, 2) -rs = np.array2string(R_shear_stars) -print_stats(rs, stats_file, verbose=verbose) + print_stats("shear response matrix:", stats_file, verbose=verbose) + R_shear_stars = np.mean(star_metacal.R_shear, 2) + rs = np.array2string(R_shear_stars) + print_stats(rs, stats_file, verbose=verbose) -print_stats("selection response matrix:", stats_file, verbose=verbose) -rs = np.array2string(star_metacal.R_selection) -print_stats(rs, stats_file, verbose=verbose) + print_stats("selection response matrix:", stats_file, verbose=verbose) + rs = np.array2string(star_metacal.R_selection) + print_stats(rs, stats_file, verbose=verbose) # - # ### Plot distribution of response matrix elements @@ -757,14 +776,11 @@ linestyles = ["-", "-", ":", ":"] # + -labels = ["$R_{11}$ galaxies", "$R_{22}$ galaxies", "$R_{11}$ stars", "$R_{22}$ stars"] - -xs = [ - gal_metacal.R_shear[0, 0], - gal_metacal.R_shear[1, 1], - star_metacal.R_shear[0, 0], - star_metacal.R_shear[1, 1], -] +labels = ["$R_{11}$ galaxies", "$R_{22}$ galaxies"] +xs = [gal_metacal.R_shear[0, 0], gal_metacal.R_shear[1, 1]] +if star_cat_path: + labels += ["$R_{11}$ stars", "$R_{22}$ stars"] + xs += [star_metacal.R_shear[0, 0], star_metacal.R_shear[1, 1]] title = shape out_name = f"R_{shape}_diag.pdf" @@ -779,19 +795,16 @@ x_range, n_bin, out_path, - colors=colors, - linestyles=linestyles, + colors=colors[: len(xs)], + linestyles=linestyles[: len(xs)], ) # + -labels = ["$R_{12}$ galaxies", "$R_{21}$ galaxies", "$R_{12}$ stars", "$R_{21}$ stars"] - -xs = [ - gal_metacal.R_shear[0, 1], - gal_metacal.R_shear[1, 0], - star_metacal.R_shear[0, 1], - star_metacal.R_shear[1, 0], -] +labels = ["$R_{12}$ galaxies", "$R_{21}$ galaxies"] +xs = [gal_metacal.R_shear[0, 1], gal_metacal.R_shear[1, 0]] +if star_cat_path: + labels += ["$R_{12}$ stars", "$R_{21}$ stars"] + xs += [star_metacal.R_shear[0, 1], star_metacal.R_shear[1, 0]] title = shape out_name = f"R_{shape}_offdiag.pdf" out_path = os.path.join(plot_dir, out_name) @@ -805,8 +818,8 @@ x_range, n_bin, out_path, - colors=colors, - linestyles=linestyles, + colors=colors[: len(xs)], + linestyles=linestyles[: len(xs)], ) # - @@ -845,49 +858,50 @@ ) # + -xs = [star_metacal.ns["g1"][mask_ns_stars], star_metacal.ns["g2"][mask_ns_stars]] -weights = [star_metacal.ns["w"][mask_ns_stars]] * 2 - -title = "stars" -out_name = f"ell_stars_{shape}.pdf" -out_path = os.path.join(plot_dir, out_name) - -plot_histograms( - xs, - labels, - title, - x_label, - y_label, - x_range, - n_bin, - out_path, - weights=weights, - colors=colors, - linestyles=linestyles, -) -# - - -x_range = (-0.15, 0.15) -n_bin = 250 - -# + -xs = [dd[key_PSF_g1][mask_ns_stars], dd[key_PSF_g2][mask_ns_stars]] -title = "PSF" -out_name = f"ell_PSF_{shape}.pdf" -out_path = os.path.join(plot_dir, out_name) - -plot_histograms( - xs, - labels, - title, - x_label, - y_label, - x_range, - n_bin, - out_path, - colors=colors, - linestyles=linestyles, -) +if star_cat_path: + xs = [star_metacal.ns["g1"][mask_ns_stars], star_metacal.ns["g2"][mask_ns_stars]] + weights = [star_metacal.ns["w"][mask_ns_stars]] * 2 + + title = "stars" + out_name = f"ell_stars_{shape}.pdf" + out_path = os.path.join(plot_dir, out_name) + + plot_histograms( + xs, + labels, + title, + x_label, + y_label, + x_range, + n_bin, + out_path, + weights=weights, + colors=colors, + linestyles=linestyles, + ) + # - + + x_range = (-0.15, 0.15) + n_bin = 250 + + # + + xs = [dd[key_PSF_g1][mask_ns_stars], dd[key_PSF_g2][mask_ns_stars]] + title = "PSF" + out_name = f"ell_PSF_{shape}.pdf" + out_path = os.path.join(plot_dir, out_name) + + plot_histograms( + xs, + labels, + title, + x_label, + y_label, + x_range, + n_bin, + out_path, + colors=colors, + linestyles=linestyles, + ) # - # ## Magnitudes @@ -951,7 +965,7 @@ # ### Write basic shape catalogue spv_cat.write_shape_catalog( - f"{output_shape_cat_base}_{shape}.fits", + f"{output_shape_cat_stem}_{shape}{output_ext}", ra, dec, w, @@ -990,7 +1004,7 @@ # Extended catalogue with SNR, individual R matrices, ext_cols spv_cat.write_shape_catalog( - f"{output_shape_cat_base}_extended_{shape}.fits", + f"{output_shape_cat_stem}_extended_{shape}{output_ext}", ra, dec, w, @@ -1020,4 +1034,4 @@ ra = dd["RA"][cut_overlap] dec = dd["DEC"][cut_overlap] tile_id = dd["TILE_ID"][cut_overlap] - write_galaxy_cat(f"{output_shape_cat_base}.fits", ra, dec, tile_id) + write_galaxy_cat(f"{output_shape_cat_stem}{output_ext}", ra, dec, tile_id) diff --git a/scripts/calibration/params.py b/scripts/calibration/params.py index 2aa5ffea..ea871119 100644 --- a/scripts/calibration/params.py +++ b/scripts/calibration/params.py @@ -102,6 +102,9 @@ ## Output +### Output file format extension: '.fits' or '.hdf5' +output_format = ".hdf5" + ### Additional output columns add_cols = [ "FLUX_RADIUS", diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index 37ed67b4..e0ab212e 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -56,7 +56,7 @@ def get_calibrated_quantities(gal_metacal): return g_corr, g_uncorr, w, mask -def get_calibrated_m_c(gal_metacal): +def get_calibrated_m_c(gal_metacal, additive_correction=True): """Get Calibrated C. Return catalogue quantities for objects calibrated for multiplicative and @@ -66,6 +66,11 @@ def get_calibrated_m_c(gal_metacal): ---------- gal_metacal : dict galaxy metacalibration catalogue + additive_correction : bool, optional, default=True + if False, do not subtract the additive bias c from the shear + estimates; use for constant-shear image sims, where the mean + shear is the signal (see issue #226). c and c_err are still + computed and returned Returns ------- @@ -99,10 +104,11 @@ def get_calibrated_m_c(gal_metacal): c_err[comp] = np.std(g_uncorr[comp]) # Shear estimate corrected for additive bias - g_corr_mc = np.zeros_like(g_corr) - c_corr = np.linalg.inv(gal_metacal.R).dot(c) - for comp in (0, 1): - g_corr_mc[comp] = g_corr[comp] - c_corr[comp] + g_corr_mc = np.copy(g_corr) + if additive_correction: + c_corr = np.linalg.inv(gal_metacal.R).dot(c) + for comp in (0, 1): + g_corr_mc[comp] = g_corr[comp] - c_corr[comp] return g_corr_mc, g_uncorr, w, mask_metacal, c, c_err diff --git a/src/sp_validation/catalog.py b/src/sp_validation/catalog.py index b1b175d9..e39d2076 100644 --- a/src/sp_validation/catalog.py +++ b/src/sp_validation/catalog.py @@ -12,6 +12,7 @@ """ import getpass +import os import h5py import numpy as np @@ -308,6 +309,35 @@ def match_subsample( return ra, dec, g +def match_catalogs_radec(ra1, dec1, ra2, dec2, thresh_deg=0.0002): + """Match two catalogues by RA/Dec. + + Match each object in catalogue 2 to the nearest in catalogue 1 + within a threshold. + + Parameters + ---------- + ra1, dec1 : array_like + coordinates of reference catalogue [deg] + ra2, dec2 : array_like + coordinates of catalogue to match [deg] + thresh_deg : float, optional + maximum separation [deg], default 0.0002 + + Returns + ------- + idx1 : ndarray of int + indices into catalogue 1 of matched objects + idx2 : ndarray of int + indices into catalogue 2 of matched objects + """ + coord1 = coords.SkyCoord(ra=ra1 * u.degree, dec=dec1 * u.degree) + coord2 = coords.SkyCoord(ra=ra2 * u.degree, dec=dec2 * u.degree) + idx1, sep, _ = coord2.match_to_catalog_sky(coord1) + mask = sep.deg < thresh_deg + return idx1[mask], np.where(mask)[0] + + def match_stars2(ra_gal, dec_gal, ra_star, dec_star, thresh=0.0002): """Add docstring. @@ -542,55 +572,91 @@ def write_shape_catalog( ) ) - # Write columns to FITS file - cols = [] - for col, _ in col_info_arr: - cols.append(col) - table_hdu = fits.BinTableHDU.from_columns(cols) - - # Add human-readable descriptions - for idx, col_info in enumerate(col_info_arr): - table_hdu.header[f"TTYPE{idx + 1}"] = ( - col_info[0].name, - col_info[1], - ) + ext = os.path.splitext(output_path)[1].lower() - # Primary HDU with information in header - primary_header = fits.Header() + if ext in (".hdf5", ".hdf", ".h5"): + # Build flat list of (name, 1d-array) pairs, splitting 2D columns + fields = [] + for col, _ in col_info_arr: + arr = np.asarray(col.array) + if arr.ndim == 2: + for idx in range(arr.shape[1]): + fields.append((f"{col.name}_{idx}", arr[:, idx])) + else: + fields.append((col.name, arr)) + + # Build structured numpy array and write as single "data" dataset + dtype = np.dtype([(name, arr.dtype) for name, arr in fields]) + structured = np.empty(len(fields[0][1]), dtype=dtype) + for name, arr in fields: + structured[name] = arr + + with h5py.File(output_path, "w") as f: + f.create_dataset("data", data=structured) + if add_header: + for key, val in add_header.items(): + f.attrs[key] = str(val) + if all(v is not None for v in (R, R_shear, R_select, c)): + f.attrs["R"] = R + f.attrs["R_shear"] = R_shear + f.attrs["R_select"] = R_select + f.attrs["c"] = c + if c_err is not None: + f.attrs["c1_err"] = c_err[0] + f.attrs["c2_err"] = c_err[1] + if sigma_epsilon is not None: + f.attrs["sig_eps"] = sigma_epsilon + if alpha_leakage is not None: + f.attrs["alpha"] = alpha_leakage - if add_header: - primary_header.update(add_header) + else: + # Write columns to FITS file + cols = [col for col, _ in col_info_arr] + table_hdu = fits.BinTableHDU.from_columns(cols) + + # Add human-readable descriptions + for idx, col_info in enumerate(col_info_arr): + table_hdu.header[f"TTYPE{idx + 1}"] = ( + col_info[0].name, + col_info[1], + ) - primary_header = cat.write_header_info_sp( - primary_header, - software_name="sp_validation", - software_version=__version__, - author=getpass.getuser(), - ) + # Primary HDU with information in header + primary_header = fits.Header() - if all(v is not None for v in (R, R_shear, R_select, c)): - cat.add_shear_bias_to_header(primary_header, R, R_shear, R_select, c) - if c_err is not None: - primary_header["c1_err"] = (c_err[0], "Standard deviation of c_1") - primary_header["c2_err"] = (c_err[1], "Standard deviation of c_2") + if add_header: + primary_header.update(add_header) - primary_header["w"] = "DES weight" + primary_header = cat.write_header_info_sp( + primary_header, + software_name="sp_validation", + software_version=__version__, + author=getpass.getuser(), + ) - if sigma_epsilon is not None: - primary_header["sig_eps"] = (sigma_epsilon, "Shape noise RMS") + if all(v is not None for v in (R, R_shear, R_select, c)): + cat.add_shear_bias_to_header(primary_header, R, R_shear, R_select, c) + if c_err is not None: + primary_header["c1_err"] = (c_err[0], "Standard deviation of c_1") + primary_header["c2_err"] = (c_err[1], "Standard deviation of c_2") - if alpha_leakage: - primary_header["alpha"] = ( - alpha_leakage, - "Mean scale-dependent PSF leakage", - ) + primary_header["w"] = "DES weight" + + if sigma_epsilon is not None: + primary_header["sig_eps"] = (sigma_epsilon, "Shape noise RMS") + + if alpha_leakage: + primary_header["alpha"] = ( + alpha_leakage, + "Mean scale-dependent PSF leakage", + ) - primary_hdu = fits.PrimaryHDU(header=primary_header) + primary_hdu = fits.PrimaryHDU(header=primary_header) - # Final file - hdu_list = fits.HDUList([primary_hdu, table_hdu]) + # Final file + hdu_list = fits.HDUList([primary_hdu, table_hdu]) - hdu_list.writeto(output_path, overwrite=True) + hdu_list.writeto(output_path, overwrite=True) def write_galaxy_cat(output_path, ra, dec, tile_id): diff --git a/src/sp_validation/catalog_builders.py b/src/sp_validation/catalog_builders.py index 1375ef8d..78dc7838 100644 --- a/src/sp_validation/catalog_builders.py +++ b/src/sp_validation/catalog_builders.py @@ -1124,6 +1124,22 @@ def read_cat(self, load_into_memory=False): fpath = self._params["input_path"] verbose = self._params["verbose"] + # Image-simulation path: a single per-run comprehensive catalogue in + # FITS, not the joined multi-patch HDF5 the data path builds. Read the + # FITS table directly into memory; there is no separate data_ext group. + extension = os.path.splitext(fpath)[1] + if extension == ".fits": + if verbose: + print(f"Reading FITS file {fpath}, HDU 1...") + dat = fits.getdata(fpath, 1) + dat_ext = None + if verbose: + print( + f"Found {len(dat)} (~{format.millify(len(dat))}) objects" + + " in catalogue" + ) + return dat, dat_ext + if verbose: print(f"Reading HDF5 file {fpath}...") diff --git a/src/sp_validation/pseudo_cl.py b/src/sp_validation/pseudo_cl.py index 34cbc68d..c9355ec9 100644 --- a/src/sp_validation/pseudo_cl.py +++ b/src/sp_validation/pseudo_cl.py @@ -280,32 +280,3 @@ def get_pseudo_cls_catalog( cl_all = wsp.decouple_cell(cl_coupled) return ell_eff, cl_all, wsp - - -# NaMaster spin-2 × spin-2 spectrum order: EE, EB, BE, BB. -_NMT_EE = 0 - - -def bandpower_window_from_workspace(wsp): - """Extract the bandpower window matrix ``W`` for a spin-2×spin-2 workspace. - - NaMaster's ``get_bandpower_windows()`` returns a four-index array - ``(n_cl_out, n_bpw, n_cl_in, n_ell)`` describing how each output bandpower - is built from the input multipoles across the EE/EB/BE/BB spectra. SACC's - ``BandpowerWindow`` model (one window per bandpower, shared across the - stored spectra) needs the per-spectrum *decoupling* window, i.e. the - diagonal EE←EE block (equal to BB←BB and EB←EB, verified identical). - - Returns - ------- - window_ells : np.ndarray - Multipoles the window spans, ``arange(n_ell)`` — the ``ell`` axis of - ``compute_coupled_cell``. - window_weights : np.ndarray - ``W`` of shape ``(n_ell, n_bpw)`` — one column per bandpower, the layout - :func:`sp_validation.sacc_io.add_pseudo_cl` expects. - """ - bpw = wsp.get_bandpower_windows() # (n_cl_out, n_bpw, n_cl_in, n_ell) - diagonal = bpw[_NMT_EE, :, _NMT_EE, :] # (n_bpw, n_ell) - window_ells = np.arange(diagonal.shape[1], dtype=float) - return window_ells, diagonal.T From 7e1639899c3aa01d4a3704f35a956000db186ad2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 03:27:40 +0200 Subject: [PATCH 040/160] simplify: trim comments to Google style; dedupe helpers; drop dead knobs Comment/docstring pass over the PR-4 diff: cut PR/issue archaeology, restated defaults and "identical/bit-matching" claims (not true under blinding), keeping the load-bearing contracts (canonical part order, placeholder-cov consequence, scale_cut semantics, TreeCorr bin edges, CovTauTh k-major plus-folded layout). Code: - common.py gains base_version() and pseudo_cl_tag(); generate_cosmocov_ini imports both instead of copying build_redshift_path, fixing the drift where the cov_th lookup stripped only _leak_corr while the n(z) path also stripped _ecut{N}. cosmo_val.smk / inference.smk share the one pseudo-Cl tag. - run_cosmocov_chain.sh anchors the checkout on its own location; container, bind list and cosmocov binary come from the environment. - assemble_sacc: drop the unused --pseudo-cl-cov-hdu knob (the three named HDUs are always written; missing ones now raise). - assemble_analysis_sacc seeds tracers + metadata from parts[0], retiring _n_source_bins and the nz/metadata arguments. - Covariance mode derives from the grid: XI_GRIDS loses its "covariance" field, xi_grid_of returns just the label and compares binnings numerically, and run_2pcf loses its --covariance flag. - inference.smk: drop the dead inference_prep / inference_fiducial rules. Also restores bandpower_window_from_workspace (deleted in error by f14774b2) and develop's NON_PATH_KEYS overlay entries in test_config_paths_exist. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014hSQTC6WwTH1p9w4FuKJGz --- cosmo_inference/README.md | 14 +- papers/bmodes/scripts/run_xi_sweep.py | 10 +- papers/cosmo_val/config/config.yaml | 9 +- pyproject.toml | 5 +- src/sp_validation/b_modes.py | 37 ++-- src/sp_validation/cosmo_val/cosebis.py | 15 +- src/sp_validation/cosmo_val/pseudo_cl.py | 17 +- .../cosmo_val/psf_systematics.py | 9 +- src/sp_validation/cosmo_val/pure_eb.py | 9 +- src/sp_validation/cosmo_val/real_space.py | 16 +- src/sp_validation/cosmo_val/sacc_writers.py | 66 +++---- src/sp_validation/pseudo_cl.py | 29 +++ .../tests/test_bmodes_workflow_dry_run.py | 28 +-- src/sp_validation/tests/test_cli_seams.py | 21 +- .../tests/test_config_paths_exist.py | 9 +- src/sp_validation/tests/test_pseudo_cl.py | 21 +- src/sp_validation/tests/test_sacc_writers.py | 6 +- workflow/common.py | 57 +++--- workflow/rules/cosmo_val.smk | 101 ++++------ workflow/rules/inference.smk | 116 ++--------- workflow/rules/twopoint.smk | 74 +++---- workflow/scripts/assemble_sacc.py | 186 +++++------------- workflow/scripts/cv_cosebis.py | 12 +- workflow/scripts/cv_pseudo_cl.py | 5 +- workflow/scripts/cv_pure_eb.py | 14 +- workflow/scripts/generate_cosmocov_ini.py | 40 ++-- workflow/scripts/generate_pseudo_cl.py | 29 +-- workflow/scripts/run_2pcf.py | 37 ++-- workflow/scripts/run_cosmocov_chain.sh | 39 ++-- workflow/scripts/run_rho_tau.py | 5 +- 30 files changed, 360 insertions(+), 676 deletions(-) diff --git a/cosmo_inference/README.md b/cosmo_inference/README.md index 5d753010..aa52e57f 100644 --- a/cosmo_inference/README.md +++ b/cosmo_inference/README.md @@ -7,17 +7,9 @@ This folder contains the files neccessary to run the cosmological inference pipe To run the pipeline, one would need to have installed [CosmoSIS](https://cosmosis.readthedocs.io/en/latest/). To sample the PSF leakage parameters, the fork of [cosmosis-standard-library](https://github.com/sachaguer/cosmosis-standard-library/) of Sacha Guerrini has to be used. ### To Run -The inference pipeline is now orchestrated through Python. Run the main Snakemake workflow from the parent directory: - -```bash -snakemake -j inference_fiducial -``` - -This will automatically execute all steps: -1. Calculate 2PCF ($\xi_{pm}$) via `cosmo_val.py` -2. Compute covariance matrices using CosmoCov -3. Prepare CosmoSIS data (FITS) via `cosmosis_fitting.py` -4. Run CosmoSIS inference +The `inference_fiducial` Snakemake target is dormant: the SACC migration +removed the per-product FITS its `inference_prep` rule consumed, and rewiring +inference onto the assembled `{version}.sacc` is tracked separately. For standalone FITS data preparation (real-space inputs plus optional pseudo-$C_\ell$ data), you can also use the Python script directly: diff --git a/papers/bmodes/scripts/run_xi_sweep.py b/papers/bmodes/scripts/run_xi_sweep.py index 5e411c34..a4674cb5 100644 --- a/papers/bmodes/scripts/run_xi_sweep.py +++ b/papers/bmodes/scripts/run_xi_sweep.py @@ -38,9 +38,7 @@ GRIDS = { "reporting": dict(min_sep=1.0, max_sep=250.0, nbins=20, npatch=1), - "integration": dict( - min_sep=0.5, max_sep=300.0, nbins=1000, npatch=1, covariance="diagonal" - ), + "integration": dict(min_sep=0.5, max_sep=300.0, nbins=1000, npatch=1), } @@ -74,10 +72,8 @@ def _from_cli(argv=None): versions = a.versions or nonfiducial_versions(config) for ver in versions: for grid in a.grids: - # The sweep consumes only the .txt dump (cosebis_version_comparison - # reconstructs it by binning). run_2pcf is born-as-SACC; its default - # part name carries the binning, so the two grids per version land - # in distinct files without an explicit sacc_out. + # The sweep consumes only the .txt dump; the SACC part's default + # name carries the binning, so the two grids land in distinct files. run_2pcf( ver=ver, cat_config=a.cat_config, diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index fac4cdbb..1e528850 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -58,12 +58,9 @@ cosmo_val: kmax: 20 kmax_extrapolate: 500 - # Integration-grid ξ± (the shared fine grid measured once per version by the - # binning-agnostic `xi` rule, which resolves this binning to grid='integration' - # and attaches a DiagonalCovariance). Both estimators consume this one - # part: pure-E/B uses the full range (it must strictly contain the reporting - # grid [1, 250]); COSEBIs scale-cuts it up to 0.9. Owned here, not inside either - # consumer's block. Decoupled from covariance.smk's own FIDUCIAL grid. + # Integration-grid ξ±: the shared fine grid both B-mode estimators consume. + # Pure-E/B uses the full range, which must strictly contain the reporting grid + # [1, 250]; COSEBIs scale-cuts it up to 0.9. integration: min_sep: 0.08 max_sep: 300 diff --git a/pyproject.toml b/pyproject.toml index 75e91dd2..a7385d3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,9 +79,8 @@ dependencies = [ "pymaster", "regions", "reproject", - # sp_validation.sacc_io (assembled on this branch) uses concatenate_data_sets - # and BlockDiagonalCovariance, both from sacc's 2.x rewrite; the lock already - # resolves to 2.4, this just makes the floor honest. + # sacc_io uses BlockDiagonalCovariance, from sacc's 2.x rewrite; the lock + # already resolves to 2.4, this just makes the floor honest. "sacc>=2.4,<3", # scipy 1.18 ported FITPACK from Fortran to C, changing the return shape of # RectBivariateSpline(scalar, scalar, grid=False) from 0-d `array(x)` to diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index 2b7a4e96..f843007d 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -258,20 +258,12 @@ def pure_EB(corrs): def cosebis_from_xi(theta, xip, xim, nmodes, scale_cut=None): """COSEBIs (Eₙ, Bₙ) from ξ± arrays through the pipeline kernel (values only). - The values-only seam of :func:`calculate_cosebis`, for callers holding - ξ± arrays rather than a TreeCorr ``GGCorrelation`` — e.g. deriving COSEBIs - from an integration-ξ± SACC part. Calls the same ``cosmo_numba`` kernel - (``COSEBIS.cosebis_from_xipm``) directly on the values; the covariance/χ² - machinery stays with :func:`calculate_cosebis`. - - ``scale_cut`` follows the :func:`sacc_io.add_cosebis` writer contract: - ``(theta_min, theta_max)`` are min/max of the *retained* bin centres - after the pipeline's ``scale_cut_to_bins``. The cut is contiguous in an - ascending grid, so selecting ``theta_min ≤ θ ≤ theta_max`` inclusively - reproduces exactly the retained set, and the kernel is built on that - set's min/max support and fed only the retained ξ± — bit-matching - :func:`calculate_cosebis`'s ``theta_cut``/``xip_cut``/``xim_cut`` path. - Identical inputs ⇒ identical numbers. + The values-only seam of :func:`calculate_cosebis`, for callers holding ξ± + arrays rather than a TreeCorr ``GGCorrelation``; the covariance/χ² machinery + stays with :func:`calculate_cosebis`. + + ``scale_cut`` follows the :func:`sacc_io.add_cosebis` contract: ``(theta_min, + theta_max)`` are min/max of the *retained bin centres*, selected inclusively. """ from cosmo_numba.B_modes.cosebis import COSEBIS @@ -294,18 +286,13 @@ def pure_eb_from_xi( ): """Pure-E/B correlation functions from ξ± arrays through the pipeline kernel. - The values-only seam of :func:`calculate_pure_eb_correlation`, for - callers holding ξ± arrays rather than TreeCorr correlations — e.g. - deriving pure-E/B from SACC parts. Calls the same ``cosmo_numba`` kernel - (``get_pure_EB_modes``) directly on the values. - The reporting grid must be a strict sub-range of the integration grid; + The values-only seam of :func:`calculate_pure_eb_correlation`, for callers + holding ξ± arrays rather than TreeCorr correlations. + ``tmin``/``tmax`` are the reporting correlation's TreeCorr *bin edges* - (``gg.left_edges[0]`` / ``gg.right_edges[-1]``) — the pipeline's - convention, carried on SACC files by ``sacc_io.add_pure_eb``. A - reporting point coinciding with the integration boundary is degenerate - (no interior support) and comes back NaN, exactly as - :func:`calculate_pure_eb_correlation` returns it — never a spurious - finite value. + (``gg.left_edges[0]`` / ``gg.right_edges[-1]``). The reporting grid must be a + strict sub-range of the integration grid: a reporting point on the + integration boundary has no interior support and comes back NaN. Returns ------- diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index 406f0a11..2f472fce 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -169,16 +169,13 @@ def cosebis_to_sacc_part( ): """Write the COSEBIs SACC part at the fiducial scale cut. - ``results`` is the object ``calculate_cosebis`` returned (single dict or + ``results`` is what ``calculate_cosebis`` returned (single dict or multi-cut mapping). Only the fiducial cut's ``{En, Bn, cov}`` becomes the - part — a ``FullCovariance`` must cover every stored point and the cuts - overlap in mode space, so the non-fiducial cuts stay in the diagnostic - ``.npz`` sidecar. The nz/metadata are the version's. - - ``en_override`` is the consume-the-part plumbing: the E-mode ``En`` written - to the part in place of ``result["En"]`` — re-derived from the integration - ξ± SACC part at the fiducial scale cut (Bn and the covariance stay from the - raw estimator ``result``). With ``None`` the behaviour is unchanged. + part: the covariance must cover every stored point and the cuts overlap + in mode space, so the non-fiducial cuts stay in the ``.npz`` sidecar. + + ``en_override`` replaces ``result["En"]`` with En derived from the + integration ξ± part; Bn and the covariance stay from ``result``. """ result, scale_cut = self._fiducial_cosebis_result(results, fiducial_scale_cut) if en_override is not None: diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 2a55934a..ab871448 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -465,15 +465,10 @@ def calculate_pseudo_cl(self, out_path=None): the ``ELL``/``EE``/``EB``/``BB`` arrays the plotting and B-mode-summary consumers read by column name. - ``out_path`` is the exact destination the part is *born at* — the - Snakemake-declared output. It must resolve per version; single-version - rules (the tagged blinded producer) pass their tagged output directly. - When ``None`` (multi-version diagnostic / the ``pseudo_cls`` property) - each part defaults to the untagged native ``pseudo_cl_{ver}.sacc``. + ``out_path`` is the exact destination the part is born at (one version + only); ``None`` defaults each part to ``pseudo_cl_{ver}.sacc``. Skip-if-exists keys on this final path, so no two rules ever share an - undeclared native basename (a tagged product born at its native name and - then renamed would let one rule's skip-if-exists silently adopt — and - the rename delete — another rule's declared, differently-blinded file). + undeclared native basename. """ self.print_start("Computing pseudo-Cl's") @@ -514,10 +509,8 @@ def calculate_pseudo_cl(self, out_path=None): @staticmethod def _load_pseudo_cl_sacc(out_path): """Read a pseudo-Cl SACC part into the ELL/EE/EB/BB dict consumers use.""" - # Pipeline-internal readback of a part this producer just wrote: the - # born-as-SACC parts are unblinded real-data measurements (blinding is a - # downstream Smokescreen step), so the fail-closed load must be told this - # is a legitimate pre-blind consumer. + # Readback of a part this producer just wrote — a legitimate pre-blind + # consumer, so the fail-closed load is opted out of. s = sacc_io.load(out_path, allow_unblinded=True) ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(s, SACC_BIN) return {"ELL": ell, "EE": ee, "EB": eb, "BB": bb} diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index 6f72ac1f..b20d1085 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -57,12 +57,9 @@ def rho_tau_to_sacc_part( """Write the ρ/τ SACC part for one version. ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage from the handler tables. The - ``CovTauTh`` theory covariance ``cov_tau_{base}_th.npy`` — a - ``(3·nbin, 3·nbin)`` plus-folded k-major block over ``{τ0, τ2, τ5}`` — is - passed as ``tau_cov_th`` when it exists (the τ-plus inference block); its - absence falls back to a diagonal placeholder for the whole part (loudly: - the τ inference block is then only a variance diagonal, not the theory - covariance). ρ always carries a diagnostic ``varrho`` diagonal. + ``CovTauTh`` theory covariance ``cov_tau_{base}_th.npy`` is passed as + ``tau_cov_th`` when it exists; without it the τ block falls back — + loudly — to a variance diagonal. """ tau_cov_path = os.path.join(out_dir, f"cov_tau_{base}_th.npy") tau_cov_th = np.load(tau_cov_path) if os.path.exists(tau_cov_path) else None diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 29e1f888..d3126af7 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -140,13 +140,10 @@ def pure_eb_to_sacc_part(self, version, out_path, results, eb_override=None): ``results`` is the dict ``calculate_pure_eb`` returned: the six pure-mode arrays under ``sacc_io.PURE_KEYS``, the ``"cov"`` block (in ``PURE_KEYS`` order), and the reporting-grid TreeCorr object ``"gg"`` whose ``meanr`` - is the shared ``theta``. + is the shared ``theta``. The covariance must cover every stored point. - ``eb_override`` is the consume-the-part plumbing: the six pure-mode arrays - (a mapping keyed by ``sacc_io.PURE_KEYS``) written in place of ``results``' - — re-derived from the reporting + integration ξ± SACC parts (the covariance - stays blind-invariant from the raw estimator ``results``). With ``None`` the - behaviour is unchanged. + ``eb_override`` replaces the six arrays with ones derived from the + reporting + integration ξ± parts; the covariance stays from ``results``. """ theta = results["gg"].meanr source = eb_override if eb_override is not None else results diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index ce3b7ecc..e12ddbbe 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -46,11 +46,8 @@ def calculate_2pcf(self, ver, npatch=None, **treecorr_config): - If a patch file for the given configuration does not exist, it is created during the process. - The ``.txt`` TreeCorr dump is the only raw byproduct written here - (read back by the covariance machinery and the skip-if-exists). The - analysis ξ± data product is born as SACC in ``run_2pcf.py`` (one - binning-agnostic driver for both the reporting and the fine - integration grid), which calls ``xi_to_sacc``; there is no - DES-style ξ FITS writer anymore. + (read back by the covariance machinery and the skip-if-exists); the + ξ± data product is born as SACC in ``run_2pcf.py``. """ self.print_magenta(f"Computing {ver} ξ±") @@ -99,11 +96,10 @@ def calculate_2pcf(self, ver, npatch=None, **treecorr_config): # Process the catalog & write the correlation functions gg.process(cat_gal) - # Patch results + the jackknife covariance only make sense (and are - # only affordable) for npatch > 1: at npatch=1 var_method is "shot", - # so the "covariance" is just the varxip/varxim already written as - # per-bin columns, while the dense (2*nbins)^2 block would dominate - # the file on the fine integration grid (nbins ~ 1000). + # Patch results and the jackknife covariance need npatch > 1: at + # npatch=1 var_method is "shot", so the covariance adds nothing over + # the varxip/varxim columns while the dense (2*nbins)^2 block would + # dominate the file on the fine integration grid (nbins ~ 1000). write_cov = int(npatch) > 1 gg.write(out_fname, write_patch_results=write_cov, write_cov=write_cov) diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index de5fddf3..de8e425c 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -4,26 +4,17 @@ TreeCorr / NaMaster / b_modes arrays) and :mod:`sp_validation.sacc_io` (which knows the file layout). Each ``*_to_sacc`` function turns one already-computed statistic into a single-statistic SACC — a *part* — carrying that statistic's -own covariance as its one covariance block. The Snakemake DAG writes one part -per rule; :func:`assemble_analysis_sacc` then loads the parts and rebuilds the -single ``{version}.sacc`` analysis file with a ``BlockDiagonalCovariance`` -assembled from the per-part blocks in canonical order (per the SACC layout -contract, via the validated :func:`sp_validation.sacc_io.assemble_covariance` — -*not* ``sacc.concatenate_data_sets``, whose unvalidated block-diagonal the -contract rules out). - -The integration-grid ``{version}_xi_integration.sacc`` is an intermediate -per-part file (:func:`xi_to_sacc` with ``grid="integration"`` and a -``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim``); COSEBIs and pure-E/B -consume it. It is blinded at birth on data runs (per PR #253) but does not join -the terminal ``{version}.sacc`` — Snakemake provenance covers its traceability -(see #247 ruling). +own covariance as its one block. The Snakemake DAG writes one part per rule; +:func:`assemble_analysis_sacc` then rebuilds the single ``{version}.sacc`` +analysis file with a ``BlockDiagonalCovariance`` over the per-part blocks in +canonical order. Everything here is single-bin today (``bins=(0, 0)``); the interface is tomography-native so a future round supplies real bin pairs unchanged. """ import numpy as np +import sacc from .. import sacc_io as sio from ..pseudo_cl import bandpower_window_from_workspace @@ -132,22 +123,13 @@ def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov_th=None): """One ρ/τ part: ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage. ``rho_stats`` / ``tau_stats`` are the ``shear_psf_leakage`` handler tables - (columns ``theta``, ``rho_{k}_p``, ``varrho_{k}_p``, ``rho_{k}_m``, … and - the τ analogue). Both diagnostics stay out of the blind and only τ enters - inference, so the covariance is a block-diagonal placeholder except for the - τ-plus theory block: - - - ρ (all 6·nbin points): diagonal from ``varrho`` — a diagnostic placeholder, - not consumed by inference. - - τ (6·nbin points, per-k ``[τ+; τ−]``): the ``CovTauTh`` theory covariance - ``tau_cov_th`` scattered into the τ-plus rows/columns. ``CovTauTh.build_cov`` - returns a ``(3·nbin, 3·nbin)`` k-major matrix over ``{τ0, τ2, τ5}`` with the - plus/minus contributions folded into one component per k (verified against - the write-side); it therefore aligns to our τ-plus points ``{τ0+, τ2+, τ5+}`` - in k-major order, and today's CosmoSIS chain (``covdat_to_fits``) consumes - exactly this flavor for τ. The τ-minus points carry only a ``vartau`` - diagonal (no theory covariance for them exists). ``tau_cov_th=None`` falls - back to a fully diagonal τ block (a flagged placeholder, not the design). + (columns ``theta``, ``rho_{k}_p``, ``varrho_{k}_p``, … and the τ analogue). + ρ carries a ``varrho`` diagonal (a diagnostic, not consumed by inference); + τ carries a ``vartau`` diagonal with ``tau_cov_th`` scattered into the τ-plus + rows/columns. ``CovTauTh.build_cov`` returns a ``(3·nbin, 3·nbin)`` k-major + matrix over ``{τ0, τ2, τ5}`` with plus/minus folded into one component per k, + so it aligns to the τ-plus points in k-major order; the τ-minus points have + no theory covariance. ``tau_cov_th=None`` leaves the τ block fully diagonal. """ s = sio.new_sacc(nz, metadata) theta_rho = np.asarray(rho_stats["theta"]) @@ -183,8 +165,7 @@ def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov_th=None): ] ) if tau_cov_th is None: - # Fully diagonal placeholder — a DiagonalCovariance (compact, honest) for - # the standalone diagnostic file; assemble reads it back via .dense. + # Compact DiagonalCovariance; assembly reads it back via .dense. s.add_covariance(np.concatenate([rho_var, tau_var])) return s tau_cov_th = np.asarray(tau_cov_th) @@ -218,21 +199,20 @@ def _copy_data_points(dst, src): dst.add_data_point(dp.data_type, dp.tracers, dp.value, **dp.tags) -def assemble_analysis_sacc(nz, metadata, parts): +def assemble_analysis_sacc(parts): """Rebuild the single ``{version}.sacc`` analysis file from parts. Each part is a single-statistic Sacc (from a ``*_to_sacc`` writer, loaded - from disk) carrying its own covariance = its block. This re-adds every - part's data points into one Sacc in the order the parts are given — which - must be the canonical order (ξ± reporting, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ) - — and assembles a single ``BlockDiagonalCovariance`` from the per-part covariance - blocks. Point insertion order and block order therefore agree by - construction, which ``sacc_io.assemble_covariance`` validates (contiguous, - tiling, square) and raises on if they don't. + from disk) carrying its own covariance = its block. Tracers and metadata are + seeded from ``parts[0]`` (every part describes the same catalogue version). + Data points are re-added in the order the parts are given, which must be the + canonical order (ξ± reporting, pseudo-Cℓ, COSEBIs, pure-E/B, ρ/τ), and the + per-part blocks become one ``BlockDiagonalCovariance``. Insertion order and + block order therefore agree by construction — validated by + :func:`sp_validation.sacc_io.assemble_covariance`. Parameters ---------- - nz, metadata : see :func:`sp_validation.sacc_io.new_sacc`. parts : sequence of sacc.Sacc Single-statistic parts, each with a covariance, in canonical order. @@ -241,7 +221,9 @@ def assemble_analysis_sacc(nz, metadata, parts): sacc.Sacc The analysis Sacc with a ``BlockDiagonalCovariance`` covering every point. """ - s = sio.new_sacc(nz, metadata) + s = sacc.Sacc() + s.tracers.update(parts[0].tracers) + s.metadata.update(parts[0].metadata) blocks = [] cursor = 0 for part in parts: diff --git a/src/sp_validation/pseudo_cl.py b/src/sp_validation/pseudo_cl.py index c9355ec9..34cbc68d 100644 --- a/src/sp_validation/pseudo_cl.py +++ b/src/sp_validation/pseudo_cl.py @@ -280,3 +280,32 @@ def get_pseudo_cls_catalog( cl_all = wsp.decouple_cell(cl_coupled) return ell_eff, cl_all, wsp + + +# NaMaster spin-2 × spin-2 spectrum order: EE, EB, BE, BB. +_NMT_EE = 0 + + +def bandpower_window_from_workspace(wsp): + """Extract the bandpower window matrix ``W`` for a spin-2×spin-2 workspace. + + NaMaster's ``get_bandpower_windows()`` returns a four-index array + ``(n_cl_out, n_bpw, n_cl_in, n_ell)`` describing how each output bandpower + is built from the input multipoles across the EE/EB/BE/BB spectra. SACC's + ``BandpowerWindow`` model (one window per bandpower, shared across the + stored spectra) needs the per-spectrum *decoupling* window, i.e. the + diagonal EE←EE block (equal to BB←BB and EB←EB, verified identical). + + Returns + ------- + window_ells : np.ndarray + Multipoles the window spans, ``arange(n_ell)`` — the ``ell`` axis of + ``compute_coupled_cell``. + window_weights : np.ndarray + ``W`` of shape ``(n_ell, n_bpw)`` — one column per bandpower, the layout + :func:`sp_validation.sacc_io.add_pseudo_cl` expects. + """ + bpw = wsp.get_bandpower_windows() # (n_cl_out, n_bpw, n_cl_in, n_ell) + diagonal = bpw[_NMT_EE, :, _NMT_EE, :] # (n_bpw, n_ell) + window_ells = np.arange(diagonal.shape[1], dtype=float) + return window_ells, diagonal.T diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index 136be047..3b7eecae 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -33,14 +33,11 @@ def _repo_root() -> Path: def _dry_run(workflow_dir, targets, *extra_snakemake_args): """Construct a dry run of the paper workflow at ``workflow_dir``. - Returns the CompletedProcess. PYTHONUNBUFFERED satisfies the Snakefile's - ``envvars:`` declaration without depending on the invoking shell. A dry run - resolves the DAG only — it never dispatches jobs — so drop any inherited - SNAKEMAKE_PROFILE (e.g. the login shell's "slurm" profile), which would - otherwise force an executor plugin the test environment need not have. And - invoke snakemake through sys.executable (the interpreter pytest, hence - snakemake, lives in) — a bare python3.12 resolves off PATH to e.g. an - intel-python without snakemake. + PYTHONUNBUFFERED satisfies the Snakefile's ``envvars:`` declaration. A dry + run never dispatches jobs, so any inherited SNAKEMAKE_PROFILE is dropped + rather than requiring its executor plugin. snakemake is invoked through + sys.executable, since a bare python3.12 may resolve off PATH to an + interpreter without it. """ env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} env.pop("SNAKEMAKE_PROFILE", None) @@ -77,24 +74,15 @@ def test_bmodes_workflow_dry_runs(): @requires_candide_data def test_cosmo_val_workflow_assemble_dry_runs(): """The cosmo_val workflow (the only one including cosmo_val.smk) resolves the - born-as-SACC + assemble DAG, and assemble pulls the tagged pseudo-Cl + cov. - - Targets the assemble_sacc_all rule so every version's assemble_sacc job - appears. The dry run resolves the DAG structure only — it never executes the - assemble script — so the placeholder-cov opt-in (cosmo_val.allow_placeholder_cov) - is irrelevant here; a real run would need it (or a wired --xi-cov) to proceed, - which is the fail-loud-by-default behaviour asserted in test_assemble_sacc.""" + born-as-SACC + assemble DAG, and assemble pulls the tagged pseudo-Cl + cov.""" version = "SP_v1.4.6.3_leak_corr" result = _dry_run(_repo_root() / "papers/cosmo_val", ["assemble_sacc_all"]) assert result.returncode == 0, result.stdout - # assemble_sacc must be in the DAG and pull the tagged, blinded pseudo-Cl - # part + its NaMaster covariance (not the untagged cv_pseudo_cl diagnostic), - # plus all five per-statistic parts. + # assemble_sacc must pull the tagged pseudo-Cl part + its NaMaster + # covariance (not the untagged cv_pseudo_cl diagnostic), plus every part. out = result.stdout assert "rule assemble_sacc:" in out, out assert f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" in out, out assert f"pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits" in out, out - # The ξ± part is named by its reporting binning (one binning-agnostic `xi` - # rule serves the reporting and integration grids alike). for part in ("_xi_minsep=", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): assert part in out, f"missing {part} part in assemble DAG:\n{out}" diff --git a/src/sp_validation/tests/test_cli_seams.py b/src/sp_validation/tests/test_cli_seams.py index 4eaaa2de..ce601d25 100644 --- a/src/sp_validation/tests/test_cli_seams.py +++ b/src/sp_validation/tests/test_cli_seams.py @@ -1,11 +1,9 @@ """Smoke tests for workflow CLI seams — cheap guards against signature rot. -A CLI script that calls a workflow function with a removed/renamed kwarg -TypeErrors only at invocation time (the compute is cluster-only, so it is never -exercised by the fast suite). These tests bind the exact call each seam makes -against the current signature via ``inspect.signature(...).bind(...)`` — no -compute, no data — so a drifted kwarg (e.g. run_xi_sweep's dropped save_fits) -fails here instead of on the cluster. +The compute these scripts drive is cluster-only, so a removed or renamed kwarg +would only TypeError at invocation. Each test binds the exact call one seam +makes against the current signature (``inspect.signature(...).bind(...)``) — no +compute, no data — so the drift fails here instead of on the cluster. """ import importlib.util @@ -30,17 +28,11 @@ def _load(path, name): def test_run_xi_sweep_run_2pcf_call_binds(): - """The kwargs run_xi_sweep passes to run_2pcf must bind to its signature. - - Mirrors the call in papers/bmodes/scripts/run_xi_sweep.py — if run_2pcf drops - or renames a parameter (save_fits was removed by the SACC migration), the - bind raises TypeError here rather than on every cluster invocation. - """ + """The kwargs run_xi_sweep passes to run_2pcf must bind to its signature.""" root = _repo_root() run_2pcf_mod = _load(root / "workflow/scripts/run_2pcf.py", "run_2pcf_seam") sig = inspect.signature(run_2pcf_mod.run_2pcf) - # Exactly the keyword set run_xi_sweep._from_cli passes (grid params spread - # from GRIDS: min_sep/max_sep/nbins/npatch, plus covariance on the fine grid). + # Exactly the keyword set run_xi_sweep._from_cli passes. sig.bind( ver="V", cat_config="/cfg.yaml", @@ -50,7 +42,6 @@ def test_run_xi_sweep_run_2pcf_call_binds(): max_sep=300.0, nbins=1000, npatch=1, - covariance="diagonal", ) # And the removed kwarg must NOT bind (guards against a silent re-add). with pytest.raises(TypeError): diff --git a/src/sp_validation/tests/test_config_paths_exist.py b/src/sp_validation/tests/test_config_paths_exist.py index 9ae74ca0..2db03bc1 100644 --- a/src/sp_validation/tests/test_config_paths_exist.py +++ b/src/sp_validation/tests/test_config_paths_exist.py @@ -23,7 +23,14 @@ "catalog", "catalogue", ) -NON_PATH_KEYS = ("extra_output",) +# Keys whose values are never filesystem paths to check. ``extra_output`` is a +# flag, not a path. ``why``/``replace``/``with``/``drop`` are the declaration +# keys of a mask *overlay* (config/calibration/*.overlay.yaml): ``why`` is +# rationale prose and ``replace``/``with``/``drop`` are verbatim blocks of base +# config text -- content, not paths -- so the path walker must not treat them as +# files to stat. (The overlay's one real path, ``base:``, is deliberately not +# listed, so it is still validated.) +NON_PATH_KEYS = ("extra_output", "why", "replace", "with", "drop") PATH_PREFIX_KEYS = ("nz.dndz.path",) TEXT_SUFFIXES = ( ".fits", diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 8c095a71..6fa15d0e 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -514,10 +514,7 @@ def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): """End-to-end catalog path: SACC round-trip of ell + EE/EB/BB. The catalog method has no random noise debiasing, so it is reproducible to - the same ~2e-12 catalog-path float noise. calculate_pseudo_cl_catalog is - born-as-SACC: it writes a pseudo-Cl part (EE/BB/EB + shared bandpower - window) via pseudo_cl_to_sacc_part; we pin the round-tripped spectra read - back through sacc_io.get_pseudo_cl. + the same ~2e-12 catalog-path float noise; we pin the round-tripped spectra. """ ver = cv._test_version cv._pseudo_cls = {ver: {}} @@ -525,12 +522,9 @@ def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): cv.calculate_pseudo_cl_catalog(ver, out_path) assert os.path.exists(out_path) - # The born-as-SACC part is unblinded type='data'; reading it back for the - # round-trip assertion is a pre-blind consumer. s = sacc_io.load(out_path, allow_unblinded=True) ell, ee, bb, eb, window = sacc_io.get_pseudo_cl(s, SACC_BIN) - # A shared BandpowerWindow rides the part per the SACC layout contract. - assert window is not None + assert window is not None # the shared BandpowerWindow rides the part npt.assert_allclose( ell, @@ -598,15 +592,8 @@ def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): def test_calculate_pseudo_cl_out_path_born_at_declared_name(cv): - """calculate_pseudo_cl(out_path=...) writes to the given path, not the - untagged native name — the anti-collision seam. - - The tagged producer (rule pseudo_cl, blind=A) and the untagged diagnostic - (rule cv_pseudo_cl, blind=None) both call calculate_pseudo_cl; if the tagged - one wrote the native pseudo_cl_{ver}.sacc and renamed, its skip-if-exists - could silently adopt — and the rename delete — the diagnostic's differently- - blinded file. Born-at-declared-name makes the two paths provably disjoint. - """ + """calculate_pseudo_cl(out_path=...) writes to the given path, never the + untagged native name — so the tagged and diagnostic rules stay disjoint.""" ver = cv._test_version cv._pseudo_cls = {} tagged = cv._output_path(f"pseudo_cl_{ver}_blind=A_powspace_nbins=32.sacc") diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index e3a96a5d..52f612eb 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -262,7 +262,7 @@ def get_bandpower_windows(self): def test_assemble_analysis_sacc_block_diagonal_covariance(tmp_path): nz = {0: _nz()} parts = _make_parts(nz) - s = sw.assemble_analysis_sacc(nz, META, parts) + s = sw.assemble_analysis_sacc(parts) assert type(s.covariance).__name__ == "BlockDiagonalCovariance" assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) # every point covered; blocks placed and cross-blocks zero @@ -302,7 +302,7 @@ def test_assemble_analysis_sacc_requires_covariance(): ) ) # no covariance with pytest.raises(ValueError, match="own covariance block"): - sw.assemble_analysis_sacc(nz, META, parts) + sw.assemble_analysis_sacc(parts) def test_assemble_from_reloaded_parts(tmp_path): @@ -313,6 +313,6 @@ def test_assemble_from_reloaded_parts(tmp_path): for i, part in enumerate(parts): sio.save(part, str(tmp_path / f"part{i}.sacc"), type="mock") reloaded.append(sio.load(str(tmp_path / f"part{i}.sacc"))) - s = sw.assemble_analysis_sacc(nz, META, reloaded) + s = sw.assemble_analysis_sacc(reloaded) assert type(s.covariance).__name__ == "BlockDiagonalCovariance" assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) diff --git a/workflow/common.py b/workflow/common.py index 7d0ec2c4..63081e4e 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -5,14 +5,10 @@ import re from pathlib import Path -# Absolute path to the generic workflow's scripts, anchored on this module's own -# location (common.py lives in workflow/, is `from common import *`'d into every -# Snakefile, and so resolves to the generic workflow dir of the running checkout -# regardless of which paper composes it — unlike workflow.basedir, which under -# `module` composition reflects the composing paper). Rules that shell out to a -# script directly (rather than through Snakemake's `script:` directive) -# interpolate this instead of a hardcoded pure_eb/ compat-symlink -# path. /automnt/n17data is the automount of the container-bound /n17data. +# Absolute path to the generic workflow's scripts, for rules that shell out to a +# script directly rather than through Snakemake's `script:` directive. Anchored +# on this module's own location, not workflow.basedir — under `module` +# composition basedir reflects the composing paper, not the running checkout. WORKFLOW_SCRIPTS = os.path.join(os.path.dirname(os.path.realpath(__file__)), "scripts") # Output roots are env-overridable so a reproduction run can write into a @@ -28,12 +24,9 @@ ) ) CAT_CONFIG = "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" -# NB: "blind" here is the glass-mock multi-catalogue A/B/C variant convention -# (three mock realisations), NOT Smokescreen blinding. The name predates the -# blind-at-birth work and is kept because it is baked into on-disk filenames we -# do not own (e.g. sguerrini's nz_{version}_{A|B|C}.txt) and into the covariance -# / inference path builders below. Smokescreen concealment is a separate axis -# (the concealed=True SACC stamp), tracked by issues #241/#247. +# "blind" is the glass-mock A/B/C realisation convention, NOT Smokescreen +# blinding (a separate axis: the concealed=True SACC stamp). The name is baked +# into on-disk filenames we do not own (e.g. nz_{version}_{A|B|C}.txt). BLINDS = ["A", "B", "C"] BLOCK_PAIRS = [("++", "1"), ("--", "2"), ("+-", "3")] @@ -48,7 +41,6 @@ # silent failures. Apply with: wildcard_constraints: **WILDCARD_CONSTRAINTS WILDCARD_CONSTRAINTS = { "version": r"SP_v[\d.]+(_w_iv)?(_ecut\d+)?(_leak_corr)?", - # glass-mock A/B/C variant, not Smokescreen blinding — see BLINDS above. "blind": r"[ABC]", "nbins": r"\d+", "min_sep": r"[0-9.]+", @@ -173,22 +165,37 @@ def covariance_path( return str(COSMO_INFERENCE / f"data/covariance/{base}/{base}{suffix}") +def base_version(version): + """Strip the derived-catalogue suffixes to the base catalogue version. + + The `_leak_corr` / `_ecut{N}` variants share their parent's n(z) and + `cov_th` survey parameters, so lookups keyed on either must strip both. + """ + return re.sub(r"_ecut\d+", "", re.sub(r"_leak_corr$", "", version)) + + def build_redshift_path(version, blind): """Construct n(z) filepath for given catalog version and blind.""" - base_version = re.sub(r"_leak_corr$", "", version) - base_version = re.sub(r"_ecut\d+", "", base_version) - if "v1.4.11" in base_version: - base_version = "SP_v1.4.6" - version_dir = base_version.replace("SP_", "") - return ( - f"/n17data/sguerrini/UNIONS/WL/nz/{version_dir}/nz_{base_version}_{blind}.txt" - ) + base = base_version(version) + if "v1.4.11" in base: + base = "SP_v1.4.6" + version_dir = base.replace("SP_", "") + return f"/n17data/sguerrini/UNIONS/WL/nz/{version_dir}/nz_{base}_{blind}.txt" + + +def pseudo_cl_tag(config): + """Fiducial harmonic-binning tag the pseudo-Cl producers stamp into filenames. + + Single definition shared by the producer (twopoint.smk) and the consumers + (cosmo_val.smk, inference.smk), which reconstruct the name from config. + """ + fiducial = config["harmonic"]["fiducial"] + return f"blind={fiducial['blind']}_{fiducial['binning']}_nbins={fiducial['nbins']}" def get_shear_catalog(wildcards): """Resolve shear catalog path from config for a given version.""" - base_version = wildcards.version.replace("_leak_corr", "") - cat_config = CATALOG_CONFIG[base_version] + cat_config = CATALOG_CONFIG[wildcards.version.replace("_leak_corr", "")] shear_path = cat_config["shear"]["path"] if shear_path.startswith("/"): return shear_path diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 3923e5d0..afcdbac5 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -101,28 +101,17 @@ def cv_cosebis_npz(version): def cv_pseudo_cl_sacc(version): """Untagged pseudo-Cl SACC part cv_pseudo_cl writes (B-mode diagnostic). - This is the harmonic-space BB diagnostic cv_summarize_bmodes reads. The - *analysis* file's pseudo-Cl part is the tagged, blinded inference product - instead (see cv_pseudo_cl_analysis_sacc) so {version}.sacc stays byte- - comparable against today's cosmosis_fitting.py assembly (PR-3's converter). + The analysis file carries the tagged inference product instead (see + cv_pseudo_cl_analysis_sacc). """ return str(COSMO_VAL / f"pseudo_cl_{version}.sacc") -# Fiducial harmonic-binning tag the pseudo-Cl producer (twopoint.smk rules -# pseudo_cl / pseudo_cl_cov) stamps into the analysis-grade filename. Mirrors -# inference.smk's PSEUDO_CL_TAG so the analysis file carries the same pseudo-Cl -# the inference pipeline consumes (canonical: blind=A, powspace, nbins=32). -_HARMONIC_FIDUCIAL = config["harmonic"]["fiducial"] -_PSEUDO_CL_TAG = ( - f"blind={_HARMONIC_FIDUCIAL['blind']}" - f"_{_HARMONIC_FIDUCIAL['binning']}" - f"_nbins={_HARMONIC_FIDUCIAL['nbins']}" -) +_PSEUDO_CL_TAG = pseudo_cl_tag(config) def cv_pseudo_cl_analysis_sacc(version): - """Tagged, blinded pseudo-Cl SACC part the analysis file carries.""" + """Tagged pseudo-Cl SACC part the analysis file carries.""" return str(COSMO_VAL / f"pseudo_cl_{version}_{_PSEUDO_CL_TAG}.sacc") @@ -152,14 +141,12 @@ def cv_xi_sacc(version, grid): """ξ± SACC part the `xi` rule writes for a version on a named grid. Named by its binning (xi_binning, twopoint.smk), which is what binds the xi - job's wildcards; the grid label and the covariance treatment are resolved - from that binning by the rule. - - grid='reporting' is the analysis part (no covariance block until assembly - injects the CosmoCov one); grid='integration' is the fine-grid part COSEBIs - and pure-E/B consume, carrying its own DiagonalCovariance from TreeCorr - varxip/varxim. The integration part is intermediate: it stays standalone and - is NOT folded into the terminal {version}.sacc (see #247 ruling). + job's wildcards; the rule resolves the grid label from that binning. + + grid='reporting' is the analysis part (its covariance is injected at + assembly); grid='integration' is the fine-grid part COSEBIs and pure-E/B + consume. The integration part stays standalone — it is not folded into the + terminal {version}.sacc. """ return str(COSMO_VAL / f"{version}_xi_{xi_binning(grid)}.sacc") @@ -379,7 +366,11 @@ rule cv_pure_eb: rule cv_cosebis: - """COSEBIs E/B decomposition for one version (config-space, fine binning).""" + """COSEBIs E/B decomposition for one version (config-space, fine binning). + + Mixed provenance by design: En derives from the (blindable) integration ξ± + part, while Bn and the jackknife covariance need the patched raw measurement. + """ input: xi=lambda w: cv_xi_txt(w.version), xi_integration=lambda w: cv_xi_sacc(w.version, "integration"), @@ -440,38 +431,24 @@ rule cv_summarize_bmodes: # --------------------------------------------------------------------------- # Terminal analysis file: assemble the per-statistic SACC parts into {version}.sacc # --------------------------------------------------------------------------- -# The five born-as-SACC parts (xi_reporting, pseudo_cl, cosebis, pure_eb, rho_tau) -# are each written by their own rule carrying its own covariance block, except -# ξ± reporting and pseudo-Cℓ which are born cov-less by design. assemble_sacc.py -# loads the parts in canonical order and rebuilds one {version}.sacc with a -# single BlockDiagonalCovariance (point-insertion order = block order). -# -# The integration-grid ξ± (grid='integration') is deliberately NOT gathered here: -# it persists as its own per-part intermediate {version}_xi_integration.sacc, -# consumed by COSEBIs/pure-E/B, with Snakemake provenance covering traceability -# (see #247 ruling). The terminal file carries the analysis vector only. +# assemble_sacc.py loads the five parts in canonical order (xi_reporting, +# pseudo_cl, cosebis, pure_eb, rho_tau) and rebuilds one {version}.sacc with a +# single BlockDiagonalCovariance. The integration-grid ξ± is deliberately not +# gathered: it stays an intermediate consumed by COSEBIs/pure-E/B. # -# The pseudo-Cℓ part is the TAGGED, blinded inference product (blind=A, powspace, -# nbins=32) — the same pseudo-Cℓ today's cosmosis_fitting.py consumes — so the -# analysis file stays byte-comparable against it (PR-3's converter). Its real -# NaMaster covariance is injected here from the matching pseudo_cl_cov FITS -# (COVAR_EE_EE/BB_BB/EB_EB → block-diagonal, dropping cross-spectra, matching the -# B-mode PTE's use of COVAR_BB_BB). The ξ± reporting block is the one piece not yet -# sourced from its real covariance: the CosmoCov theory .txt is blind/gaussian/ -# mask-keyed and lives deep in the inference tree, so wiring it couples cosmo_val -# to the whole inference covariance DAG — that sourcing is PR-3's converter -# territory. Until then a documented diagonal placeholder keeps the ξ block (and -# so the BlockDiagonalCovariance) structurally valid; it is a flagged stand-in, never a -# science covariance, and plugs out via --xi-cov the moment PR 3 lands. +# The pseudo-Cℓ part is the tagged inference product, and its NaMaster covariance +# is injected here from the matching pseudo_cl_cov FITS. The ξ± reporting block +# has no real covariance wired yet — the CosmoCov theory .txt is blind/gaussian/ +# mask-keyed and lives deep in the inference tree, so sourcing it couples +# cosmo_val to the whole inference covariance DAG; it plugs in via --xi-cov. def cv_assemble_inputs(version): """The per-statistic SACC parts + covariance inputs assemble_sacc consumes. - Each part's filename carries enough to bind its producing rule's wildcards - (the reporting ξ± and ρ/τ parts their reporting binning; the pseudo-Cℓ part its - fiducial harmonic tag). pseudo_cl (+ its cov) is included only when the - config toggles the harmonic-space BB into the analysis. + Each part's filename carries enough to bind its producing rule's wildcards. + pseudo_cl (+ its cov) is included only when the config toggles the + harmonic-space BB into the analysis. """ parts = dict( xi_reporting=cv_xi_sacc(version, "reporting"), @@ -493,26 +470,18 @@ rule assemble_sacc: sacc=cv_analysis_sacc("{version}"), params: version="{version}", - # Run type (data|mock) gates unblinded loading in assemble_sacc.py: a - # 'data' run fails closed on unblinded parts, a 'mock' run loads freely. - # Production runs on real catalogues, so the default is 'data'. PR #253's - # blind-at-birth conceals each data part, letting the 'data' run assemble. + # Run type gates unblinded loading: a 'data' run fails closed on + # unblinded parts, a 'mock' run loads freely. type=CV.get("type", "data"), - # Statistics this rule wired (same toggles as cv_assemble_inputs). The - # script validates part_paths against this so a typo'd input keyword - # can't silently drop a statistic from the terminal file. + # Statistics this rule wired; the script validates part_paths against it + # so a typo'd input keyword can't silently drop one. expected=lambda w: [ k for k in cv_assemble_inputs(w.version) if k != "pseudo_cl_cov" ], - # ξ± reporting has no real covariance wired yet (its CosmoCov theory block is - # PR-3's converter territory, plugging in via --xi-cov). By DEFAULT this - # is fatal: assemble_sacc.py raises rather than ship {version}.sacc — the - # terminal science file — with a var=1.0 placeholder as its LEADING - # covariance block (~20 orders off the real ξ± variance → silent - # catastrophic χ²/PTE for any consumer). Only an explicit config opt-in - # (cosmo_val.allow_placeholder_cov: true — dry-run / test configs) attaches - # the flagged diagonal placeholder. The pseudo-Cℓ block is real (from the - # pseudo_cl_cov input); COSEBIs / pure-E/B / ρ/τ carry their own. + # Without a real ξ± covariance, assembly raises by default rather than + # ship {version}.sacc with a var=1.0 leading block — ~20 orders off the + # real variance, i.e. silently catastrophic χ²/PTE for any consumer. The + # opt-in is for dry-run and test configs only. placeholder_var=(1.0 if CV.get("allow_placeholder_cov", False) else None), resources: mem_mb=8000, diff --git a/workflow/rules/inference.smk b/workflow/rules/inference.smk index ff0c982d..0366420f 100644 --- a/workflow/rules/inference.smk +++ b/workflow/rules/inference.smk @@ -1,9 +1,7 @@ # Imports from Snakefile: FIDUCIAL, COSMO_INFERENCE, COSMO_VAL, covariance_path, build_redshift_path, fiducial_binning_suffix -# NOTE: dormant subsystem. The file-name plumbing (config-driven paths + the -# producer-tagged pseudo-Cl names) is fixed and the DAG is valid, but it has not -# been run end-to-end. Reviving it still needs the FITS-CONTENT plumbing -# reconciled: cosmosis_fitting.py reads ELL/EE/BB + COVAR_FULL, while the -# producers write PSEUDO_CELL/ELL + COVAR_BB_BB. +# NOTE: dormant subsystem, not run end-to-end. Reviving it needs the FITS +# content reconciled: cosmosis_fitting.py reads ELL/EE/BB + COVAR_FULL, while +# the producers write PSEUDO_CELL/ELL + COVAR_BB_BB. # Output root for CosmoSIS data products + configs. COSMO_INFERENCE (common.py) # already resolves to THIS repo's cosmo_inference dir, so the products land @@ -14,7 +12,6 @@ COSMO_INFERENCE_RUNDIR = str(COSMO_INFERENCE) # External chain/mock locations are deployment-specific, so they live in config. INFERENCE = config["inference"] -CHAINS_DIR = INFERENCE["chains_dir"] # CosmoSIS chain output root (real data) GLASS_MOCK_DATA_DIR = INFERENCE["glass_mock_data_dir"] # precomputed mock xi/Cl products GLASS_MOCK_CHAINS_DIR = INFERENCE["glass_mock_chains_dir"] # mock chain output root @@ -32,112 +29,23 @@ GLASS_MOCK_CONFIG_PATTERN = str( / f"cosmosis_config/cosmosis_pipeline_glass_mocks_{GLASS_MOCK_VERSION}_glass_mock_{{mock_id}}.ini" ) -# Fiducial harmonic-binning tag the pseudo-Cl producer (twopoint.smk) stamps -# into the filename. These are NOT inference_prep wildcards, so the consumer -# reads them from config to reconstruct the exact name the producer emits -# (canonical: blind=A, powspace, nbins=32 — see twopoint.smk pseudo_cl_all). -HARMONIC_FIDUCIAL = config["harmonic"]["fiducial"] -PSEUDO_CL_TAG = ( - f"blind={HARMONIC_FIDUCIAL['blind']}" - f"_{HARMONIC_FIDUCIAL['binning']}" - f"_nbins={HARMONIC_FIDUCIAL['nbins']}" -) +PSEUDO_CL_TAG = pseudo_cl_tag(config) def pseudo_cl_assets(version): - """Return pseudo-Cl and covariance paths for the requested catalog version. + """Pseudo-Cl and covariance paths for a catalog version. - The producer (twopoint.smk rules pseudo_cl / pseudo_cl_cov) writes - wildcard-tagged names; the consumer reconstructs them from the fiducial - harmonic-binning config so the requested path matches byte-for-byte. + The producer (twopoint.smk) writes wildcard-tagged names; the consumer + reconstructs them from the fiducial harmonic-binning config. """ cl_path = PSEUDO_CL_DIR / f"pseudo_cl_{version}_{PSEUDO_CL_TAG}.fits" cov_path = PSEUDO_CL_DIR / f"pseudo_cl_cov_{version}_{PSEUDO_CL_TAG}.fits" return str(cl_path), str(cov_path) -# --------------------------------------------------------------------------- -# DORMANT — pre-SACC cosmosis assembly. Migration to native SACC deferred to -# PR 7 (native-SACC inference consumption); do NOT deep-migrate here. -# -# The SACC migration (PR 4) removed the data products several of these inputs -# name, so this rule's DAG no longer resolves and is NOT reachable from the -# cosmo_val suite (cosmo_val_all never requests it). Stale inputs: -# - xi_plus / xi_minus FITS: the `xi` rule now emits the reporting ξ± SACC part -# ({version}_xi_reporting_...sacc), not per-sign FITS. -# - pseudo_cl / pseudo_cl_cov via pseudo_cl_assets(): the `pseudo_cl` rule now -# writes .sacc (pseudo_cl_assets still requests .fits). -# PR 7 rewires this to consume the assembled {version}.sacc (built by -# cosmo_val.smk's assemble_sacc rule) directly, retiring cosmosis_fitting.py's -# per-product FITS assembly. Until then the inference target is knowingly red. -# --------------------------------------------------------------------------- -rule inference_prep: - input: - # Processed covariance matrix - use centralized covariance_path() - cov_matrix=lambda w: covariance_path(w.version, w.blind, min_sep=w.min_sep, max_sep=w.max_sep, nbins=w.nbins), - # Xi FITS files — PRE-SACC (no longer produced; see dormant note above) - xi_plus=str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - xi_minus=str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - # n(z) file (using new location with base version mapping) - nz_file=lambda w: build_redshift_path(w.version, w.blind), - # rho/tau stats - rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - # tau covariance (tracked as dependency) - tau_cov=str(COSMO_VAL / "rho_tau_stats/cov_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}_th.npy"), - # pseudo_cl / pseudo_cl_cov — PRE-SACC (.fits path; producer now writes .sacc) - pseudo_cl=lambda w: pseudo_cl_assets(w.version)[0], - pseudo_cl_cov=lambda w: pseudo_cl_assets(w.version)[1], - output: - fits_file=str( - COSMO_INFERENCE_PROD - / "data/{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}/cosmosis_{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits" - ), - config_file=str( - COSMO_INFERENCE_PROD - / "cosmosis_config/cosmosis_pipeline_{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.ini" - ) - params: - cosmosis_root="{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}", - data_dir=f"{CHAINS_DIR}/{{version}}_{{blind}}_minsep={{min_sep}}_maxsep={{max_sep}}_nbins={{nbins}}_npatch={{npatch}}", - output_root=str(COSMO_INFERENCE_PROD), - threads: 1 - resources: - mem_mb=8000, - runtime=10, - shell: - """ - cd {COSMO_INFERENCE_RUNDIR} - - # Run inference preparation step with cosmosis_fitting.py - python scripts/cosmosis_fitting.py \ - --cosmosis-root {params.cosmosis_root} \ - --nz-file {input.nz_file} \ - --data-dir {params.data_dir} \ - --output-root {params.output_root} \ - --xi {input.xi_plus} {input.xi_minus} \ - --cov-xi {input.cov_matrix} \ - --use-rho-tau \ - --rho-stats {input.rho_stats} \ - --tau-stats {input.tau_stats} \ - --cov-tau {input.tau_cov} \ - --cl-file {input.pseudo_cl} \ - --cov-cl {input.pseudo_cl_cov} - """ - - -rule inference_fiducial: - input: - # Use the same output patterns as inference_prep with FIDUCIAL params - rules.inference_prep.output.fits_file.format( - version=FIDUCIAL["version"], blind=FIDUCIAL["blind"], - min_sep=FIDUCIAL["min_sep"], max_sep=FIDUCIAL["max_sep"], - nbins=FIDUCIAL["nbins"], npatch=FIDUCIAL["npatch"] - ), - rules.inference_prep.output.config_file.format( - version=FIDUCIAL["version"], blind=FIDUCIAL["blind"], - min_sep=FIDUCIAL["min_sep"], max_sep=FIDUCIAL["max_sep"], - nbins=FIDUCIAL["nbins"], npatch=FIDUCIAL["npatch"] - ) +# DORMANT: the SACC migration removed the data products rule inference_prep +# (and its inference_fiducial target) named, so its DAG no longer resolves; the +# rules are dropped rather than left red. Rewiring inference onto the assembled +# {version}.sacc is tracked separately. rule inference_glass_mocks: @@ -195,7 +103,5 @@ rule inference_prep_glass_mock: """ localrules: - inference_prep, inference_prep_glass_mock, - inference_fiducial, inference_glass_mocks, diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 8d449ffc..76f321d0 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -3,19 +3,12 @@ # --------------------------------------------------------------------------- # ξ± angular grids # --------------------------------------------------------------------------- -# A grid IS a binning: (min_sep, max_sep, nbins, npatch) plus how the -# born-as-SACC part carries its covariance. The reporting grid is the analysis -# one (its ξ covariance is injected at assembly from CosmoCov, so the part is -# written bare); the integration grid is the fine grid COSEBIs and pure-E/B -# integrate over, whose only covariance estimate is TreeCorr's shot-noise -# varxip/varxim — attached as a DiagonalCovariance. -# -# Both grids are measured by the single `xi` rule below: files are named by -# binning, so the grid label and the covariance mode are *resolved* from the -# wildcards rather than duplicated into a second rule. Workflows that carry no -# cosmo_val block (e.g. papers/bmodes) fall back to their fiducial grids; a -# binning matching no named grid is measured as a plain reporting-style -# measurement (no covariance). +# A grid is a binning: (min_sep, max_sep, nbins, npatch). `reporting` is the +# analysis grid; `integration` is the fine grid COSEBIs and pure-E/B integrate +# over. Both are measured by the single `xi` rule below, whose files are named +# by binning, so the grid label is resolved from the wildcards rather than +# duplicated into a second rule. Workflows carrying no cosmo_val block (e.g. +# papers/bmodes) fall back to their fiducial grids. def _xi_grids(): cv = config.get("cosmo_val", {}) reporting = ( @@ -37,14 +30,11 @@ def _xi_grids(): } ) integration.setdefault("npatch", 1) - return { - "reporting": {**reporting, "covariance": "none"}, - "integration": {**integration, "covariance": "diagonal"}, - } + return {"reporting": reporting, "integration": integration} XI_GRIDS = _xi_grids() -XI_DEFAULT_GRID = ("reporting", "none") +XI_KEYS = ("min_sep", "max_sep", "nbins", "npatch") def xi_binning(grid): @@ -57,22 +47,26 @@ def xi_binning(grid): def xi_grid_of(wildcards): - """(grid label, covariance mode) for the binning a job was requested with.""" - key = (wildcards.min_sep, wildcards.max_sep, wildcards.nbins, wildcards.npatch) + """Grid label for the binning a job was requested with. + + Compared numerically, so a "300" wildcard matches a 300.0 config value. + Binnings matching no named grid (e.g. papers/bmodes' nbins=10000 + convergence check) are measured as plain reporting-style measurements. + """ + key = tuple(float(getattr(wildcards, k)) for k in XI_KEYS) for name, g in XI_GRIDS.items(): - if tuple(str(g[k]) for k in ("min_sep", "max_sep", "nbins", "npatch")) == key: - return name, g["covariance"] - return XI_DEFAULT_GRID + if tuple(float(g[k]) for k in XI_KEYS) == key: + return name + return "reporting" rule xi: """TreeCorr ξ±(θ) for one version on one angular grid. Binning-agnostic: the reporting and integration measurements are the same - job with different wildcards. The raw TreeCorr .txt byproduct (read back by - the covariance machinery and by the skip-if-exists) and the born-as-SACC - part are named by that binning, so a request for either binds unambiguously - — and the grid label + covariance treatment come from XI_GRIDS. + job with different wildcards. The raw TreeCorr .txt byproduct and the + born-as-SACC part are both named by that binning, so a request for either + binds unambiguously; the grid label comes from XI_GRIDS. """ input: catalog=get_shear_catalog, @@ -87,8 +81,7 @@ rule xi: nbins="{nbins}", npatch="{npatch}", cat_config=CAT_CONFIG, - grid=lambda w: xi_grid_of(w)[0], - covariance=lambda w: xi_grid_of(w)[1], + grid=lambda w: xi_grid_of(w), resources: # The fine integration grid needs more memory and wall time than the # ~20-bin reporting one; scale on nbins rather than splitting the rule. @@ -121,10 +114,8 @@ rule rho_tau_stats: output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - # Born-as-SACC ρ/τ part (ρ_0…ρ_5 autos + τ_0/τ_2/τ_5 leakage, carrying - # its own covariance block) that the assemble_sacc rule consumes; - # calculate_rho_tau_stats writes it alongside the FITS via - # rho_tau_to_sacc_part. + # Born-as-SACC ρ/τ part the assemble_sacc rule consumes, written + # alongside the FITS by calculate_rho_tau_stats. rho_tau=str(COSMO_VAL / "rho_tau_stats/rho_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 48 params: @@ -148,16 +139,11 @@ wildcard_constraints: rule pseudo_cl: - """Generate pseudo-Cl data vector (born as SACC) with configurable binning. - - NB: the ``blind`` wildcard is the glass-mock A/B/C variant (three mock - catalogues), NOT Smokescreen blinding — see common.py BLINDS. The Smokescreen - concealed=True stamp is a separate axis on the SACC file. - """ + """Generate pseudo-Cl data vector (born as SACC) with configurable binning.""" output: pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.sacc"), wildcard_constraints: - blind="[ABC]", # glass-mock variant, not Smokescreen blinding + blind="[ABC]", params: version="{version}", blind="{blind}", @@ -177,15 +163,11 @@ rule pseudo_cl: rule pseudo_cl_cov: - """Generate pseudo-Cl covariance with configurable binning. - - NB: ``blind`` is the glass-mock A/B/C variant, not Smokescreen blinding - (see common.py BLINDS). - """ + """Generate pseudo-Cl covariance with configurable binning.""" output: pseudo_cl_cov=str(COSMO_VAL / "pseudo_cl_cov_{version}_blind={blind}_{binning}_nbins={nbins}.fits"), wildcard_constraints: - blind="[ABC]", # glass-mock variant, not Smokescreen blinding + blind="[ABC]", params: version="{version}", blind="{blind}", diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index cfad6a8f..97a86f43 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -1,40 +1,20 @@ """Assemble the terminal ``{version}.sacc`` analysis file from per-statistic parts. -Dual-mode. Under Snakemake (``script:`` directive) the injected ``snakemake`` -object supplies the parts + covariance inputs; as a standalone CLI (argparse) -the same assembly runs from explicit flags (the lightcone/ASTRA path). - -Each per-statistic ``*.sacc`` *part* (written born-as-SACC by the mixins and the -run_2pcf / generate_pseudo_cl scripts) holds one statistic. The assembler loads -them in canonical order — ξ± reporting, pseudo-Cℓ, COSEBIs, pure-E/B, ρ/τ — and -calls :func:`sacc_writers.assemble_analysis_sacc`, which rebuilds one Sacc with a -single ``BlockDiagonalCovariance`` (point-insertion order = block order, -validated by ``sacc_io.assemble_covariance``). - -Covariance sourcing (the part-by-part decision) ------------------------------------------------ -``assemble_analysis_sacc`` REQUIRES every part to carry its own covariance block. -The COSEBIs, pure-E/B and ρ/τ parts already do (their writers attach it). The -ξ± reporting and pseudo-Cℓ parts are born cov-less by design; this script injects -their blocks before assembly: - -* **ξ± reporting** — the CosmoCov theory covariance ``.txt`` (``--xi-cov``). For the - single-bin round it is already ``[ξ+; ξ−]``-ordered (CosmoCov / covdat_to_fits: - ``STRT_0=0`` XI_PLUS, ``STRT_1=len/2`` XI_MINUS), which is exactly the SACC - ξ insertion order, so ``np.loadtxt`` → ``add_covariance`` needs no permutation. -* **pseudo-Cℓ** — the NaMaster iNKA / OneCovariance covariance FITS - (``--pseudo-cl-cov`` + ``--pseudo-cl-cov-hdu``). The FITS carries the 16 - EE/EB/BE/BB cross-blocks (each ``nbp × nbp``); SACC stores EE, BB, EB (in that - order), so we assemble the block-diagonal ``[EE_EE; BB_BB; EB_EB]``. The - cross-spectrum blocks (EE↔BB, …) are dropped — matching how the B-mode PTE - today reads only ``COVAR_BB_BB``. **TODO(PR-cov):** carry the full dense - EE/BB/EB cross-covariance once the analysis needs cross-spectrum correlations. - -When a cov input is absent the assembly cannot proceed on a real product; pass -``--allow-placeholder`` to attach a documented diagonal placeholder -(``placeholder_var`` on every point of the cov-less parts) so the DAG dry-run and -the fast test can still produce a structurally-valid ``BlockDiagonalCovariance``. The -placeholder is a flagged stand-in, never a science covariance. +Dual-mode: under Snakemake (``script:``) the injected ``snakemake`` object +supplies the inputs; as a standalone CLI the same assembly runs from flags. + +Each part is a single-statistic SACC written by the cosmo_val mixins, run_2pcf +or generate_pseudo_cl. Parts load in canonical order (ξ± reporting, pseudo-Cℓ, +COSEBIs, pure-E/B, ρ/τ) and are rebuilt into one Sacc with a single +``BlockDiagonalCovariance``. + +Every part must carry a covariance block. COSEBIs, pure-E/B and ρ/τ are born +with one; ξ± reporting and pseudo-Cℓ are not, so their blocks are injected here +from the CosmoCov ``.txt`` (``--xi-cov``) and the NaMaster covariance FITS +(``--pseudo-cl-cov``). The pseudo-Cℓ cross-spectrum blocks (EE↔BB, …) are +dropped — matching what the B-mode PTE reads today. ``--allow-placeholder VAR`` +attaches a flagged diagonal stand-in instead, so a dry run or the fast test can +still build a structurally valid covariance. """ import argparse @@ -44,54 +24,44 @@ from sp_validation import sacc_io from sp_validation.cosmo_val.sacc_writers import assemble_analysis_sacc -# NaMaster iNKA covariance FITS: per-spectrum HDU names. SACC insertion order is -# EE, BB, EB, so the block-diagonal is assembled in that order. -_CL_HDU = {"EE": "COVAR_EE_EE", "BB": "COVAR_BB_BB", "EB": "COVAR_EB_EB"} -_CL_ORDER = ("EE", "BB", "EB") +# NaMaster iNKA covariance FITS: per-spectrum HDU names, in SACC insertion order. +_CL_HDUS = ("COVAR_EE_EE", "COVAR_BB_BB", "COVAR_EB_EB") -# Canonical part order — the order assemble_analysis_sacc inserts points in, which -# must match the covariance block order. Missing parts are simply skipped. +# Canonical part order — the order points are inserted in, which must match the +# covariance block order. Missing parts are simply skipped. CANONICAL = ("xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau") -def _pseudo_cl_cov_block(cov_fits, hdu): - """Block-diagonal ``[EE_EE; BB_BB; EB_EB]`` from the NaMaster iNKA cov FITS. - - ``hdu`` selects the file flavor: for the per-spectrum iNKA file we read the - three named diagonal HDUs; for a single dense HDU (OneCovariance / g+ng - ``COVAR_FULL``) that already spans EE/BB/EB we return it as-is. - """ +def _pseudo_cl_cov_block(cov_fits): + """Block-diagonal ``[EE; BB; EB]`` from the NaMaster iNKA covariance FITS.""" from astropy.io import fits with fits.open(cov_fits) as hdul: - names = {h.name for h in hdul} - if all(_CL_HDU[s] in names for s in _CL_ORDER): - blocks = [np.asarray(hdul[_CL_HDU[s]].data, float) for s in _CL_ORDER] - n = blocks[0].shape[0] - full = np.zeros((3 * n, 3 * n)) - for i, block in enumerate(blocks): - full[i * n : (i + 1) * n, i * n : (i + 1) * n] = block - return full - return np.asarray(hdul[hdu].data, float) + missing = [name for name in _CL_HDUS if name not in {h.name for h in hdul}] + if missing: + raise ValueError(f"{cov_fits} lacks the pseudo-Cℓ cov HDUs {missing}") + blocks = [np.asarray(hdul[name].data, float) for name in _CL_HDUS] + n = blocks[0].shape[0] + full = np.zeros((3 * n, 3 * n)) + for i, block in enumerate(blocks): + full[i * n : (i + 1) * n, i * n : (i + 1) * n] = block + return full -def _attach_cov(part, name, xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var): - """Ensure ``part`` carries a covariance, injecting the xi/pseudo-Cℓ block. +def _attach_cov(part, name, xi_cov, pseudo_cl_cov, placeholder_var): + """Ensure ``part`` (mutated in place) carries a covariance block. - ``part`` is mutated in place. cosebis/pure_eb/rho_tau parts already carry - their covariance and pass straight through. Raises loudly if a required xi / - pseudo-Cℓ block is missing and no placeholder was requested. + Raises if a required ξ± / pseudo-Cℓ block is missing and no placeholder was + requested. """ if part.covariance is not None: return part - if name == "xi_reporting": - if xi_cov is not None: - part.add_covariance(np.loadtxt(xi_cov)) - return part - elif name == "pseudo_cl": - if pseudo_cl_cov is not None: - part.add_covariance(_pseudo_cl_cov_block(pseudo_cl_cov, pseudo_cl_cov_hdu)) - return part + if name == "xi_reporting" and xi_cov is not None: + part.add_covariance(np.loadtxt(xi_cov)) + return part + if name == "pseudo_cl" and pseudo_cl_cov is not None: + part.add_covariance(_pseudo_cl_cov_block(pseudo_cl_cov)) + return part if placeholder_var is None: raise ValueError( f"the {name!r} part carries no covariance and no covariance input was " @@ -110,7 +80,6 @@ def assemble_sacc( expected=None, xi_cov=None, pseudo_cl_cov=None, - pseudo_cl_cov_hdu="COVAR_FULL", placeholder_var=None, allow_unblinded=False, ): @@ -119,24 +88,17 @@ def assemble_sacc( Parameters ---------- version : str - Catalogue version (stored in the assembled file's metadata). + Catalogue version, for error messages. part_paths : dict - ``{statistic: path}`` with statistic in :data:`CANONICAL`. Only the - present statistics are assembled; order is forced to canonical. - out_path : str - Destination ``{version}.sacc``. + ``{statistic: path}`` with statistic in :data:`CANONICAL`. Only present + statistics are assembled; order is forced to canonical. expected : sequence of str, optional - Statistics that MUST be present in ``part_paths`` (from the caller's - config toggles). Raises loudly if any is missing or has no path — so a - typo'd input keyword (``cosebi`` for ``cosebis``) can't silently drop a - statistic from the terminal file. Names not in :data:`CANONICAL` are - rejected too (catches a typo in the expected list itself). - xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var + Statistics that must be present, from the caller's config toggles. A + typo'd input keyword would otherwise silently drop a statistic. + xi_cov, pseudo_cl_cov, placeholder_var Covariance sourcing — see the module docstring. allow_unblinded : bool, optional - Passed to :func:`sacc_io.load` for every part. Default ``False`` fails - closed on unblinded real data; the caller sets it ``True`` only for mock - runs. See the load loop for the PR #253 blind-at-birth seam. + Passed to :func:`sacc_io.load` for every part; ``True`` only for mocks. """ if expected is not None: unknown = [name for name in expected if name not in CANONICAL] @@ -153,46 +115,20 @@ def assemble_sacc( "silently dropped from the terminal analysis file" ) parts = [] - nz = metadata = None for name in CANONICAL: path = part_paths.get(name) if path is None: continue - # Fail closed on real data by default: a data-type part loads only when - # it already carries the concealed=True blinding stamp. allow_unblinded - # is set True only for mock runs (see the caller). This is the seam for - # PR #253's blind-at-birth: once each part is concealed at write time, a - # data run assembles with allow_unblinded=False untouched. part = sacc_io.load(path, allow_unblinded=allow_unblinded) - if nz is None: - # The nz tracers + metadata are identical across parts (same version); - # take them from the first loaded part for the assembled file. - nz = {i: sacc_io.get_nz(part, i) for i in range(_n_source_bins(part))} - metadata = dict(part.metadata) - parts.append( - _attach_cov( - part, name, xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var - ) - ) + parts.append(_attach_cov(part, name, xi_cov, pseudo_cl_cov, placeholder_var)) if not parts: raise ValueError(f"no parts found for {version}: {part_paths}") - s = assemble_analysis_sacc(nz, metadata, parts) - # Assembly preserves its parts' provenance: every part was written by - # sacc_io.save and therefore carries the type=data|mock stamp in its - # metadata (copied into the assembled file above). - sacc_io.save(s, out_path, type=metadata["type"]) + s = assemble_analysis_sacc(parts) + sacc_io.save(s, out_path, type=s.metadata["type"]) print(f"Assembled {len(parts)} parts -> {out_path}") return s -def _n_source_bins(part): - """Count the ``source_{i}`` NZ tracers on a part (single-bin round -> 1).""" - i = 0 - while sacc_io.source_name(i) in part.tracers: - i += 1 - return i - - def _from_snakemake(smk): p = smk.params inp = smk.input @@ -201,23 +137,15 @@ def _from_snakemake(smk): for name in CANONICAL if hasattr(inp, name) and getattr(inp, name) } - # The rule declares which statistics it wired (from its config toggles); a - # typo in an input keyword drops the part from part_paths above, so validate - # against this expected list rather than trusting the hasattr filter. - expected = list(p["expected"]) - # Fail closed on real data: only a mock run may read unblinded parts. The - # run type comes from config (default 'data' — the production catalogues). - run_type = p.get("type", "data") assemble_sacc( version=p["version"], part_paths=part_paths, out_path=str(smk.output[0]), - expected=expected, + expected=list(p["expected"]), xi_cov=getattr(inp, "xi_cov", None), pseudo_cl_cov=getattr(inp, "pseudo_cl_cov", None), - pseudo_cl_cov_hdu=p.get("pseudo_cl_cov_hdu", "COVAR_FULL"), placeholder_var=p.get("placeholder_var", None), - allow_unblinded=(run_type == "mock"), + allow_unblinded=(p.get("type", "data") == "mock"), ) @@ -240,14 +168,7 @@ def _from_cli(argv=None): ) ap.add_argument("--xi-cov", default=None, help="CosmoCov ξ covariance .txt") ap.add_argument( - "--pseudo-cl-cov", - default=None, - help="NaMaster/OneCovariance pseudo-Cℓ cov FITS", - ) - ap.add_argument( - "--pseudo-cl-cov-hdu", - default="COVAR_FULL", - help="HDU name for a single dense pseudo-Cℓ cov (EE/BB/EB-spanning)", + "--pseudo-cl-cov", default=None, help="NaMaster pseudo-Cℓ covariance FITS" ) ap.add_argument( "--allow-placeholder", @@ -264,7 +185,6 @@ def _from_cli(argv=None): out_path=a.out, xi_cov=a.xi_cov, pseudo_cl_cov=a.pseudo_cl_cov, - pseudo_cl_cov_hdu=a.pseudo_cl_cov_hdu, placeholder_var=a.allow_placeholder, allow_unblinded=(a.type == "mock"), ) diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index 84991d7b..726e16ba 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -30,10 +30,7 @@ fiducial_scale_cut=fiducial_scale_cut, ) -# Consume the integration-grid ξ± SACC part: re-derive the fiducial-cut E-mode En -# from it through the same cosmo_numba kernel plot_cosebis' raw path uses -# (b_modes.cosebis_from_xi). Bn and the covariance stay blind-invariant from the -# raw plot_cosebis result. Version-agnostic — every version binds the part. +# Re-derive En from the integration ξ± part via the same kernel; Bn and cov stay raw. from sp_validation import sacc_io from sp_validation.b_modes import cosebis_from_xi @@ -41,8 +38,7 @@ theta, xip, xim = sacc_io.get_xi(integ, (0, 0), grid="integration") en_part, _ = cosebis_from_xi(theta, xip, xim, p["nmodes"], scale_cut=fiducial_scale_cut) -# Born-as-SACC COSEBIs part at the fiducial scale cut (plot_cosebis stored the -# multi-cut results on the instance); En comes from the consumed part. +# Born-as-SACC COSEBIs part at the fiducial scale cut. cv.cosebis_to_sacc_part( version, snakemake.output["sacc"], @@ -51,9 +47,7 @@ en_override=en_part, ) -# Overwrite the raw fiducial-cut En plot_cosebis wrote into the diagnostic npz with -# the part-derived En (identical to the SACC part's). Bn / cov / PTE fields are -# untouched, so the B-mode summary reader is unaffected. +# Sync the npz's En with the part-derived values; Bn / cov / PTE untouched. npz_path = snakemake.output["npz"] data = dict(np.load(npz_path, allow_pickle=True)) data["En"] = np.asarray(en_part) diff --git a/workflow/scripts/cv_pseudo_cl.py b/workflow/scripts/cv_pseudo_cl.py index 43923e4e..3f6374bf 100644 --- a/workflow/scripts/cv_pseudo_cl.py +++ b/workflow/scripts/cv_pseudo_cl.py @@ -2,9 +2,8 @@ plot_pseudo_cl triggers calculate_pseudo_cl, which writes the born-as-SACC pseudo_cl_{version}.sacc part for every version (EE/BB/EB with the shared -bandpower window — the BB spectrum cv_summarize_bmodes reads) and the -cell_ee.png figure. The per-version SACC parts are the declared outputs and -feed both cv_summarize_bmodes and the assemble_sacc rule. +bandpower window) and the cell_ee.png figure. The per-version SACC parts are +the declared outputs, read by cv_summarize_bmodes. """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index d4797922..bff39ecb 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -27,12 +27,8 @@ ) results = cv._pure_eb_results[version] -# Consume the reporting + integration ξ± SACC parts: re-derive the six pure-mode -# arrays through the same cosmo_numba kernel plot_pure_eb' raw path uses -# (b_modes.pure_eb_from_xi). The covariance stays blind-invariant from the raw -# result. tmin/tmax are the reporting grid's TreeCorr bin edges (from the raw -# reporting gg — add_xi stores no edges). Version-agnostic — every version binds -# both parts. +# Re-derive the pure modes from the ξ± parts via the same kernel; cov stays raw. +# tmin/tmax are the reporting grid's TreeCorr bin edges (add_xi stores no edges). from sp_validation import sacc_io from sp_validation.b_modes import pure_eb_from_xi @@ -44,12 +40,10 @@ ti, xpi, xmi = sacc_io.get_xi(integ, (0, 0), grid="integration") modes = pure_eb_from_xi(tr, xpr, xmr, ti, xpi, xmi, tmin, tmax) -# Born-as-SACC pure-E/B part; the six pure-mode blocks come from the consumed parts. +# Born-as-SACC pure-E/B part; the six blocks come from the consumed parts. cv.pure_eb_to_sacc_part(version, snakemake.output["sacc"], results, eb_override=modes) -# Overwrite the raw pure-mode arrays plot_pure_eb wrote into the diagnostic npz with -# the part-derived ones (identical to the SACC part's). theta / cov / PTE fields are -# untouched, so the B-mode summary reader is unaffected. +# Sync the npz's pure modes with the part-derived values; theta / cov / PTE untouched. npz_path = snakemake.output["npz"] data = dict(np.load(npz_path, allow_pickle=True)) for key, arr in modes.items(): diff --git a/workflow/scripts/generate_cosmocov_ini.py b/workflow/scripts/generate_cosmocov_ini.py index 11e785cf..6996c04f 100644 --- a/workflow/scripts/generate_cosmocov_ini.py +++ b/workflow/scripts/generate_cosmocov_ini.py @@ -1,12 +1,9 @@ """Generate a CosmoCov ``.ini`` for one (version, blind, grid, flavour, mask). -CLI refactor of the former ``rule covariance_ini`` heredoc. Cosmology is read -from the frozen ``planck18.json`` snapshot (the cosmology_snapshot lc output — -source of truth is cs_util.cosmo.PLANCK18); survey (area, n_eff, -sigma_e) from the catalog config's per-version ``cov_th``; n(z) via the same -path convention as workflow/common.build_redshift_path; the footprint mask -power spectrum is passed explicitly (empty string for the unmasked variant). -The emitted ``.ini`` is byte-compatible with the paper's covariance_ini rule. +Cosmology comes from the frozen ``planck18.json`` snapshot, survey parameters +(area, n_eff, sigma_e) from the catalog config's per-version ``cov_th``, and +n(z) from ``workflow/common.build_redshift_path``. The footprint mask power +spectrum is passed explicitly (empty string for the unmasked variant). python generate_cosmocov_ini.py \ --version SP_v1.4.6.3_leak_corr --blind A \ @@ -18,23 +15,27 @@ """ import argparse +import importlib.util import json import os -import re +import sys import yaml -def build_redshift_path(version, blind): - """Replicate workflow/common.build_redshift_path.""" - base_version = re.sub(r"_leak_corr$", "", version) - base_version = re.sub(r"_ecut\d+", "", base_version) - if "v1.4.11" in base_version: - base_version = "SP_v1.4.6" - version_dir = base_version.replace("SP_", "") - return ( - f"/n17data/sguerrini/UNIONS/WL/nz/{version_dir}/nz_{base_version}_{blind}.txt" +def _load_workflow_common(): + """Load ``workflow/common.py`` (this script also runs outside Snakemake).""" + path = os.path.join( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "common.py" ) + spec = importlib.util.spec_from_file_location("workflow_common", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +common = _load_workflow_common() INI_TEMPLATE = """\ @@ -116,8 +117,7 @@ def main(argv=None): with open(a.cat_config) as f: cat_config = yaml.safe_load(f) - base_version = a.version.replace("_leak_corr", "") - cov_th = cat_config[base_version]["cov_th"] + cov_th = cat_config[common.base_version(a.version)]["cov_th"] ng_value = "1" if a.gaussian == "ng" else "0" @@ -131,7 +131,7 @@ def main(argv=None): area=cov_th["A"], sigma_e=cov_th["sigma_e"], n_e=cov_th["n_e"], - nz=build_redshift_path(a.version, a.blind), + nz=common.build_redshift_path(a.version, a.blind), mask=a.mask_cls, min_sep=a.min_sep, max_sep=a.max_sep, diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index 0cec054e..1299a36d 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -4,12 +4,10 @@ object supplies the parameters and the native product is renamed to the tagged output filename the rule declares; as a standalone CLI (argparse) the same compute runs from explicit flags and the primitive's native -``pseudo_cl_{ver}.sacc`` is left in place under ``--out`` (no rename — each -lc/ASTRA recipe gets its own output directory, so the untagged native name is -unambiguous and the primitives' skip-if-exists never collides across nbins -runs). The C_ell data vector is born as SACC (EE/BB/EB with a shared bandpower -window) — see ``sp_validation.cosmo_val.sacc_writers.pseudo_cl_to_sacc``. The -CLI form is what the lightcone/ASTRA recipe calls, so the measurement is driven +``pseudo_cl_{ver}.sacc`` is left in place under ``--out`` (each lc/ASTRA recipe +gets its own output directory, so the untagged name is unambiguous). The C_ell +data vector is born as SACC (EE/BB/EB with a shared bandpower window). The CLI +form is what the lightcone/ASTRA recipe calls, so the measurement is driven directly (no nested Snakemake) with lc handling orchestration: python generate_pseudo_cl.py \ @@ -52,9 +50,8 @@ def generate_pseudo_cl( version : str Catalog version (e.g., "SP_v1.4.6_leak_corr") out_path : str - Exact destination the SACC part is *born at* — its final (possibly - tagged) name. No native-basename + rename step, so this producer's - skip-if-exists never collides with the untagged cv_pseudo_cl diagnostic. + Exact destination the SACC part is born at — its final (possibly tagged) + name. Skip-if-exists keys on it, so no two rules share a basename. cat_config : str Path to catalog configuration YAML nside : int @@ -136,16 +133,11 @@ def generate_pseudo_cl( cv = CosmologyValidation(**cv_kwargs) - # Calculate pseudo-Cls only (no covariance). The data vector is born as a - # SACC part directly at out_path (its final, possibly-tagged name) — no - # shared native basename, no rename, so this producer's skip-if-exists never - # collides with the untagged cv_pseudo_cl diagnostic (which would otherwise - # let one rule adopt + delete the other's differently-blinded file). + # Pseudo-Cls only (no covariance), born directly at the final out_path. cv.calculate_pseudo_cl(out_path=out_path) if os.path.exists(out_path): - # Pipeline-internal readback of the unblinded data part just written - # (blinding is a downstream Smokescreen step). + # Readback of the part just written — a legitimate pre-blind consumer. s = sacc_io.load(out_path, allow_unblinded=True) ell = sacc_io.get_pseudo_cl(s, (0, 0))[0] print(f"Generated pseudo-Cl with {len(ell)} ell bins") @@ -220,9 +212,8 @@ def _from_cli(argv=None): with open(a.cosmo_json) as f: cosmo_params = json.load(f) - # lc/ASTRA path: --out is a per-recipe directory; the untagged native name - # is unambiguous there (each recipe gets its own tree, so no cross-nbins or - # cross-blind collision). + # lc/ASTRA path: --out is a per-recipe directory, so the untagged name is + # unambiguous there. out_path = os.path.join(a.out, f"pseudo_cl_{a.ver}.sacc") generate_pseudo_cl( version=a.ver, diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 10dafc53..19f50682 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -14,17 +14,12 @@ The measurement is binning-agnostic: the reporting and the fine integration grids are the same compute with different ``--min-sep/--max-sep/--nbins``. -``CosmologyValidation.calculate_2pcf`` does the TreeCorr work and writes the -``.txt`` dump (a raw byproduct the covariance machinery and the convergence -consumers read back); the ξ± data product is then born as SACC here, a *part* -named by its binning and tagged with its ``grid``: - -* ``--grid reporting`` (default) — ``--covariance none``: no covariance block, - because the ξ block is supplied at assembly from the CosmoCov theory - covariance. -* ``--grid integration`` — ``--covariance diagonal``: a ``DiagonalCovariance`` - from TreeCorr ``varxip``/``varxim``, the only covariance estimate available at - npatch=1, which is what COSEBIs and pure-E/B consume. +``CosmologyValidation.calculate_2pcf`` writes the ``.txt`` dump (a raw +byproduct); the ξ± data product is born as SACC here, a *part* named by its +binning and tagged with its ``--grid``. The reporting part carries no covariance +(its block is supplied at assembly from CosmoCov); the integration part carries +a ``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim``, the only estimate +available at npatch=1, which is what COSEBIs and pure-E/B consume. ``output_dir`` is passed explicitly (rather than via the ``COSMO_VAL`` env hook) so lc can point each run at its own ``{output}`` tree. @@ -50,7 +45,6 @@ def run_2pcf( output_dir, sacc_out=None, grid="reporting", - covariance="none", ): """Measure ξ±(θ) for ``ver`` and write its reporting SACC part. @@ -84,8 +78,6 @@ def run_2pcf( ) # Born-as-SACC ξ± part. theta = meanr; theta_nom = rnom. - if covariance not in ("none", "diagonal"): - raise ValueError(f"unknown covariance mode {covariance!r}") s = xi_to_sacc( cv.sacc_nz(ver), cv.sacc_metadata(ver), @@ -97,7 +89,7 @@ def run_2pcf( npairs=gg.npairs, weight=gg.weight, variances=( - np.concatenate([gg.varxip, gg.varxim]) if covariance == "diagonal" else None + np.concatenate([gg.varxip, gg.varxim]) if grid == "integration" else None ), ) out_path = sacc_out or os.path.join( @@ -123,10 +115,9 @@ def _from_snakemake(smk): # class defaults (./cat_config.yaml, COSMO_VAL env) otherwise. cat_config=p.get("cat_config", "./cat_config.yaml"), output_dir=p.get("output_dir", None), - # Grid label + covariance treatment are resolved by the rule from the - # binning wildcards (workflow/rules/twopoint.smk XI_GRIDS). + # Grid label is resolved by the rule from the binning wildcards + # (workflow/rules/twopoint.smk XI_GRIDS). grid=p.get("grid", "reporting"), - covariance=p.get("covariance", "none"), # Write the SACC part exactly where the rule declares it (the .txt # byproduct still lands under the resolved output dir via _output_path). sacc_out=smk.output["sacc"], @@ -160,13 +151,8 @@ def _from_cli(argv=None): "--grid", default="reporting", choices=["reporting", "integration"], - help="SACC grid tag of the measured part", - ) - ap.add_argument( - "--covariance", - default="none", - choices=["none", "diagonal"], - help="Covariance carried by the SACC part (diagonal: varxip/varxim)", + help="SACC grid tag; 'integration' also attaches the varxip/varxim " + "DiagonalCovariance", ) a = ap.parse_args(argv) run_2pcf( @@ -178,7 +164,6 @@ def _from_cli(argv=None): cat_config=a.cat_config, output_dir=a.out, grid=a.grid, - covariance=a.covariance, ) diff --git a/workflow/scripts/run_cosmocov_chain.sh b/workflow/scripts/run_cosmocov_chain.sh index 80a80370..3a7d3a84 100644 --- a/workflow/scripts/run_cosmocov_chain.sh +++ b/workflow/scripts/run_cosmocov_chain.sh @@ -1,12 +1,11 @@ #!/usr/bin/env bash # CosmoCov covariance chain (lc-native, container:none recipe). # -# Faithful port of covariance_ini -> covariance_cosmocov (x3 blocks) -> -# covariance_cat -> covariance_process. The CosmoCov C++ binary runs on the -# bare host (module load gcc/intelpython/openmpi, as in the original -# container:None rule); the .ini generation and cosmocov_process step run inside -# the sp_validation apptainer container. The 3 shear-shear blocks (++,--,+-) are -# independent and run in parallel. +# covariance_ini -> covariance_cosmocov (x3 blocks) -> covariance_cat -> +# covariance_process. The CosmoCov C++ binary runs on the bare host (module load +# gcc/intelpython/openmpi); the .ini generation and cosmocov_process steps run +# inside the sp_validation apptainer container. The 3 shear-shear blocks +# (++,--,+-) are independent and run in parallel. # # Usage: # run_cosmocov_chain.sh --version SP_v1.4.6.3_leak_corr --blind A \ @@ -14,13 +13,19 @@ # --planck18-json /planck18.json \ # --cat-config --mask-cls \ # --out +# +# The checkout is this script's own (workflow/scripts/../..). Deployment paths +# come from the environment, with the current candide values as defaults: +# SPV_CONTAINER apptainer image (default /n17data/cdaley/containers/containers/) +# SPV_BIND apptainer --bind list +# COSMOCOV CosmoCov `cov` binary (also settable with --cosmocov) set -euo pipefail -CONTAINER=/n17data/cdaley/containers/containers/ -WT=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra +WT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" SRC=$WT/src -BIND=/home,/scratch,/automnt,/n17data,/n23data1,/n09data -COSMOCOV=/n23data1/n06data/lgoh/scratch/UNIONS/CosmoCov/covs/cov +CONTAINER=${SPV_CONTAINER:-/n17data/cdaley/containers/containers/} +BIND=${SPV_BIND:-/home,/scratch,/automnt,/n17data,/n23data1,/n09data} +COSMOCOV=${COSMOCOV:-/n23data1/n06data/lgoh/scratch/UNIONS/CosmoCov/covs/cov} VERSION=""; BLIND="A"; MINSEP=""; MAXSEP=""; NBINS=""; GAUSSIAN="" PLANCK18=""; CATCONFIG=""; MASKCLS=""; OUT="" @@ -42,12 +47,10 @@ while [ $# -gt 0 ]; do done mkdir -p "$OUT" -# Absolutize OUT before any `cd` below: the CosmoCov binary writes its block -# files into cwd, so we cd into OUT (line ~61); every other OUT-relative path -# ($INI, block logs, covariance.txt, cosmocov_process output) must therefore be -# absolute or it re-resolves against the new cwd and double-nests. lc templates -# {output} as a project-relative path, so this makes the recipe robust to both -# relative (lc) and absolute (direct-run) --out. +# Absolutize OUT before the `cd "$OUT"` below (the CosmoCov binary writes its +# blocks into cwd): every other OUT-relative path would otherwise re-resolve +# against the new cwd and double-nest. lc templates {output} as a +# project-relative path, so both relative and absolute --out must work. OUT="$(cd "$OUT" && pwd)" INI="$OUT/covariance.ini" @@ -66,13 +69,13 @@ module unload intelpython 2>/dev/null || true; module load intelpython/3-2024.1. module load openmpi cd "$OUT" -# BLOCK_PAIRS = [("++","1"), ("--","2"), ("+-","3")] — one CosmoCov invocation per block +# One CosmoCov invocation per block; see common.py BLOCK_PAIRS. for idx in 1 2 3; do ( "$COSMOCOV" "$idx" "$INI" > "$OUT/cosmocov_block_${idx}.log" 2>&1 ) & done wait -# Concatenate blocks in BLOCK_PAIRS order (++, --, +-) — as covariance_cat does +# Concatenate blocks in BLOCK_PAIRS order (++, --, +-), as covariance_cat does. CAT="$OUT/covariance.txt" : > "$CAT" for pm_idx in "++:1" "--:2" "+-:3"; do diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index 9df0bbfd..bda566f8 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -48,9 +48,8 @@ cv.calculate_rho_tau_stats() -# Confirm CosmologyValidation produced the requested outputs. calculate_rho_tau_stats -# writes the rho/tau FITS *and* the born-as-SACC rho_tau part (via -# rho_tau_to_sacc_part); the part feeds the assemble_sacc rule. +# Confirm CosmologyValidation produced the requested outputs: the rho/tau FITS +# and the born-as-SACC rho_tau part the assemble_sacc rule consumes. outputs = snakemake.output # type: ignore for label in ("rho_stats", "tau_stats", "rho_tau"): target = Path(outputs[label]) From 9bdc437c6ca41c0022def5915c03b4e37c6c8701 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 03:59:57 +0200 Subject: [PATCH 041/160] simplify: trim blinding comments to Google style; dedupe and reshape Comments/docstrings: state each load-bearing invariant once at its home (custody triple in blinding's module docstring, scheme-mismatch in draw_scheme, assembly gate in assert_consistent_blind, NaN guard in _concealing_factor) and point at it from elsewhere. Code: - blinding_paths: new dependency-free module owning init_paths/part_paths; blinding re-exports it and workflow/common imports it instead of hand-mirroring the layout. - coerce_fields: one unknown-key-check + float-coercion helper behind both from_overrides and config_digest (digests unchanged). - _pairs, _apply_blocks, functools.partial: collapse the duplicated pair-discovery and blind/unblind block loops. - blinding.verify() is public and drives the CLI, which drops its duplicate overwrite pre-check in favour of the library's FileExistsError. - Memoize WeakLensingTracer per (cosmology, bin), mirroring _COSMO_CACHE; share the CAMB matter-power z-grid between make_camb_params and xi_camb. - CosmologyValidation takes blind_root at init and resolves each version's commitment itself, instead of threading commitment_path through the three part writers (fixes the latent one-commitment-for-every-version stamp in the psf_systematics per-version loop). - blinding.smk derives BLINDABLE_STEM from the name-builders, and its two rules shell out to scripts/blind_data_vector.py; the workflow/scripts wrappers are gone. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014hSQTC6WwTH1p9w4FuKJGz --- scripts/blind_data_vector.py | 80 +-- src/sp_validation/blinding.py | 572 +++++++----------- src/sp_validation/blinding_paths.py | 26 + src/sp_validation/blinding_theory.py | 156 +++-- src/sp_validation/cosmo_val/core.py | 19 + src/sp_validation/cosmo_val/cosebis.py | 16 +- .../cosmo_val/psf_systematics.py | 28 +- src/sp_validation/cosmo_val/pure_eb.py | 10 +- src/sp_validation/cosmo_val/sacc_writers.py | 2 +- src/sp_validation/sacc_io.py | 34 +- src/sp_validation/tests/test_blinding.py | 20 +- .../tests/test_blinding_wiring.py | 45 +- workflow/Snakefile | 5 +- workflow/common.py | 83 ++- workflow/rules/blinding.smk | 71 +-- workflow/rules/cosmo_val.smk | 19 +- workflow/rules/twopoint.smk | 22 +- workflow/scripts/assemble_sacc.py | 5 +- workflow/scripts/blind_init.py | 12 - workflow/scripts/blind_part.py | 19 - workflow/scripts/cv_cosebis.py | 4 - workflow/scripts/cv_pure_eb.py | 11 +- workflow/scripts/run_2pcf.py | 6 +- workflow/scripts/run_rho_tau.py | 11 +- 24 files changed, 485 insertions(+), 791 deletions(-) create mode 100644 src/sp_validation/blinding_paths.py delete mode 100644 workflow/scripts/blind_init.py delete mode 100644 workflow/scripts/blind_part.py diff --git a/scripts/blind_data_vector.py b/scripts/blind_data_vector.py index 1050d226..e31cb5b3 100644 --- a/scripts/blind_data_vector.py +++ b/scripts/blind_data_vector.py @@ -2,24 +2,11 @@ """Script blind_data_vector.py -Per-part data-vector blinding with :mod:`sp_validation.blinding` -(Smokescreen-fork concealment, hash-commitment custody). - -``blind-init`` runs once per catalogue version: it draws an OS-entropy seed -(never printed, never written in plaintext), publishes a repo-committable -``commitment.json`` (the seed commitment + the config digest + the installed -Smokescreen fork's draw scheme), and encrypts the seed into a Fernet bundle. -Those three, not the seed alone, are what reproduces a blind (see -``blinding.draw_scheme``). ``blind-part`` blinds one intermediate part SACC (reporting ξ±, integration ξ±, -or pseudo-Cℓ) under that fixed state, escrows the true vector into a per-part -encrypted bundle beside the blinded output, and deletes the plaintext part. -``unblind`` verifies all three commitments and restores a true part -(bit-for-bit when the part's escrow bundle is beside it); it also works on the -assembled ``{version}.sacc`` (integration rows selected by the ``grid`` tag). -``verify`` is a cheap, seedless check that a blinded file matches a commitment -— seedless, but not environment-independent: it also compares both recorded -draw schemes against the installed fork, so it reports a problem on a machine -whose Smokescreen draws differently even when file and commitment agree. +CLI for :mod:`sp_validation.blinding`. ``blind-init`` fixes the blind for one +catalogue version; ``blind-part`` conceals one intermediate part SACC under it, +escrowing the true vector and deleting the plaintext; ``unblind`` verifies the +custody triple and restores a true part (or the assembled file); ``verify`` is +a cheap seedless check of a blinded file against a commitment. :Authors: Cail Daley @@ -66,21 +53,10 @@ def _blind_init(args): config = _config_from_args(args) blind_dir = pathlib.Path(args.blind_dir) blind_dir.mkdir(parents=True, exist_ok=True) - # Refuse before drawing anything: a blind is a one-shot custody event and - # silently overwriting a previous blind's state would destroy the record - # tying that blind to its seed. - clashes = [ - p - for p in blinding.init_paths(str(blind_dir)).values() - if pathlib.Path(p).exists() - ] - if clashes: - raise SystemExit( - "refusing to overwrite existing blind state:\n " - + "\n ".join(clashes) - + "\nPick a fresh --blind-dir (never overwrite a blind)." - ) - blinding.blind_init(str(blind_dir), config=config, label=args.label) + try: + blinding.blind_init(str(blind_dir), config=config, label=args.label) + except FileExistsError as exc: + raise SystemExit(f"{exc}\nPick a fresh blind dir (never overwrite a blind).") print( "Commit the commitment JSON to the repo; keep the bundle + key safe " "and separated (colocation in the blind dir is not at-rest protection)." @@ -106,44 +82,12 @@ def _unblind(args): def _verify(args): - """Seedless check: blinded-file metadata ↔ commitment JSON ↔ this install. - - No seed is read, so this cannot confirm that the blind is *subtractable* — - only that the file's three custody stamps agree with the commitment, and - that the recorded draw scheme is the one this install implements. That last - check makes the result environment-dependent by design: a machine carrying a - different Smokescreen could not unblind the file, so it reports a problem. - - Loads with ``allow_unblinded=True``: the whole job here is to report on a - file's custody state, including the state where the file is not concealed at - all, which the fail-closed loader would otherwise raise on before any - diagnostic could be assembled. - """ + # allow_unblinded=True: reporting that a file is *not* concealed is one of + # the outcomes here, so the fail-closed loader must not pre-empt it. s = sacc_io.load(args.blinded, allow_unblinded=True) with open(args.commitment, encoding="utf-8") as f: commitment = json.load(f) - problems = [] - if not s.metadata.get("concealed"): - problems.append("file is not marked concealed") - if s.metadata.get("blind_commitment") != commitment["seed_commitment"]: - problems.append("blind_commitment does not match the committed seed commitment") - if s.metadata.get("blind_config_digest") != commitment["config_digest"]: - problems.append("blind_config_digest does not match the committed digest") - # The draw scheme is checked two ways: file ↔ commitment, and the file's - # against the installed fork (see blinding.draw_scheme). - file_scheme = s.metadata.get("blind_draw_scheme") - committed = commitment.get("draw_scheme") - if file_scheme != committed: - problems.append( - f"blind_draw_scheme {file_scheme!r} does not match the committed " - f"draw_scheme {committed!r}" - ) - try: - blinding._assert_draw_scheme(file_scheme, "the blinded file") - except ValueError as exc: - problems.append(str(exc)) - if "seed_smokescreen" in s.metadata: - problems.append("PLAINTEXT SEED LEAKED into file metadata (seed_smokescreen)") + problems = blinding.verify(s, commitment) if problems: raise SystemExit("verification FAILED:\n " + "\n ".join(problems)) print( diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 97fb7121..90f93023 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -2,71 +2,38 @@ :Name: blinding.py -:Description: Smokescreen blinding wiring, per part, at birth. Each blindable - intermediate SACC product — reporting ξ±, integration ξ±, pseudo-Cℓ — is shifted, - the moment the pipeline computes it, by a difference of theory vectors - between the fiducial cosmology and a *hidden* cosmology drawn inside a - fixed amplitude envelope, so no one can read S8 off the data until the - collaboration agrees to unblind (Muir et al. 2019: - ``d → d + t(hidden) − t(fiducial)``). Only blinded parts persist on disk. - - **The fork is the concealment engine.** The hidden cosmology is drawn by - the ``UNIONS-WL/Smokescreen`` fork's own CCL-native, per-key-independent - draw. Blinding is a vector operation, so this module calls the fork's - vector core directly — ``smokescreen.concealing_factor(fiducial_params, - shifts_dict, *, seed, theory_fn)``, which returns - ``t(hidden) − t(fiducial)`` and never sees a SACC. This module supplies - the two sp_validation-specific pieces: the three ``theory_fn`` backends - matching each part's row layout (reporting ξ±, integration ξ±, pseudo-Cℓ) - and the SACC handling around the returned factors. - - **Envelope calibration.** The blinding intent is an amplitude smear of a - chosen (S8, Ωm) box. The fork draws in CCL-native primitives, so - :meth:`BlindingConfig.shifts_dict` maps the intended box to - ``{sigma8, Omega_c}`` half-widths evaluated at the fiducial — - ``σ8 = S8/√(Ωm/0.3)``, ``Ω_c = Ωm − Ω_b − Ω_ν``. Under the installed draw - scheme the deltas depend only on ``(key, seed, shift_distr)``, so one seed - yields one hidden cosmology across all part passes and the blinded parts - are mutually consistent by construction. That "only" is a property of the - *draw scheme*, not of the seed: :func:`draw_scheme` records which scheme a - blind was drawn under and every custody gate refuses an install that - disagrees (see **Custody** below). - - **Derived statistics are born blinded.** COSEBIs and pure-E/B are never - touched by this module: the pipeline's own estimators - (:mod:`sp_validation.b_modes`) run in their normal downstream place on - the already-blinded integration ξ± part, so their outputs are born blinded. - Covariance and ρ/τ PSF diagnostics are never blinded — blinding hides - the vector, not the uncertainty, and the shift is pure E-mode so the - B-mode null tests stay honest under the blind. - - **Custody: hash commitment, no keyholder.** :func:`blind_init` runs once - per catalogue version: it draws an OS-entropy seed, publishes - the seed commitment (:func:`seed_commitment`), a canonical config digest, - and the installed fork's ``DRAW_SCHEME`` as a repo-committable - ``commitment.json``, and encrypts the seed into a Fernet bundle - (``smokescreen.encryption``) — the plaintext seed is never written. The - three together, not the seed alone, are what reproduces a blind (see - :func:`draw_scheme`). - - Each :func:`blind_part` call reads that fixed state, conceals one part, - escrows the part's true vector into its own encrypted bundle beside the - blinded output, and deletes the plaintext part. Every path that assembles - parts into the one-file product goes through - :func:`sp_validation.sacc_io.gather` — the production assembler is passed - *into* it rather than wrapped around it — and gather calls - :func:`assert_consistent_blind` to fail closed unless every blindable part - carries the identical ``blind_commitment``, ``blind_config_digest`` and - ``blind_draw_scheme``. :func:`unblind_part` verifies all three against the - commitment *before* subtracting anything, then restores the true part. - - Custody has no back door, including for its own history: a blind carrying no - scheme record is refused by :func:`_assert_draw_scheme` ahead of every seed - read, with no CLI override — none exists, the scheme having been bound - before the first real blind. +:Description: Smokescreen blinding, per part, at birth. Each blindable + intermediate SACC product — reporting ξ±, integration ξ±, pseudo-Cℓ — is + shifted the moment the pipeline computes it by a difference of theory + vectors between the fiducial and a *hidden* cosmology drawn inside a fixed + amplitude envelope (Muir et al. 2019: ``d → d + t(hidden) − t(fiducial)``), + so S8 cannot be read off the data before the collaboration unblinds. Only + blinded parts persist on disk. + + The ``UNIONS-WL/Smokescreen`` fork draws the hidden cosmology and computes + the shift; blinding is a vector operation, so this module calls its vector + core (``smokescreen.concealing_factor``), which never sees a SACC. What is + sp_validation-specific is supplied here: the ``theory_fn`` backends + matching each part's row layout, and the SACC handling around the factors. + + Derived statistics are born blinded — COSEBIs and pure-E/B run downstream + on the already-blinded integration ξ±. Covariance and ρ/τ are never blinded: + blinding hides the vector, not the uncertainty, and the shift is pure + E-mode so B-mode null tests stay honest under the blind. + + **Custody: hash commitment, no keyholder.** A blind is reproduced by a + *triple* — seed, config digest, and Smokescreen ``DRAW_SCHEME`` (the seed + alone is not enough; see :func:`draw_scheme`). :func:`blind_init` fixes all + three per catalogue version, publishing the commitment triple as a + repo-committable ``commitment.json`` and encrypting the seed into a Fernet + bundle — the plaintext seed is never written. Every gate that draws or + subtracts a shift fails closed unless the triple matches + (:func:`_assert_draw_scheme`, :func:`assert_consistent_blind`, + :func:`unblind_sacc`), and there is no override. """ import dataclasses +import functools import hashlib import json import os @@ -76,7 +43,8 @@ import numpy as np from . import sacc_io -from .blinding_theory import TheoryConfig, cl_ee, xi_ccl, xi_ell_grid +from .blinding_paths import init_paths, part_paths # noqa: F401 (re-exported) +from .blinding_theory import TheoryConfig, cl_ee, coerce_fields, xi_ccl, xi_ell_grid # --------------------------------------------------------------------------- # @@ -103,13 +71,8 @@ def shifts_dict(self): Evaluated at the fiducial: a ΔS8 half-width maps to ``ΔS8/√(Ωm_fid/0.3)`` in σ8 (at fixed Ωm), and a ΔΩm half-width maps - one-to-one to Ω_c (Ω_b and Ω_ν are fixed). Exact enough for a - blinding smear — the target is a characteristic amplitude, not a - precise (S8, Ωm) posterior. Under the installed draw scheme the fork - draws each key independently as ``U(fid − h, fid + h)``, so adding or - resizing one key never moves another; :func:`draw_scheme` is what - makes that independence a guarantee rather than a hope, by refusing an - install whose draw semantics differ from the blind's. + one-to-one to Ω_c (Ω_b and Ω_ν fixed). Exact enough for a smear whose + target is a characteristic amplitude, not a precise posterior. """ return { "sigma8": self.s8_half_width / np.sqrt(self.theory.Omega_m / 0.3), @@ -119,32 +82,28 @@ def shifts_dict(self): def config_digest(self): """sha256 of a canonical serialization of the full blinding config. - Binds the envelope half-widths, the complete fiducial - :class:`TheoryConfig` (cosmology + the two ``halofit_version`` - tokens + IA fields) into one digest: JSON with sorted keys over the - full ordered field set. Every ``float``-declared field is coerced - through :func:`float` before serialization, so the digest depends on - the numeric *value*, not on whether a config wrote ``-1`` or ``-1.0`` - — an int-vs-float literal difference (routine when configs come from - YAML/CLI/humans) can no longer split one physical cosmology into two - digests and deny a legitimate unblind. Python's ``json`` then emits - each float via its shortest round-trip ``repr``, so two runs of the - same config produce byte-identical digests. Checked (with - the seed commitment) at unblind, so a wrong envelope or a mismatched - P(k) recipe cannot silently subtract a wrong shift. + Binds the envelope half-widths and the complete fiducial + :class:`TheoryConfig` into one digest, as JSON with sorted keys. + Fields go through :func:`~sp_validation.blinding_theory.coerce_fields`, + so the digest depends on the numeric value rather than on an + int-vs-float literal, and ``json`` emits floats by shortest round-trip + ``repr`` — two runs of one config give byte-identical digests. Checked + with the seed commitment at unblind. """ - payload = { - "s8_half_width": float(self.s8_half_width), - "omega_m_half_width": float(self.omega_m_half_width), - "theory": { - f.name: ( - float(getattr(self.theory, f.name)) - if f.type in (float, "float") - else getattr(self.theory, f.name) - ) + payload = coerce_fields( + type(self), + { + "s8_half_width": self.s8_half_width, + "omega_m_half_width": self.omega_m_half_width, + }, + ) + payload["theory"] = coerce_fields( + TheoryConfig, + { + f.name: getattr(self.theory, f.name) for f in dataclasses.fields(self.theory) }, - } + ) return hashlib.sha256( json.dumps(payload, sort_keys=True).encode("utf-8") ).hexdigest() @@ -156,17 +115,7 @@ def from_overrides(cls, overrides): ``theory`` may be given as a :class:`TheoryConfig` or a mapping of TheoryConfig overrides, mirroring :meth:`TheoryConfig.from_overrides`. """ - by_name = {f.name: f for f in dataclasses.fields(cls)} - unknown = set(overrides) - set(by_name) - if unknown: - raise ValueError( - f"unknown BlindingConfig fields {sorted(unknown)}; " - f"valid fields are {sorted(by_name)}" - ) - overrides = { - name: (float(v) if by_name[name].type in (float, "float") else v) - for name, v in overrides.items() - } + overrides = coerce_fields(cls, overrides) theory = overrides.get("theory") if theory is not None and not isinstance(theory, TheoryConfig): overrides["theory"] = TheoryConfig.from_overrides(dict(theory)) @@ -179,16 +128,8 @@ def from_overrides(cls, overrides): def seed_commitment(seed): """Public commitment for a seed: the fork's domain-separated sha256 digest. - Safe to publish and commit to the repo — it ties a blinded file to its - blind without revealing the seed, and lets unblind refuse a wrong seed. - - This delegates to :func:`smokescreen.seed_commitment` so the commitment has - exactly one definition across the blinding stack. That function hashes - ``smokescreen.COMMITMENT_DOMAIN + str(seed)``, *not* the bare seed, and the - domain prefix is load-bearing: the fork derives the RNG base seed from the - bare sha256 of the same string, so an undomained commitment would publish - that base seed verbatim in its first 16 hex characters and the blind would - be recoverable from the artifact meant to protect it. + Domain-separated because the fork derives its RNG base seed from the bare + sha256 of the same string — an undomained commitment would publish it. """ from smokescreen import seed_commitment as _fork_seed_commitment @@ -198,20 +139,15 @@ def seed_commitment(seed): def draw_scheme(): """The installed Smokescreen fork's shift-draw semantics version. - ``smokescreen.DRAW_SCHEME`` is the fork's own version number for *how* a - seed becomes a set of parameter deltas — scheme 1 is upstream DESC's one - global RNG stream consumed over the sorted keys, scheme 2 (this fork) a - per-key RNG derived from ``(seed, key)``. Two installs that agree on - ``(seed, config)`` but disagree on this number draw *different* hidden - cosmologies from the same inputs. - - That is the one blinding failure with no loud symptom: a blind added under - one scheme and subtracted under another leaves a residual smooth - cosmological shift in the "unblinded" vector, which every hash, digest and - escrow check would still pass. So the scheme is bound into the blind's - custody state — ``commitment.json`` and every blinded file's - ``blind_draw_scheme`` — and re-checked against this function wherever a - shift is drawn or subtracted (:func:`_assert_draw_scheme`). + ``smokescreen.DRAW_SCHEME`` versions *how* a seed becomes parameter deltas + (scheme 1: upstream DESC's single global RNG over the sorted keys; scheme + 2: this fork's per-key RNG from ``(seed, key)``). Two installs agreeing on + ``(seed, config)`` but not on this number draw different hidden cosmologies. + + This is the one blinding failure with no loud symptom — a scheme mismatch + leaves a smooth residual cosmological shift in the "unblinded" vector, and + every hash, digest and escrow check still passes. Hence the scheme is + custody state, checked wherever a shift is drawn or subtracted. """ from smokescreen import DRAW_SCHEME @@ -248,16 +184,10 @@ def _assert_draw_scheme(recorded, what): def hidden_params(seed, config): """The hidden CCL parameter point the fork realizes for ``(seed, config)``. - Re-runs the fork's own draw (``smokescreen.param_shifts.draw_param_shifts`` - — per-key-independent, local RNG) on the calibrated envelope and overlays - the deltas on the fiducial, exactly as - ``smokescreen.concealing_factor`` does internally. Deterministic under a - fixed draw scheme: same ``(seed, config)`` ⇒ same hidden point on any - install whose :func:`draw_scheme` matches — the reproducibility contract - unblinding relies on, and what makes every part share one hidden cosmology - under one seed. This is an introspection helper (what *was* the hidden - cosmology, once revealed); the blinding path itself never calls it, and so - does not check the scheme here — every gate that acts on a shift does. + Re-runs the fork's draw and overlays the deltas on the fiducial, as + ``concealing_factor`` does internally. Introspection only (what *was* the + hidden cosmology, once revealed) — the blinding path never calls it, so it + does not gate on the draw scheme; every gate that acts on a shift does. """ from smokescreen.param_shifts import draw_param_shifts @@ -278,26 +208,25 @@ def source_bins(s): ) -def xi_pairs(s, grid): - """Unordered source-bin pairs ``(i ≤ j)`` carrying ξ+ on ``grid``.""" +def _pairs(s, data_type, **tags): + """Unordered source-bin pairs ``(i ≤ j)`` carrying ``data_type`` in ``s``.""" bins = source_bins(s) return [ (i, j) for a, i in enumerate(bins) for j in bins[a:] - if len(s.indices(sacc_io.XI_PLUS, sacc_io._pair((i, j)), grid=grid)) + if len(s.indices(data_type, sacc_io._pair((i, j)), **tags)) ] +def xi_pairs(s, grid): + """Unordered source-bin pairs ``(i ≤ j)`` carrying ξ+ on ``grid``.""" + return _pairs(s, sacc_io.XI_PLUS, grid=grid) + + def cl_pairs(s): """Source-bin pairs ``(i ≤ j)`` carrying pseudo-Cℓ_EE.""" - bins = source_bins(s) - return [ - (i, j) - for a, i in enumerate(bins) - for j in bins[a:] - if len(s.indices(sacc_io.CL_EE, sacc_io._pair((i, j)))) - ] + return _pairs(s, sacc_io.CL_EE) def _xi_indices(s, grid): @@ -334,12 +263,10 @@ def _pair_nz(s, i, j): def xi_theory_fn(block, theory, grid): """``theory_fn`` for a ξ± sub-SACC block (reporting or integration grid). - Reads each bin's n(z) directly from the block's own tracers and lays the - output out to match the block's SACC rows element-for-element: for every - pair, ξ± is computed at that pair's stored θ (the ``theta`` tag, - arcmin) and scattered to the rows ``block.indices`` reports — the - block's own row order, never an assumed pairing. IA enters from - ``theory``'s NLA fields, identically at every parameter point. + Reads each bin's n(z) from the block's own tracers and lays the output out + to match the block's SACC rows element-for-element: per pair, ξ± at that + pair's stored θ (the ``theta`` tag, arcmin), scattered to the rows + ``block.indices`` reports — never an assumed pairing. """ pairs = xi_pairs(block, grid) layout = [] @@ -366,12 +293,10 @@ def theory_fn(params): def cl_theory_fn(block, theory): """``theory_fn`` for the pseudo-Cℓ_EE sub-SACC block. - Per pair: theory Cℓ_EE on the stored ``BandpowerWindows`` support, - binned by the same window matrix the measurement used (``W @ Cℓ_EE``), - scattered to the block's own rows. The concealing factor the fork forms - from this is ``W @ ΔCℓ_EE`` — the shift lands in the measured - bandpowers; ΔBB = ΔEB ≡ 0 by construction (pure E-mode shift), so those - blocks are simply not part of this backend's rows. + Per pair: theory Cℓ_EE on the stored ``BandpowerWindows`` support, binned + by the same window matrix the measurement used (``W @ Cℓ_EE``), scattered + to the block's own rows — so the shift lands in the measured bandpowers. + ΔBB = ΔEB ≡ 0 for a pure E-mode shift, hence EE-only rows. """ layout = [] for i, j in cl_pairs(block): @@ -403,23 +328,17 @@ def theory_fn(params): def _blindable_blocks(s): """The blindable blocks of a SACC as ``(name, indices, factory)``. - Works identically on a standalone part (which carries exactly one block) - and on the assembled one-file product (whose integration rows are selected by - the ``grid`` tag — the layout contract's per-block tag selection). - ``indices`` are each block's recorded row indices (ascending); ``factory`` - builds the matching ``theory_fn`` from the SACC those indices point into. - Blocks absent from the file are simply not listed. + Works identically on a standalone part (exactly one block) and on the + assembled file (integration rows selected by the ``grid`` tag). ``indices`` + are each block's recorded row indices (ascending); ``factory`` builds the + matching ``theory_fn``. Blocks absent from the file are not listed. """ blocks = [] for grid in ("reporting", "integration"): idx = _xi_indices(s, grid) if len(idx): blocks.append( - ( - f"{grid} ξ±", - idx, - lambda sub, theory, grid=grid: xi_theory_fn(sub, theory, grid), - ) + (f"{grid} ξ±", idx, functools.partial(xi_theory_fn, grid=grid)) ) idx = _cl_ee_indices(s) if len(idx): @@ -430,24 +349,17 @@ def _blindable_blocks(s): def _concealing_factor(s, indices, factory, config, seed): """The fork-computed additive concealing factor for one block of ``s``. - Blinding is a vector operation, so this goes straight to the fork's vector - core: ``smokescreen.concealing_factor`` draws the hidden deltas from - ``seed``, overlays them on the fiducial, evaluates the block's - ``theory_fn`` at both points and differences them. No data vector and no - SACC reach the fork — nothing is carved out of ``s``, and ``s`` itself is - not modified. - - The factory reads its layout off ``s`` directly, so the returned vector is - full-length: a block's ``theory_fn`` fills only its own rows and leaves - every other row NaN. Slicing to ``indices`` drops those NaNs by - construction; the finite check then proves the converse — that the - factory filled *all* of this block's rows. A row the block claims but the - factory cannot cover (a pair carrying ξ− without ξ+, say) would otherwise - write NaN into the data vector, silently. - - Both :func:`blind_sacc` and :func:`unblind_sacc` reach the fork through - this one function, so the added and the subtracted shift cannot drift - apart. + ``smokescreen.concealing_factor`` draws the hidden deltas from ``seed``, + evaluates the block's ``theory_fn`` at both cosmologies and differences + them. No data vector and no SACC reach the fork, and ``s`` is not modified. + Both :func:`blind_sacc` and :func:`unblind_sacc` come through here, so the + added and subtracted shifts cannot drift apart. + + A ``theory_fn`` fills only its own rows and leaves the rest NaN, so + slicing to ``indices`` drops the NaNs by construction and the finite check + proves the converse: that *all* of this block's rows were filled. A row + the block claims but the factory cannot cover (a pair with ξ− but no ξ+, + say) would otherwise be shifted by NaN, silently. Returns ------- @@ -488,34 +400,38 @@ def _concealed(s): return bool(s.metadata.get("concealed")) +def _apply_blocks(src, seed, config, sign, verb, log): + """Return a copy of ``src`` with each block's concealing factor applied. + + ``sign`` is ``+1`` to conceal and ``-1`` to reveal; the factor itself comes + from the one :func:`_concealing_factor` call both directions share. + """ + dst = src.copy() + for name, indices, factory in _blindable_blocks(src): + factor = _concealing_factor(src, indices, factory, config, seed) + _set_values(dst, indices, np.asarray(dst.mean)[indices] + sign * factor) + log(f"[{verb}] {name}: {len(indices)} points") + return dst + + def blind_sacc(part, seed, config=None, label="smokescreen", log=print): """Return a blinded copy of a part SACC (covariance and tags untouched). - Per blindable block present (a standalone part carries exactly one — - reporting ξ±, integration ξ±, or pseudo-Cℓ_EE): ask the fork for the - block's concealing factor (:func:`_concealing_factor`) and add it at the - block's recorded indices. Row order, tags, n(z) and covariance are - untouched — only ``value`` changes, and only on blindable rows. - Provenance is stamped and any leaked seed key stripped. A file with no - blindable block (e.g. a ρ/τ diagnostic part) is refused loudly — it should - never see a blind call. + Adds each blindable block's concealing factor at the block's recorded + indices; only ``value`` changes, and only on blindable rows. Provenance is + stamped and any leaked seed key stripped. A file with no blindable block + (a ρ/τ diagnostic part, say) is refused loudly. """ config = config or BlindingConfig() if _concealed(part): raise ValueError("already concealed — unblind first") - blocks = _blindable_blocks(part) - if not blocks: + if not _blindable_blocks(part): raise ValueError( "no blindable block (reporting/integration ξ± or pseudo-Cℓ_EE) in this SACC " "— ρ/τ diagnostic parts are never blinded" ) - blinded = part.copy() - for name, indices, factory in blocks: - factor = _concealing_factor(part, indices, factory, config, seed) - _set_values(blinded, indices, np.asarray(blinded.mean)[indices] + factor) - log(f"[blind] {name}: shifted {len(indices)} points via Smokescreen fork") - + blinded = _apply_blocks(part, seed, config, +1, "blind", log) _stamp_provenance(blinded, seed_commitment(seed), label, config.config_digest()) return blinded @@ -523,15 +439,11 @@ def blind_sacc(part, seed, config=None, label="smokescreen", log=print): def unblind_sacc(blinded, seed, config=None, log=print): """Recover the true part SACC from a blinded one + the revealed ``seed``. - Verifies the stamped draw scheme against the installed fork, then - the seed commitment and the config digest against the stamped metadata (loud - failure on any of the three — verification precedes subtraction), then - recomputes each block's shift through :func:`_concealing_factor`, the same - call :func:`blind_sacc` added it with, and subtracts it. Works on a - standalone part or on the assembled file (integration rows selected by the - ``grid`` tag). Derived statistics, if present (assembled file), are *not* - recomputed here — the pipeline's own estimators re-derive them from the - unblinded integration ξ±. + Verifies the custody triple — draw scheme, seed commitment, config digest — + against the file's stamps before subtracting anything, then recomputes each + block's shift the same way :func:`blind_sacc` added it. Works on a part or + on the assembled file. Derived statistics, if present, are *not* recomputed + here — the pipeline re-derives them from the unblinded integration ξ±. """ config = config or BlindingConfig() if not _concealed(blinded): @@ -549,12 +461,7 @@ def unblind_sacc(blinded, seed, config=None, log=print): "added)" ) - part = blinded.copy() - for name, indices, factory in _blindable_blocks(blinded): - factor = _concealing_factor(blinded, indices, factory, config, seed) - _set_values(part, indices, np.asarray(part.mean)[indices] - factor) - log(f"[unblind] {name}: subtracted {len(indices)} shifts") - + part = _apply_blocks(blinded, seed, config, -1, "unblind", log) for key in ( "concealed", "blind", @@ -567,20 +474,11 @@ def unblind_sacc(blinded, seed, config=None, log=print): def _stamp_provenance(s, commitment, label, config_digest): - """Stamp the blind's public provenance; strip any leaked seed. - - ``blind_commitment`` (sha256 of the seed) ties the file to its blind - without revealing it; ``blind_config_digest`` pins the envelope + - fiducial the shift was drawn against; ``blind_draw_scheme`` pins the - Smokescreen draw semantics that turned the seed into that shift (see - :func:`draw_scheme`); ``concealed``/``blind`` mark the file. The three - together are what an unblind must reproduce. ``seed_smokescreen`` — the - raw seed upstream Smokescreen's writer would stamp — is popped - defensively: the seed must never ride a kept file. - - The stamped scheme is always the installed fork's: every caller has already - checked the blind's recorded scheme against it (:func:`_assert_draw_scheme`), - so the two agree by the time this runs. + """Stamp the custody triple and the ``concealed``/``blind`` marks. + + Pops ``seed_smokescreen`` (which upstream Smokescreen's writer would + stamp): the seed never rides a kept file. The scheme stamped is the + installed fork's, which every caller has already checked the blind against. """ s.metadata.pop("seed_smokescreen", None) s.metadata["concealed"] = True @@ -593,25 +491,14 @@ def _stamp_provenance(s, commitment, label, config_digest): def stamp_concealed_passthrough(s, commitment_path): """Stamp a part concealed under an existing blind, values untouched. - A born-blinded derived statistic (COSEBIs / pure-E/B) has its E-mode vector - re-derived from the *already-blinded* integration ξ± before this call, so its - values are blind by construction and only the provenance stamp is missing. - ρ/τ carries no cosmological vector at all — the stamp merely clears it for - assembly under the blind (values unchanged either way). Both cases need the - custody keys the fail-closed load gate and :func:`assert_consistent_blind` - check: ``concealed``, ``blind`` (label), ``blind_commitment`` - (= ``seed_commitment``), ``blind_config_digest``, ``blind_draw_scheme``. This - reads those from the version's ``commitment.json`` (written by - :func:`blind_init`) and stamps them via :func:`_stamp_provenance`, so a - pass-through part shares the exact same custody state as the blinded - ξ±/pseudo-Cℓ parts. The committed draw scheme is checked against the - installed fork first: a pass-through part is blind because it was derived - from parts blinded under that scheme, so stamping it from an install that - draws differently would mint a custody claim this install cannot honour. - - Unlike :func:`blind_sacc`, this shifts nothing and does not require a - blindable block — it is the seam for parts blinded (or made blind-irrelevant) - upstream of the SACC writer. + The seam for parts already blind (COSEBIs / pure-E/B, re-derived from the + blinded integration ξ±) or blind-irrelevant (ρ/τ, no cosmological vector): + it shifts nothing and needs no blindable block, only the custody stamp that + lets the load gate and :func:`assert_consistent_blind` admit the part. The + stamp is read from the version's ``commitment.json``, so a pass-through + part carries the exact custody state of the blinded parts. The committed + scheme is checked first — stamping from an install that draws differently + would mint a custody claim it cannot honour. """ with open(commitment_path, encoding="utf-8") as f: commitment = json.load(f) @@ -633,28 +520,17 @@ def stamp_concealed_passthrough(s, commitment_path): def assert_consistent_blind(parts): """Assert every blindable part shares one blind; return the shared stamp. - Custody logic for the terminal :func:`sp_validation.sacc_io.gather`: a - part is *blindable* if it carries a blindable block (ξ± or pseudo-Cℓ_EE); - ρ/τ diagnostic and covariance-only parts are exempt. Fails closed — - ``ValueError`` — if blinded and plaintext blindable parts are mixed, or - if two parts carry different - ``blind_commitment``/``blind_config_digest``/``blind_draw_scheme`` (they - were blinded under different seeds, configs or draw semantics and must - never be combined), or if the shared draw scheme is not the installed - fork's (this install could not unblind what it is about to assemble). - The consistency key is ``(commitment, digest, scheme)`` — the - ``blind`` *label* is informational provenance, not custody state, so - parts blinded under one seed+config but tagged with different ``--label`` - values assemble cleanly (a distinct warning is logged, not a failure). - - **Unconcealed parts must be declared mocks** (PRD §4, "Mocks vs data"): - when no blindable part is concealed, every blindable part's metadata must - carry ``type == "mock"`` — the tag :mod:`sp_validation.sacc_io` stamps - when the data vector is computed. An unconcealed ``type == "data"`` part, - or one missing the tag, fails assembly closed: skipping the blind can - never silently expose real data. A concealed+plaintext mix fails closed - regardless of ``type`` (see above). A fully-mock plaintext assembly - returns ``None``. + The assembly gate of :func:`sp_validation.sacc_io.gather`. A part is + *blindable* if it carries a blindable block (ξ± or pseudo-Cℓ_EE); ρ/τ and + covariance-only parts are exempt. Fails closed on a blinded/plaintext mix, + on parts whose custody triples disagree, or on a shared scheme this install + does not implement (it could not unblind what it is assembling). The + consistency key is the triple; the ``blind`` label is provenance, not + custody, so differing labels warn rather than fail. + + Unconcealed blindable parts must all be declared ``type == "mock"``: an + unconcealed ``type == "data"`` part, or one missing the tag, fails closed, + so skipping the blind can never silently expose real data. Returns ------- @@ -685,7 +561,6 @@ def assert_consistent_blind(parts): f"({len(concealed)} of {len(blindable)} blinded) — refusing to " "combine (a plaintext part beside blinded ones leaks the shift)" ) - # Custody state is (commitment, digest, scheme) — the label is provenance. # `.get` on the scheme so a part predating scheme binding reads as None and # fails at _assert_draw_scheme with its explanation, not with a KeyError. stamps = { @@ -725,45 +600,49 @@ def assert_consistent_blind(parts): # --------------------------------------------------------------------------- # # File-level custody: blind-init / blind-part / unblind # --------------------------------------------------------------------------- # -def init_paths(blind_dir): - """The fixed custody state written by :func:`blind_init` in ``blind_dir``.""" - return { - "commitment": os.path.join(blind_dir, "commitment.json"), - "bundle": os.path.join(blind_dir, "blind_seed.encrpt"), - "key": os.path.join(blind_dir, "blind_seed.key"), - } - - -def part_paths(part_path): - """Blinded-output and escrow-bundle paths beside a part file.""" - stem, ext = os.path.splitext(part_path) - return { - "blinded": f"{stem}_blinded{ext or '.fits'}", - "escrow": f"{stem}_escrow.encrpt", - "escrow_key": f"{stem}_escrow.key", - } +def verify(s, commitment): + """Problems found comparing a blinded SACC to a commitment, seedlessly. + + Returns a (possibly empty) list of human-readable strings. No seed is read, + so this cannot confirm the blind is *subtractable* — only that the file's + custody triple matches ``commitment`` (a parsed ``commitment.json``) and + that the recorded draw scheme is the one this install implements. That last + check is environment-dependent by design: a machine carrying a different + Smokescreen could not unblind the file, so it reports a problem. + """ + problems = [] + if not _concealed(s): + problems.append("file is not marked concealed") + if s.metadata.get("blind_commitment") != commitment["seed_commitment"]: + problems.append("blind_commitment does not match the committed seed commitment") + if s.metadata.get("blind_config_digest") != commitment["config_digest"]: + problems.append("blind_config_digest does not match the committed digest") + scheme = s.metadata.get("blind_draw_scheme") + if scheme != commitment.get("draw_scheme"): + problems.append( + f"blind_draw_scheme {scheme!r} does not match the committed " + f"draw_scheme {commitment.get('draw_scheme')!r}" + ) + try: + _assert_draw_scheme(scheme, "the blinded file") + except ValueError as exc: + problems.append(str(exc)) + if "seed_smokescreen" in s.metadata: + problems.append("PLAINTEXT SEED LEAKED into file metadata (seed_smokescreen)") + return problems def blind_init(blind_dir, config=None, label="smokescreen", log=print): """Fix the blind for one catalogue version: seed, commitment, seed bundle. - Runs once per catalogue version: - - 1. Draw an OS-entropy seed (never written in plaintext, never returned). - 2. Write ``commitment.json`` (repo-committable): the seed commitment + the - canonical config digest + the installed fork's draw scheme + the blind - label. Those first three are the full reproducibility statement — see - :func:`draw_scheme` for why the seed and config alone are not. - 3. Encrypt the seed into a Fernet bundle (``smokescreen.encryption``); - the temporary plaintext is deleted by the encryptor. - - These outputs are the blind's fixed state; every :func:`blind_part` and - :func:`unblind_part` call reads them. + Draws an OS-entropy seed (never written in plaintext, never returned), + writes the repo-committable ``commitment.json`` (the custody triple plus + the label), and encrypts the seed into a Fernet bundle. Every + :func:`blind_part` and :func:`unblind_part` call reads this fixed state. Custody caveat: the bundle and its Fernet key land in the *same* - ``blind_dir``. Fernet is only as protective as the key's separation — - anyone with both files can decrypt the seed. Keep the key out-of-band; - colocation is convenience, not at-rest protection. + ``blind_dir``, and anyone with both can decrypt the seed. Keep the key + out-of-band; colocation is convenience, not at-rest protection. Returns ------- @@ -802,11 +681,8 @@ def blind_init(blind_dir, config=None, label="smokescreen", log=print): def _read_seed(blind_dir, config): """Decrypt the seed bundle and verify it against the commitment. - The seed commitment, the config digest, and the committed draw scheme (see - :func:`draw_scheme`) are all checked before the seed is handed to any caller - — a tampered bundle, a drifted config or a Smokescreen that draws - differently from the one that fixed this blind all fail loud here, whether - the caller is about to blind or to unblind. + The whole custody triple is checked before the seed is handed to any + caller, whether it is about to blind or to unblind. Returns ------- @@ -835,15 +711,12 @@ def _read_seed(blind_dir, config): def blind_part(part_path, blind_dir, config=None, keep_input=False, log=print): """Blind one intermediate part SACC at birth, under the fixed blind state. - Reads the encrypted seed and ``commitment.json`` written by - :func:`blind_init` (verifying both hashes), conceals the part through its - matching backend (:func:`blind_sacc`), writes the blinded part beside the - input with the part's true vector escrowed into a per-part Fernet bundle, - and deletes the plaintext part — only the blinded part persists on disk. - Each part's escrow is self-contained: restoring it needs only that - part's bundle plus the seed bundle; corruption of one bundle loses one - part, not all. Pass ``keep_input=True`` to retain the plaintext part - (and own the custody implication). + Reads the fixed state :func:`blind_init` wrote, conceals the part + (:func:`blind_sacc`), writes the blinded part beside the input with the + true vector escrowed into a per-part Fernet bundle, and deletes the + plaintext — only the blinded part persists. Each escrow is self-contained, + so corruption of one bundle loses one part, not all. ``keep_input=True`` + retains the plaintext part (and its custody implication). Returns ------- @@ -889,27 +762,17 @@ def blind_part(part_path, blind_dir, config=None, keep_input=False, log=print): def unblind_part(blinded_path, blind_dir, out_path, config=None, log=print): """Unblind one blinded part (or the assembled file), verifying first. - Decrypts the seed bundle and verifies the seed commitment, the config digest - and the draw scheme against ``commitment.json``, and the same three - against the blinded file's own stamps (fail closed on any — - verification precedes subtraction), recomputes the part's shift from the - seed and subtracts it (:func:`unblind_sacc`). **The seed-subtracted - vector is the authority** — the seed plus its commitment is the custody - root of trust, and the returned vector is what the seed math produced. - - When the part's escrow bundle exists beside the blinded file it serves - two subordinate roles, and only after its stored ``seed_commitment`` is - verified to match the same commitment (so an escrow from a different - blind can never be trusted): (1) a *tighter equality check* — the seed - subtraction and the escrowed truth must agree to ``1e-6`` relative or the - unblind fails closed; (2) *ulp-residue removal* — float add-then-subtract - leaves ~ulp residue against the pre-blind truth, and once the escrow is - bound to this commitment its exact value clears that residue so the - restore is bit-for-bit. The escrow is never the source of correctness: - if it disagrees materially the guard raises, and a mismatched or - unbound escrow never determines the output. On the assembled file — no - escrow — the subtraction stands alone, with integration rows selected by the - ``grid`` tag. + Verifies the custody triple against ``commitment.json`` and against the + file's own stamps, then subtracts the seed-recomputed shift + (:func:`unblind_sacc`). The seed-subtracted vector is the authority. + + A part's escrow bundle, when present beside the blinded file and only once + its stored ``seed_commitment`` is confirmed to be this blind's, plays two + subordinate roles: a tighter equality check (disagreement beyond ``1e-6`` + relative fails closed) and removal of the ~ulp residue float + add-then-subtract leaves, making the restore bit-for-bit. It is never the + source of correctness. The assembled file has no escrow and the + subtraction stands alone. """ config = config or BlindingConfig() seed, commitment = _read_seed(blind_dir, config) @@ -940,10 +803,8 @@ def unblind_part(blinded_path, blind_dir, out_path, config=None, log=print): f"unblinded vector disagrees with the escrowed true vector " f"(max rel {residual:.2e}) — wrong escrow for this part?" ) - # Seed-bound escrow: clear the add-then-subtract ulp residue so the - # restore is bit-for-bit. Correctness already came from the seed - # subtraction above; the guard proved the escrow agrees with it. - _set_values(part, np.arange(len(true_mean)), true_mean) + # Seed-bound escrow: clear the add-then-subtract ulp residue. + _set_values(part, range(len(true_mean)), true_mean) log(f"[unblind] escrow verified (subtraction residual {residual:.2e})") # unblind_sacc stripped the concealed/blind stamps, so this is the true # revealed vector; it inherits the blinded file's provenance (data vs mock). @@ -953,17 +814,12 @@ def unblind_part(blinded_path, blind_dir, out_path, config=None, log=print): def _write_encrypted_json(encrpt_path, payload): - """Encrypt ``payload`` (JSON) to ``encrpt_path`` + sibling ``.key``; no - plaintext survives. - - We drive ``smokescreen.encryption.encrypt_file`` (Fernet) for the crypto - but write the ciphertext and key at the exact names we control. Its - ``save_file`` mode names outputs from ``basename.split('.')[0]`` — it - truncates at the first dot — so for a dotted stem (the canonical - catalogue-version case, e.g. ``v1.4.6.3_xi_integration_escrow``) it would land at - ``v1.encrpt``/``v1.key``, diverging from what :func:`part_paths` declares - and colliding across parts that share a first-dot prefix. Instead we take - the returned ``(ciphertext, key)`` and write them ourselves. + """Encrypt ``payload`` (JSON) to ``encrpt_path`` + sibling ``.key``. + + Smokescreen's ``save_file`` mode names outputs from + ``basename.split('.')[0]``, which truncates our dotted catalogue-version + stems (``v1.4.6.3_…`` → ``v1.encrpt``) and collides across parts, so we + take the returned ``(ciphertext, key)`` and write them ourselves. """ from smokescreen.encryption import encrypt_file diff --git a/src/sp_validation/blinding_paths.py b/src/sp_validation/blinding_paths.py new file mode 100644 index 00000000..42ddb415 --- /dev/null +++ b/src/sp_validation/blinding_paths.py @@ -0,0 +1,26 @@ +"""Blinding file-name conventions, with no dependencies. + +Separate from :mod:`sp_validation.blinding` so the Snakemake DAG build can +import the path layout without pulling in numpy, CCL and smokescreen. +""" + +import os + + +def init_paths(blind_dir): + """The fixed custody state ``blinding.blind_init`` writes in ``blind_dir``.""" + return { + "commitment": os.path.join(blind_dir, "commitment.json"), + "bundle": os.path.join(blind_dir, "blind_seed.encrpt"), + "key": os.path.join(blind_dir, "blind_seed.key"), + } + + +def part_paths(part_path): + """Blinded-output and escrow-bundle paths beside a part file.""" + stem, ext = os.path.splitext(str(part_path)) + return { + "blinded": f"{stem}_blinded{ext or '.fits'}", + "escrow": f"{stem}_escrow.encrpt", + "escrow_key": f"{stem}_escrow.key", + } diff --git a/src/sp_validation/blinding_theory.py b/src/sp_validation/blinding_theory.py index b44f7e73..de758aa9 100644 --- a/src/sp_validation/blinding_theory.py +++ b/src/sp_validation/blinding_theory.py @@ -6,20 +6,8 @@ configuration (:class:`TheoryConfig`) and two independent routes to the tomographic shear two-point prediction. - **Division of responsibility** (per the UNIONS layering: ``cs_util`` is - the cosmology *library* — generic machinery; ``sp_validation`` holds - *configuration* and *survey-specific implementations*). This module lives - in the blinding namespace because :class:`TheoryConfig` is configuration - and the master-layout theory backends in :mod:`sp_validation.blinding` - (reporting ξ±, integration ξ±, pseudo-Cℓ) are survey-specific. The **generic** - theory machinery below — the CCL-native ξ± path (:func:`xi_ccl`, - :func:`cl_ee`), the independent CAMB P(k)→``Pk2D`` path (:func:`xi_camb`), - and the σ8/A_s rescale (:func:`camb_As_for_sigma8`) — lives here **for - now** but is destined for ``cs_util.cosmo`` (tracked in cs_util#80): it is - cosmology-library code, not blinding-specific. There is deliberately **no** - ``sp_validation/cosmology.py`` — ``develop`` removed the local cosmology - module (#223) and moved cosmology to ``cs_util.cosmo``; this module does - not resurrect it. + The generic cosmology machinery here is destined for ``cs_util.cosmo`` + (cs_util#80). Two independent routes to the shear two-point prediction: @@ -54,6 +42,32 @@ NEFF = 3.046 T_CMB = 2.7255 +# Matter-power redshift grid shared by `make_camb_params` and `xi_camb`, so the +# CAMB run and the Pk2D built from it sample the same redshifts. +PK_ZMAX = 3.0 +PK_NZ = 48 + + +def coerce_fields(cls, overrides): + """Validate ``overrides`` against ``cls``'s fields, coercing floats. + + Unknown keys raise. Every ``float``-declared field goes through + :func:`float`, so a YAML/CLI ``w0: -1`` (int) yields the same value — and + the same config digest — as the float default ``-1.0``. Digest stability + depends on this, so it is one helper rather than three copies. + """ + by_name = {f.name: f for f in dataclasses.fields(cls)} + unknown = set(overrides) - set(by_name) + if unknown: + raise ValueError( + f"unknown {cls.__name__} fields {sorted(unknown)}; " + f"valid fields are {sorted(by_name)}" + ) + return { + name: (float(v) if by_name[name].type in (float, "float") else v) + for name, v in overrides.items() + } + # --------------------------------------------------------------------------- # # Configuration surface — the ONE place fiducial cosmology + model choices live @@ -73,17 +87,9 @@ class TheoryConfig: converted to CCL's native ``sigma8``/``Omega_c`` by :meth:`sigma8` / :meth:`omega_c`. - **Nonlinear-model tokens (load-bearing).** The HMCode2020+feedback recipe - is named by a token in each stack's API: CCL takes - ``extra_parameters['camb']['halofit_version']``, CAMB takes - ``NonLinearModel.set_params(halofit_version=…)``. :class:`TheoryConfig` - carries **two** tokens (``ccl_halofit_version``, ``camb_halofit_version``) - denoting one recipe, and each stack is fed its own — a single shared - string invites a silent stack disagreement the moment the two APIs name - the recipe differently (``mead2020`` vs ``mead2020_feedback`` differ by - several % at k ≳ 1/Mpc). Today both stacks accept the same string for the - feedback recipe, so the two defaults coincide; the cross-check test pins - the CCL token against the inference config independently. + One nonlinear recipe is named by two tokens (``ccl_halofit_version``, + ``camb_halofit_version``) because CCL and CAMB could name it differently; + each stack is fed its own so a rename cannot silently split the recipe. """ # Cosmological parameters (blind axes S8, Omega_m + the rest). @@ -99,19 +105,13 @@ class TheoryConfig: # Neutrino mass split: normal hierarchy (CosmoSIS `neutrino_hierarchy=normal`). mass_split: str = "normal" - # Boltzmann/transfer-function backend for the CCL path (#280). The default - # `boltzmann_camb` makes CCL call CAMB for the linear P(k), so blinding - # theory and the CosmoSIS+CAMB inference stack share one power-spectrum - # path. The halofit route follows the choice (see `ccl_cosmology`): only - # under `boltzmann_camb` can the nonlinear P(k) run through CAMB's HMCode - # (`matter_power_spectrum="camb"` + the tokens below); any other backend - # falls back to CCL's own halofit — consistent, but a different recipe, - # so a non-default backend is a deliberate cross-check tool, not a - # production setting. + # Boltzmann backend for the CCL path (#280). `boltzmann_camb` shares one + # power-spectrum path with the CosmoSIS+CAMB inference stack; any other + # backend falls back to CCL's own halofit (see `ccl_cosmology`), a + # deliberate cross-check tool rather than a production setting. transfer_function: str = "boltzmann_camb" - # One nonlinear recipe (CAMB HMCode2020 + baryonic feedback), two - # stack-specific tokens — see the class docstring. + # CAMB HMCode2020 + baryonic feedback — see the class docstring. ccl_halofit_version: str = "mead2020_feedback" camb_halofit_version: str = "mead2020_feedback" hmcode_logT_AGN: float = 7.5 # values_ia.ini logT_AGN central @@ -170,24 +170,8 @@ def ccl_params(self): @classmethod def from_overrides(cls, overrides): - """Build from a mapping of field overrides (fail loud on unknown keys). - - Numeric overrides are coerced to ``float`` per the field's declared - type, so a YAML/CLI ``w0: -1`` (int) yields the same value — and the - same :meth:`config_digest` — as the float default ``-1.0``. - """ - by_name = {f.name: f for f in dataclasses.fields(cls)} - unknown = set(overrides) - set(by_name) - if unknown: - raise ValueError( - f"unknown TheoryConfig fields {sorted(unknown)}; " - f"valid fields are {sorted(by_name)}" - ) - coerced = { - name: (float(v) if by_name[name].type in (float, "float") else v) - for name, v in overrides.items() - } - return cls(**coerced) + """Build from a mapping of field overrides (fail loud on unknown keys).""" + return cls(**coerce_fields(cls, overrides)) # --------------------------------------------------------------------------- # @@ -203,17 +187,11 @@ def ccl_cosmology(params, config): """A ``pyccl.Cosmology`` at ``params`` with ``config``'s nonlinear recipe. ``params`` is a plain CCL-native mapping (:meth:`TheoryConfig.ccl_params`, - possibly with keys overlaid by the hidden draw); ``config`` supplies only - the non-sampled recipe tokens (``transfer_function``, - ``ccl_halofit_version``, ``hmcode_logT_AGN``). The Boltzmann backend is - ``config.transfer_function`` (#280); the halofit route stays consistent - with that choice: under ``boltzmann_camb`` the nonlinear P(k) runs - through CAMB's HMCode2020 (``matter_power_spectrum="camb"`` + the CAMB - tokens), while any other backend has no CAMB run to hand tokens to, so it - takes CCL's own halofit (``matter_power_spectrum="halofit"``). Either - way, the same recipe sits on both sides of any theory difference. - Cosmology objects are cached per parameter point (CCL memoises its P(k) - on the object, so the cache saves repeated Boltzmann runs across blocks). + possibly with keys overlaid by the hidden draw); ``config`` supplies the + non-sampled recipe tokens. Under ``boltzmann_camb`` the nonlinear P(k) + runs through CAMB's HMCode2020; any other backend has no CAMB run to hand + tokens to and takes CCL's own halofit. Cached per parameter point: CCL + memoises P(k) on the object, so the cache saves repeated Boltzmann runs. """ import pyccl as ccl @@ -258,6 +236,14 @@ def xi_ell_grid(): ) +# Tracers are rebuilt for every pair of every block at both cosmologies of a +# blind, and each build runs CCL's lensing-kernel integral over the bin's n(z). +# Cached for the same reason as `_COSMO_CACHE`, and keyed on `id(cosmo)` +# because that cache pins every cosmology for the process lifetime, so an id +# can never be recycled onto a different object. +_TRACER_CACHE = {} + + def _tracer(cosmo, z, nz, config): """A ``WeakLensingTracer`` for one bin's n(z), NLA from ``config``. @@ -265,15 +251,28 @@ def _tracer(cosmo, z, nz, config): A nonzero ``ia_bias`` enters as the NLA amplitude ``A(z) = ia_bias · ((1+z)/(1+z_piv))^alphaz``. """ + z = np.asarray(z) + nz = np.asarray(nz) + key = ( + id(cosmo), + z.tobytes(), + nz.tobytes(), + config.ia_bias, + config.ia_z_piv, + config.ia_alphaz, + ) + if key not in _TRACER_CACHE: + _TRACER_CACHE[key] = _build_tracer(cosmo, z, nz, config) + return _TRACER_CACHE[key] + + +def _build_tracer(cosmo, z, nz, config): import pyccl as ccl - z = np.asarray(z) if config.ia_bias == 0.0: - return ccl.WeakLensingTracer(cosmo, dndz=(z, np.asarray(nz))) + return ccl.WeakLensingTracer(cosmo, dndz=(z, nz)) a_ia = config.ia_bias * ((1 + z) / (1 + config.ia_z_piv)) ** config.ia_alphaz - return ccl.WeakLensingTracer( - cosmo, dndz=(z, np.asarray(nz)), ia_bias=(z, a_ia), use_A_ia=True - ) + return ccl.WeakLensingTracer(cosmo, dndz=(z, nz), ia_bias=(z, a_ia), use_A_ia=True) def cl_ee(params, config, nz_i, nz_j, ell): @@ -318,7 +317,7 @@ def xi_ccl(params, config, nz_i, nz_j, theta_arcmin, ell=None): # --------------------------------------------------------------------------- # # Independent-CAMB path: A_s reconciliation + P(k) → Pk2D → CCL projection # --------------------------------------------------------------------------- # -def make_camb_params(config, As, *, nonlinear, zmax=3.0, n_z=48, kmax=20.0): +def make_camb_params(config, As, *, nonlinear, zmax=PK_ZMAX, n_z=PK_NZ, kmax=20.0): """A ``CAMBparams`` at ``config``'s background with amplitude ``As``. Every :class:`TheoryConfig` field CCL sees is fed to CAMB from the same @@ -380,14 +379,11 @@ def xi_camb(config, nz, theta_arcmin, *, n_ell=300, ell_max=60000, kmax=20.0, n_ """Independent-CAMB ξ± for one bin (Path B): CAMB P(k) → Pk2D → CCL. A direct pycamb run produces the HMCode2020 nonlinear ``P(k, z)`` at a - σ8-matched ``A_s`` (:func:`camb_As_for_sigma8`), extracted through - ``get_matter_power_interpolator(hubble_units=False, k_hunit=False)`` so - it comes out in CCL's native units (k in 1/Mpc, P in Mpc³) — **no** - ``·h`` / ``/h³`` conversion is applied (applying one would double-count - an h³ amplitude error). Both ``Pk2D`` axes are arranged ascending - (log-k ascending; scale factor ascending, i.e. CAMB's z-ascending grid - reversed). Projection is CCL's own Limber + FFTLog with a bare tracer - (IA off — this path exists for the cross-check). + σ8-matched ``A_s`` (:func:`camb_As_for_sigma8`), wrapped in a ``Pk2D`` and + projected by CCL's Limber + FFTLog with a bare tracer (IA off — this path + exists for the cross-check). ``hubble_units=False, k_hunit=False`` already + returns CCL's native units (k in 1/Mpc, P in Mpc³), so applying an + ``·h``/``/h³`` conversion here would double-count an h³ amplitude error. Returns ------- @@ -405,7 +401,7 @@ def xi_camb(config, nz, theta_arcmin, *, n_ell=300, ell_max=60000, kmax=20.0, n_ nonlinear=True, hubble_units=False, k_hunit=False ) k = np.geomspace(1e-4, kmax * config.h, n_k) # 1/Mpc - z = np.linspace(0.0, 3.0, 48) + z = np.linspace(0.0, PK_ZMAX, PK_NZ) # the grid make_camb_params computed pk = interp.P(z, k) # (n_z, n_k), Mpc^3 a = 1.0 / (1.0 + z) order = np.argsort(a) # Pk2D wants ascending scale factor diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 05ded922..1a830624 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -14,6 +14,7 @@ _get_pte_from_scale_cut, find_conservative_scale_cut_key, ) +from ..blinding_paths import init_paths from ..statistics import chi2_and_pte from ..version import __version__ from .catalog_characterization import CatalogCharacterizationMixin @@ -103,6 +104,11 @@ class CosmologyValidation( object writes. Custody state, not decoration: a mock campaign must be built with ``run_type='mock'`` for its parts to assemble at all (see ``blinding.assert_consistent_blind``). + blind_root : str, optional + Directory holding one ``blind_init`` state directory per catalogue + version. Given, the part writers stamp born-blinded and blind-irrelevant + parts under the version's blind (see :meth:`commitment_path`); ``None`` + (mock runs) leaves them plaintext. Attributes ---------- @@ -230,6 +236,7 @@ def __init__( cosmo_params=None, blind=None, run_type="data", + blind_root=None, ): self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method @@ -260,6 +267,7 @@ def __init__( self.path_onecovariance = path_onecovariance self.blind = blind self.run_type = run_type + self.blind_root = blind_root assert self.cell_method in ["map", "catalog"], ( "cell_method must be 'map' or 'catalog'" @@ -495,6 +503,17 @@ def results_objectwise(self): self._results_objectwise = self.init_results(objectwise=True) return self._results_objectwise + def commitment_path(self, version): + """The version's ``commitment.json``, or ``None`` when not blinding. + + Resolved per version rather than held as one path, because a single + ``CosmologyValidation`` can span several catalogue versions and each + has its own blind. + """ + if self.blind_root is None: + return None + return init_paths(os.path.join(self.blind_root, version))["commitment"] + def basename(self, version, treecorr_config=None, npatch=None): cfg = treecorr_config or self.treecorr_config patches = npatch or self.npatch diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index 260ffdfa..757d2d10 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -165,13 +165,7 @@ def _fiducial_cosebis_result(results, fiducial_scale_cut): return results[key], tuple(key) def cosebis_to_sacc_part( - self, - version, - out_path, - results, - fiducial_scale_cut=None, - en_override=None, - commitment_path=None, + self, version, out_path, results, fiducial_scale_cut=None, en_override=None ): """Write the COSEBIs SACC part at the fiducial scale cut. @@ -181,8 +175,8 @@ def cosebis_to_sacc_part( in mode space, so the non-fiducial cuts stay in the ``.npz`` sidecar. ``en_override`` replaces ``result["En"]`` with En derived from the - integration ξ± part; Bn and the covariance stay from ``result``. - ``commitment_path`` stamps the part concealed under that version's blind. + integration ξ± part; Bn and the covariance stay from ``result``. The part + is born blinded, so it is stamped under the version's blind. """ result, scale_cut = self._fiducial_cosebis_result(results, fiducial_scale_cut) if en_override is not None: @@ -193,7 +187,9 @@ def cosebis_to_sacc_part( result, scale_cut, ) - sacc_io.save(s, out_path, type=self.run_type, commitment=commitment_path) + sacc_io.save( + s, out_path, type=self.run_type, commitment=self.commitment_path(version) + ) def plot_cosebis( self, diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index b9897965..de5feb19 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -25,7 +25,7 @@ class PSFSystematicsMixin: - def calculate_rho_tau_stats(self, commitment_path=None): + def calculate_rho_tau_stats(self): """Measure ρ/τ statistics per version and write each version's SACC part.""" out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" if not os.path.exists(out_dir): @@ -45,12 +45,7 @@ def calculate_rho_tau_stats(self, commitment_path=None): npatch=self.npatch, ) self.rho_tau_to_sacc_part( - ver, - out_dir, - base, - rho_stat_handler, - tau_stat_handler, - commitment_path=commitment_path, + ver, out_dir, base, rho_stat_handler, tau_stat_handler ) self.print_done("Rho stats finished") @@ -58,20 +53,13 @@ def calculate_rho_tau_stats(self, commitment_path=None): self._tau_stat_handler = tau_stat_handler def rho_tau_to_sacc_part( - self, - version, - out_dir, - base, - rho_stat_handler, - tau_stat_handler, - commitment_path=None, + self, version, out_dir, base, rho_stat_handler, tau_stat_handler ): """Write the ρ/τ SACC part for one version. - ρ/τ is a PSF diagnostic carrying no cosmological vector, so it is never - blinded; ``commitment_path`` stamps the part concealed under the - version's blind, values untouched (see :func:`sacc_io.save`), so a data - run's assembly admits it. A mock run passes ``None``. + ρ/τ carries no cosmological vector and is never blinded; on a data run + it is stamped concealed pass-through (values untouched) so the + assembly's load gate admits it. ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage from the handler tables. The ``CovTauTh`` theory covariance ``cov_tau_{base}_th.npy`` is passed as @@ -94,7 +82,9 @@ def rho_tau_to_sacc_part( tau_cov_th=tau_cov_th, ) out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") - sacc_io.save(s, out_path, type=self.run_type, commitment=commitment_path) + sacc_io.save( + s, out_path, type=self.run_type, commitment=self.commitment_path(version) + ) @property def rho_stat_handler(self): diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 998cc372..a97c1291 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -134,9 +134,7 @@ def calculate_pure_eb( return results - def pure_eb_to_sacc_part( - self, version, out_path, results, eb_override=None, commitment_path=None - ): + def pure_eb_to_sacc_part(self, version, out_path, results, eb_override=None): """Write the pure-E/B SACC part (six ``PURE_KEYS`` blocks + covariance). ``results`` is the dict ``calculate_pure_eb`` returned: the six pure-mode @@ -146,7 +144,7 @@ def pure_eb_to_sacc_part( ``eb_override`` replaces the six arrays with ones derived from the reporting + integration ξ± parts; the covariance stays from ``results``. - ``commitment_path`` stamps the part concealed under that version's blind. + The part is born blinded, so it is stamped under the version's blind. """ theta = results["gg"].meanr source = eb_override if eb_override is not None else results @@ -158,7 +156,9 @@ def pure_eb_to_sacc_part( eb, covariance=results["cov"], ) - sacc_io.save(s, out_path, type=self.run_type, commitment=commitment_path) + sacc_io.save( + s, out_path, type=self.run_type, commitment=self.commitment_path(version) + ) def plot_pure_eb( self, diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index aa2f5cb3..d608b658 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -10,7 +10,7 @@ canonical order. The integration-grid ξ± part is an intermediate consumed by COSEBIs and -pure-E/B; it is blinded at birth on data runs but does not join ``{version}.sacc``. +pure-E/B; it is blinded at birth but does not join ``{version}.sacc``. Everything here is single-bin today (``bins=(0, 0)``); the interface is tomography-native so a future round supplies real bin pairs unchanged. diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 4cc76595..24fee0a4 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -1552,38 +1552,18 @@ def covariance_blocks(cov_list, selectors, *, gaussian=True): # --------------------------------------------------------------------------- # -# Terminal assembly (PR-6 blinding) — gather() and its blind-custody call site. +# Terminal assembly — gather() and its blind-custody call site. # --------------------------------------------------------------------------- # def gather(parts, metadata=None, assemble=None): """Assemble standalone part SACCs into the one-file ``{version}.sacc``. - Each part is an intermediate product as it came off its producing rule - (reporting ξ±, integration ξ±, pseudo-Cℓ, ρ/τ, …). Gather is **the** terminal - seam: every path that combines parts into the one-file product goes - through here, because this is where the one thing an assembler cannot know - about is enforced — **blind custody.** - - **Blind custody.** + The terminal seam: every path that combines parts into the one-file + product goes through here, because this is where the one thing an + assembler cannot know about is enforced — blind custody. :func:`sp_validation.blinding.assert_consistent_blind` runs before the - assembly — it fails closed unless every blindable part carries the - identical ``blind_commitment``/``blind_config_digest``/``blind_draw_scheme`` - (or, when nothing is blinded, every blindable part is declared - ``type='mock'``). Its returned shared stamp is written onto the assembled - file so the one-file product carries the blind it was built from; the - blinded parts already carry those keys, so the assembly preserves them and - this stamp is a consistent (idempotent) re-affirmation. - - **The assembly itself is the caller's.** Two exist and both are real: - :func:`merge` (the default) does the first-wins tracer union, in-order - point concatenation with all tags, and a covariance built from whatever the - parts carry — the right thing when parts are already covariance-bearing. - :func:`sp_validation.cosmo_val.sacc_writers.assemble_analysis_sacc` rebuilds - from the parts' own tracers/metadata and *requires* one covariance block per - part — - the right thing for the production terminal, where the ξ± and pseudo-Cℓ - parts are born cov-less and have their blocks injected first. Passing the - assembler in, rather than duplicating the custody wrapper around each one, - is what keeps the guard un-bypassable. + assembly and its returned shared stamp is written onto the result. The + assembler is passed *in* rather than wrapping this guard, which is what + keeps the guard un-bypassable. Parameters ---------- diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index f4aa7d58..cdec4e8c 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -1,22 +1,14 @@ """Tests for :mod:`sp_validation.blinding` — per-part Smokescreen blinding. Acceptance criteria AC1–AC9 of the blinding PRD, plus fast unit coverage of -the config/custody surface. Fast tests (no CCL import — the envelope -calibration, digest, commitment, fork-draw determinism, blind-init custody, -the merge alignment against a monkeypatched concealing factor, and the -assembly hash assertion) run in the default suite; the theory tests (fork + -CCL) are marked ``slow``; the derived-statistics tests additionally -``importorskip`` ``cosmo_numba``. +the config/custody surface. The theory tests (fork + CCL) are marked ``slow``; +the derived-statistics tests additionally ``importorskip`` ``cosmo_numba``. All fixtures are synthetic and deterministic. Each blindable intermediate is -its own standalone part SACC (reporting ξ±, integration ξ±, pseudo-Cℓ), as in the -per-part-at-birth architecture; derived statistics (COSEBIs, pure-E/B) are -never stored in parts — they are computed downstream from the (blinded) integration -ξ± through the pipeline seams ``b_modes.cosebis_from_xi`` / -``b_modes.pure_eb_from_xi``, exactly as the pipeline does. The blinding path -hands the fork no data vector at all (``smokescreen.concealing_factor`` is a -pure theory difference), so fixture ξ± values are smooth synthetic templates — -no theory fill is needed to blind. +its own standalone part SACC, as in the per-part-at-birth architecture; +derived statistics are computed downstream from the blinded integration ξ± +through the pipeline's own seams. The fork is handed no data vector at all, so +fixture ξ± values need only be smooth synthetic templates. """ import json diff --git a/src/sp_validation/tests/test_blinding_wiring.py b/src/sp_validation/tests/test_blinding_wiring.py index 04e18074..a5e8ccf6 100644 --- a/src/sp_validation/tests/test_blinding_wiring.py +++ b/src/sp_validation/tests/test_blinding_wiring.py @@ -1,18 +1,10 @@ -"""Tests for the Snakemake blind-at-birth wiring (issues #247/#252, PR #253). +"""Tests for the Snakemake blind-at-birth wiring, independent of a live cluster. -Two seams are covered here, both independent of a live cluster: - -1. **The path helpers in ``workflow/common.py``** must stay in lockstep with - ``sp_validation.blinding`` — common mirrors ``init_paths`` / ``part_paths`` by - hand (to keep the DAG build from importing the heavy blinding module), so a - drift between them would silently mis-wire ``blind_part``. These tests are the - guard. -2. **The data-run fail-closed assembly**: ``assemble_sacc`` must refuse an - unblinded ``type='data'`` part and succeed once every part is concealed under - one commitment — the terminal custody gate of #252. - -A candide-only test additionally asserts the blinding subgraph resolves in the -cosmo_val DAG dry-run. +Covers ``workflow/common.py``'s part-path and run-type helpers, and the +data-run fail-closed assembly: ``assemble_sacc`` must refuse an unblinded +``type='data'`` part and succeed once every part is concealed under one +commitment. A candide-only test additionally asserts the blinding subgraph +resolves in the cosmo_val DAG dry-run. """ import importlib.util @@ -51,7 +43,7 @@ def _load_module(rel_path, name): # --------------------------------------------------------------------------- # -# 1. common.py path helpers mirror sp_validation.blinding (drift guard) +# 1. common.py part-path and run-type helpers # --------------------------------------------------------------------------- # _STEMS = [ "SP_v1.4.6.3_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=100", @@ -61,19 +53,6 @@ def _load_module(rel_path, name): ] -@pytest.mark.parametrize("stem", _STEMS) -def test_blinded_path_mirrors_blinding_part_paths(stem): - part = f"/out/{stem}.sacc" - assert common.blinded_path(part) == blinding.part_paths(part)["blinded"] - - -def test_blind_state_paths_mirror_blinding_init_paths(): - version = "SP_v1.4.6.3_leak_corr" - common_paths = common.blind_state_paths(version) - ref = blinding.init_paths(common.blind_state_dir(version)) - assert common_paths == ref - - @pytest.mark.parametrize( "stem,expected", [ @@ -260,10 +239,12 @@ def test_rho_tau_part_is_stamped_concealed_from_the_commitment(tmp_path): data run's ``assemble_sacc`` dies on the ρ/τ part before custody is ever checked. """ + from sp_validation.cosmo_val.core import CosmologyValidation from sp_validation.cosmo_val.psf_systematics import PSFSystematicsMixin - blind_dir = tmp_path / "blind" - blind_dir.mkdir() + root = tmp_path / "blind" + blind_dir = root / "vSYNTH" + blind_dir.mkdir(parents=True) commitment = blinding.blind_init(str(blind_dir), log=lambda *_: None)["commitment"] theta = np.geomspace(1.0, 100.0, 6) rng = np.random.default_rng(11) @@ -282,6 +263,9 @@ class _Writer(PSFSystematicsMixin): """The writer's collaborators, stubbed — the method under test is real.""" run_type = "data" + blind_root = str(root) + # The real per-version resolution is part of what this test covers. + commitment_path = CosmologyValidation.commitment_path def sacc_nz(self, version): return {0: _nz()} @@ -300,7 +284,6 @@ def print_magenta(self, *args, **kwargs): "vSYNTH", types.SimpleNamespace(rho_stats=rho), types.SimpleNamespace(tau_stats=tau), - commitment_path=commitment, ) written = sio.load(str(out_dir / "rho_tau_vSYNTH.sacc")) diff --git a/workflow/Snakefile b/workflow/Snakefile index 0a1f3258..0cbfd3b0 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -37,9 +37,8 @@ wildcard_constraints: # Compute rules (infrastructure — raw outputs, no evidence.json) include: "rules/twopoint.smk" -# Smokescreen blind-at-birth custody (blind_init / blind_part). Generic over the -# blindable parts twopoint.smk produces; dormant unless a data run requests a -# blinded part. Included before covariance/cosmo_val so their consumers resolve. +# Smokescreen blind-at-birth custody, generic over the blindable parts +# twopoint.smk produces. Included before its consumers in covariance/cosmo_val. include: "rules/blinding.smk" include: "rules/covariance.smk" include: "rules/inference.smk" diff --git a/workflow/common.py b/workflow/common.py index f7faf8b0..0878289f 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -7,11 +7,19 @@ from snakemake.io import temp +# Dependency-free by design: the DAG build gets the blinding file-name +# conventions without importing numpy + smokescreen. +from sp_validation.blinding_paths import init_paths, part_paths + # Absolute path to the generic workflow's scripts, for rules that shell out to a # script directly rather than through Snakemake's `script:` directive. Anchored -# on this module's own location, not workflow.basedir -- under `module` +# on this module's own location, not workflow.basedir — under `module` # composition basedir reflects the composing paper, not the running checkout. WORKFLOW_SCRIPTS = os.path.join(os.path.dirname(os.path.realpath(__file__)), "scripts") +# The repo's top-level scripts/ (user-facing CLIs), anchored the same way. +REPO_SCRIPTS = os.path.join( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "scripts" +) # Output roots are env-overridable so a reproduction run can write into a # fresh tree without clobbering (or silently reusing) prior products. @@ -216,21 +224,13 @@ def get_shear_catalog(wildcards): # --------------------------------------------------------------------------- -# Smokescreen blind-at-birth custody (issues #247/#252, PR #253) +# Smokescreen blind-at-birth custody # --------------------------------------------------------------------------- # Distinct from the glass-mock A/B/C `blind` wildcard above: this is Smokescreen -# concealment (the concealed=True SACC stamp). Blinding is per part, at birth. -# The three blindable parts (reporting ξ±, integration ξ±, pseudo-Cℓ) are each -# concealed the moment they are computed, so only blinded parts persist on disk. -# A `data` run binds every ξ-derived consumer (the terminal assemble, and the -# born-blinded COSEBIs / pure-E/B) to the *_blinded parts, which pulls the -# blind_part → blind_init subgraph into the DAG. A `mock` run bypasses blinding -# entirely and binds to the plaintext parts, so the subgraph never appears. -# RUN_TYPE is the single switch that flips which files exist. -# -# The path helpers below MIRROR sp_validation.blinding.init_paths / part_paths -# by hand rather than importing blinding (which pulls in numpy + smokescreen) at -# DAG-build time. test_blinding_wiring asserts the two stay in lockstep. +# concealment (see sp_validation.blinding). RUN_TYPE is the single switch: a +# `data` run binds every ξ-derived consumer to the *_blinded parts, pulling the +# blind_part → blind_init subgraph into the DAG; a `mock` run binds the +# plaintext parts and the subgraph never appears. def run_type(): @@ -250,22 +250,23 @@ def is_data_run(): return run_type() == "data" +def blind_root(): + """Root holding one blind-init directory per version, or None on mock runs. + + What `CosmologyValidation(blind_root=...)` takes, so its part writers can + resolve each version's commitment.json themselves. + """ + return str(COSMO_VAL / "blind") if is_data_run() else None + + def blind_state_dir(version): """Per-version blind-init custody directory (commitment + encrypted seed).""" return str(COSMO_VAL / "blind" / version) def blind_state_paths(version): - """The fixed custody-state files blind_init writes for a version. - - Mirrors sp_validation.blinding.init_paths(blind_state_dir(version)). - """ - d = blind_state_dir(version) - return { - "commitment": os.path.join(d, "commitment.json"), - "bundle": os.path.join(d, "blind_seed.encrpt"), - "key": os.path.join(d, "blind_seed.key"), - } + """The fixed custody-state files blind_init writes for a version.""" + return init_paths(blind_state_dir(version)) def commitment_input(version): @@ -273,8 +274,7 @@ def commitment_input(version): A part writer stamps its output concealed from that file (sacc_io.save's `commitment=`), which is what lets a born-blinded or blind-irrelevant part - clear the fail-closed load gate the terminal assembly opens every part - through. A mock run binds nothing and the part stays plaintext. + clear the fail-closed load gate at assembly. """ if not is_data_run(): return {} @@ -282,20 +282,14 @@ def commitment_input(version): def blinded_path(part_path): - """The *_blinded sibling blind_part writes beside a plaintext part. - - Mirrors sp_validation.blinding.part_paths(part_path)["blinded"]. - """ - stem, ext = os.path.splitext(str(part_path)) - return f"{stem}_blinded{ext or '.fits'}" + """The *_blinded sibling blind_part writes beside a plaintext part.""" + return part_paths(part_path)["blinded"] def version_of(stem): - """Extract the catalogue version embedded in a blindable part's stem. + """Catalogue version embedded in a blindable part's stem. - Every blindable stem carries the version (as {version}_xi_… or - pseudo_cl_{version}_…); blind_part needs it to locate the version's blind - state. Matches the shared `version` wildcard pattern. + blind_part needs it to locate the version's blind state. """ m = re.search(WILDCARD_CONSTRAINTS["version"], stem) if m is None: @@ -307,18 +301,16 @@ def blindable_part(part_path): """On-disk path a run persists for one blindable part. Data run -> the blinded sibling (binding it pulls blind_part + blind_init - into the DAG); mock run -> the plaintext part (blinding bypassed). + into the DAG); mock run -> the plaintext part. """ return blinded_path(part_path) if is_data_run() else str(part_path) def maybe_temp(part_path): - """Wrap a producer's blindable plaintext part temp() on data runs. + """temp() a producer's blindable plaintext part on data runs. - On a data run the plaintext part's only consumer is blind_part, which - escrows the true vector before Snakemake removes the temp file — so no - plaintext blindable part persists. On a mock run the part is the real - product downstream binds to, so it is left persistent. + Its only consumer there is blind_part, which escrows the true vector before + Snakemake removes the file, so no plaintext blindable part persists. """ return temp(str(part_path)) if is_data_run() else str(part_path) @@ -381,9 +373,10 @@ def cv_init_params(config, version_list=None): nrandom_cell=cv["nrandom_cell"], cell_method=cv["cell_method"], nside_mask=cv["nside_mask"], - # Stamped as the SACC `type` of every part the cv writes; RUN_TYPE reads - # from this same key (see configure()). - run_type=cv.get("type", "data"), + # Custody state the cv's part writers need: the SACC `type` they stamp, + # and the blind whose commitment born-blinded parts are stamped under. + run_type=run_type(), + blind_root=blind_root(), ) if cv.get("path_onecovariance"): params["path_onecovariance"] = cv["path_onecovariance"] diff --git a/workflow/rules/blinding.smk b/workflow/rules/blinding.smk index 6930aadf..d43a7ae8 100644 --- a/workflow/rules/blinding.smk +++ b/workflow/rules/blinding.smk @@ -1,39 +1,29 @@ -# Smokescreen blind-at-birth custody rules (issues #247/#252, PR #253). +# Smokescreen blind-at-birth custody rules (sp_validation.blinding). # -# Two rules realise the three-verb custody surface of sp_validation.blinding on -# the DAG. They only enter the graph on a `data` run, and only when a consumer -# binds to a *_blinded part through common.blindable_part — a `mock` run never -# requests a blinded file, so blind_part and blind_init stay dormant. -# -# blind_init (once per catalogue version) draws the seed, publishes -# commitment.json + the encrypted seed bundle. -# blind_part (once per blindable part, at birth) conceals the part, escrows -# the true vector beside the blinded output, and lets Snakemake -# remove the plaintext (temp()) once it is the sole consumer. -# -# The terminal assemble_sacc rule (cosmo_val.smk) asserts the shared commitment -# across parts — the assembly-time custody check of #252. +# Dormant unless a consumer binds a *_blinded part through common.blindable_part, +# which only a `data` run does. The terminal assemble_sacc rule (cosmo_val.smk) +# asserts the shared blind across parts. + +import re -# The blindable stems: the binning-named ξ± parts (rule xi — one rule, both the -# reporting and the integration grid, told apart only by their binning tag) and -# the analysis pseudo-Cℓ (pseudo_cl). None contains "_blinded", so the generic -# blind_part rule can never blind its own output twice. The version pattern is -# the shared one from common.WILDCARD_CONSTRAINTS. -_V = WILDCARD_CONSTRAINTS["version"] -BLINDABLE_STEM = ( - rf"(?:{_V}_xi_minsep=[0-9.]+_maxsep=[0-9.]+_nbins=\d+_npatch=\d+" - rf"|pseudo_cl_{_V}_blind=[ABC]_[a-z]+_nbins=\d+)" +# The blindable stems, derived from the same name-builders the producing rules +# use so part names have one authority: the binning-named ξ± parts (rule xi, one +# per named grid) and the analysis pseudo-Cℓ. None contains "_blinded", so the +# generic blind_part rule can never blind its own output twice. +_VERSION_SLOT = "0VERSION0" # regex-inert placeholder, substituted after escaping +BLINDABLE_STEM = "(?:{})".format( + "|".join( + re.escape(stem) + for stem in ( + [f"{_VERSION_SLOT}_xi_{xi_binning(grid)}" for grid in XI_GRIDS] + + [f"pseudo_cl_{_VERSION_SLOT}_{pseudo_cl_tag(config)}"] + ) + ).replace(_VERSION_SLOT, WILDCARD_CONSTRAINTS["version"]) ) rule blind_init: - """Fix the blind for one catalogue version (blind-init). - - Draws an OS-entropy seed, writes the repo-committable commitment.json - (seed commitment + config digest + the installed fork's draw scheme) and the - Fernet-encrypted seed bundle. Runs once per version and refuses to overwrite - existing state — a blind is a one-shot custody event. - """ + """Draw the seed and publish the commitment + encrypted bundle for a version.""" output: commitment=str(COSMO_VAL / "blind" / "{version}" / "commitment.json"), bundle=str(COSMO_VAL / "blind" / "{version}" / "blind_seed.encrpt"), @@ -42,19 +32,13 @@ rule blind_init: blind_dir=lambda w: blind_state_dir(w.version), resources: runtime=5, - script: - "../scripts/blind_init.py" + shell: + "python {REPO_SCRIPTS}/blind_data_vector.py" + " blind-init {params.blind_dir}" rule blind_part: - """Blind one intermediate part SACC at birth (blind-part). - - Conceals the plaintext part through its matching theory backend, escrows the - true vector into a per-part encrypted bundle beside the blinded output, and - leaves the plaintext for Snakemake to remove (it is a temp() output of the - producing rule, and this is its only consumer on a data run). Generic over - the three blindable stems. - """ + """Conceal one part, escrowing its true vector beside the blinded output.""" input: part=str(COSMO_VAL / "{stem}.sacc"), commitment=lambda w: blind_state_paths(version_of(w.stem))["commitment"], @@ -70,5 +54,8 @@ rule blind_part: blind_dir=lambda w: blind_state_dir(version_of(w.stem)), resources: runtime=10, - script: - "../scripts/blind_part.py" + # --keep-input: the plaintext part is the producing rule's temp() output, so + # Snakemake removes it once this, its only consumer, finishes. + shell: + "python {REPO_SCRIPTS}/blind_data_vector.py" + " blind-part {input.part} --blind-dir {params.blind_dir} --keep-input" diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 177bcb91..7ad5e19e 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -340,11 +340,8 @@ rule cv_pseudo_cl: # Pure E/B modes and COSEBIs (per version), then the B-mode summary # --------------------------------------------------------------------------- -# On a data run the COSEBIs / pure-E/B parts re-derive their E-mode vector from -# the *blinded* integration ξ± (COSEBIs) or blinded reporting + integration ξ± -# (pure-E/B). blindable_part returns the plaintext part on a mock run and the -# blinded part on a data run, so the ξ± inputs bind unconditionally for every -# version; see common.commitment_input for the commitment. +# On a data run these re-derive their E-mode vector from the *blinded* ξ± parts, +# so they are born blinded; the commitment binds only there. def cv_cosebis_inputs(w): return { "xi": cv_xi_txt(w.version), @@ -469,15 +466,9 @@ def cv_assemble_inputs(version): pseudo_cl (+ its cov) is included only when the config toggles the harmonic-space BB into the analysis. """ - # blindable_part binds the raw-signal parts (reporting ξ±, analysis pseudo-Cℓ) - # to their blinded siblings on a data run and to the plaintext on a mock run. - # COSEBIs and pure-E/B are born blinded (their writers derive them from the - # blinded integration ξ± and stamp concealed=True), so they bind by their own - # name in both cases; ρ/τ is a diagnostic carrying no cosmological vector but - # is stamped concealed pass-through so the fail-closed load gate admits it on - # a data run. The integration-grid ξ± itself is blinded at birth (the same - # `xi` rule on the integration binning) but is NOT gathered into the terminal file — it persists as its - # own per-part intermediate (see #247 ruling), consumed by COSEBIs/pure-E/B. + # blindable_part binds the raw-signal parts to their blinded siblings on a + # data run. COSEBIs, pure-E/B and ρ/τ are stamped concealed by their own + # writers, so they bind by name either way. parts = dict( xi_reporting=blindable_part(cv_xi_sacc(version, "reporting")), cosebis=cv_cosebis_sacc(version), diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 10e2b343..21c3a8c3 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -72,8 +72,7 @@ rule xi: catalog=get_shear_catalog, output: txt=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), - # Blindable part: temp() on a data run so only its blinded sibling - # persists (blind_part escrows the true vector first). See common.maybe_temp. + # Blindable part: temp() on a data run so only its blinded sibling persists. sacc=maybe_temp(str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc")), threads: 24 params: @@ -84,9 +83,7 @@ rule xi: npatch="{npatch}", cat_config=CAT_CONFIG, grid=lambda w: xi_grid_of(w), - # Stamped as the part's SACC `type` — custody state at assembly - # (see blinding.assert_consistent_blind). - type=run_type(), + type=run_type(), # the part's SACC `type` — custody state at assembly resources: # The fine integration grid needs more memory and wall time than the # ~20-bin reporting one; scale on nbins rather than splitting the rule. @@ -116,8 +113,8 @@ rule run_cosmo_val: rule rho_tau_stats: - # ρ/τ has no blindable input; it binds only the commitment, to stamp its - # part concealed pass-through (see common.commitment_input). + # ρ/τ has no blindable input; it binds the commitment only to stamp its part + # concealed pass-through. input: unpack(lambda w: commitment_input(w.version)), output: @@ -134,6 +131,7 @@ rule rho_tau_stats: nbins="{nbins}", npatch="{npatch}", type=run_type(), + blind_root=blind_root(), resources: mem_mb=30000, disk_mb=20000, @@ -151,13 +149,9 @@ wildcard_constraints: rule pseudo_cl: """Generate pseudo-Cl data vector (born as SACC) with configurable binning.""" output: - # This generic rule produces every pseudo-Cℓ variant — the analysis part - # (blind=A, powspace, nbins=32) folded into {version}.sacc, plus the fine - # (COSEBIS) and glass-mock variants. Only the analysis part is a terminal - # blindable, and a data run blinds it via a requested _blinded sibling - # (blind_part reads this plaintext); the fine/mock variants are B-mode / - # validation intermediates left untouched here. The output is therefore - # not temp()'d — see the PR note on residual unblinded pseudo-Cℓ. + # One rule for every pseudo-Cℓ variant, only one of which (the analysis + # part) is blindable — so this output cannot be temp()'d and a data run + # blinds it through a requested _blinded sibling instead. pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.sacc"), wildcard_constraints: blind="[ABC]", diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 5b60787f..1b64584e 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -123,9 +123,8 @@ def assemble_sacc( parts.append(_attach_cov(part, name, xi_cov, pseudo_cl_cov, placeholder_var)) if not parts: raise ValueError(f"no parts found for {version}: {part_paths}") - # Through sacc_io.gather, the one terminal seam: assembly fails closed unless - # every blindable part shares one blind commitment + config digest + draw - # scheme, and the shared blind is stamped onto the assembled file. + # Through sacc_io.gather, the one terminal seam: it fails closed unless every + # blindable part shares one blind, and stamps that blind on the result. s = sacc_io.gather(parts, assemble=assemble_analysis_sacc) sacc_io.save(s, out_path, type=s.metadata["type"]) print(f"Assembled {len(parts)} parts -> {out_path}") diff --git a/workflow/scripts/blind_init.py b/workflow/scripts/blind_init.py deleted file mode 100644 index f9e4b682..00000000 --- a/workflow/scripts/blind_init.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Rule blind_init: fix the blind for one catalogue version. - -Thin wrapper over :func:`sp_validation.blinding.blind_init`. Draws the seed, -writes commitment.json + the encrypted seed bundle into the version's blind -directory. The plaintext seed is never written (the encryptor deletes it). -""" - -from snakemake.script import snakemake - -from sp_validation import blinding - -blinding.blind_init(snakemake.params["blind_dir"]) diff --git a/workflow/scripts/blind_part.py b/workflow/scripts/blind_part.py deleted file mode 100644 index b93313fb..00000000 --- a/workflow/scripts/blind_part.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Rule blind_part: blind one intermediate part SACC at birth. - -Thin wrapper over :func:`sp_validation.blinding.blind_part`. Conceals the part, -escrows the true vector beside the blinded output, and leaves the plaintext in -place: it is a temp() output of the producing rule, so Snakemake removes it once -this (its only consumer on a data run) finishes. keep_input=True hands that -lifecycle to Snakemake rather than deleting inside the blind step, which keeps -the blinded output and its temp input in one consistent DAG accounting. -""" - -from snakemake.script import snakemake - -from sp_validation import blinding - -blinding.blind_part( - snakemake.input["part"], - snakemake.params["blind_dir"], - keep_input=True, -) diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index fe08e086..726e16ba 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -38,9 +38,6 @@ theta, xip, xim = sacc_io.get_xi(integ, (0, 0), grid="integration") en_part, _ = cosebis_from_xi(theta, xip, xim, p["nmodes"], scale_cut=fiducial_scale_cut) -# Blinded at birth on a data run: commitment.json is bound only there. -commitment_path = snakemake.input.get("commitment") - # Born-as-SACC COSEBIs part at the fiducial scale cut. cv.cosebis_to_sacc_part( version, @@ -48,7 +45,6 @@ cv._cosebis_results[version], fiducial_scale_cut=fiducial_scale_cut, en_override=en_part, - commitment_path=commitment_path, ) # Sync the npz's En with the part-derived values; Bn / cov / PTE untouched. diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index bc6df640..bff39ecb 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -40,17 +40,8 @@ ti, xpi, xmi = sacc_io.get_xi(integ, (0, 0), grid="integration") modes = pure_eb_from_xi(tr, xpr, xmr, ti, xpi, xmi, tmin, tmax) -# Blinded at birth on a data run: commitment.json is bound only there. -commitment_path = snakemake.input.get("commitment") - # Born-as-SACC pure-E/B part; the six blocks come from the consumed parts. -cv.pure_eb_to_sacc_part( - version, - snakemake.output["sacc"], - results, - eb_override=modes, - commitment_path=commitment_path, -) +cv.pure_eb_to_sacc_part(version, snakemake.output["sacc"], results, eb_override=modes) # Sync the npz's pure modes with the part-derived values; theta / cov / PTE untouched. npz_path = snakemake.output["npz"] diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 20a73a02..41446f91 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -57,8 +57,7 @@ def run_2pcf( expects. ``sacc_out`` is the exact destination for the ξ± SACC part (the Snakemake-declared output); it defaults to the binning-named part under the resolved output directory for the CLI path. ``run_type`` - (``"data"`` or ``"mock"``) is stamped as the part's SACC ``type`` — custody - state at assembly (see ``blinding.assert_consistent_blind``). + (``"data"`` or ``"mock"``) is stamped as the part's SACC ``type``. Returns ------- @@ -124,8 +123,7 @@ def _from_snakemake(smk): # Write the SACC part exactly where the rule declares it (the .txt # byproduct still lands under the resolved output dir via _output_path). sacc_out=smk.output["sacc"], - # Custody state stamped as the part's SACC `type` (blinding). - run_type=p.get("type", "data"), + run_type=p["type"], ) diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index 0e11d01d..7e899561 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -44,16 +44,11 @@ theta_max=float(params["max_sep"]), nbins=int(params["nbins"]), npatch=int(params["npatch"]), - run_type=params.get("type", "data"), + run_type=params["type"], + blind_root=params["blind_root"], ) -# On a data run the rule binds the version's commitment.json, which stamps the -# emitted ρ/τ part concealed pass-through — values untouched (ρ/τ carries no -# cosmological vector) but clearing the fail-closed load gate at assembly. A mock -# run binds no commitment and the part stays plaintext, stamped type='mock'. -commitment_path = snakemake.input.get("commitment") # type: ignore - -cv.calculate_rho_tau_stats(commitment_path=commitment_path) +cv.calculate_rho_tau_stats() # Confirm CosmologyValidation produced the requested outputs: the rho/tau FITS # and the born-as-SACC rho_tau part the assemble_sacc rule consumes. From 8541fd7aa19850d166f06e5cbb82f0055b5de20b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 02:05:12 +0200 Subject: [PATCH 042/160] cv_cosebis: derive Bn from the integration part; unpatched integration path in claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both COSEBIs values now come from the (blindable) integration ξ± SACC part through the cosebis_from_xi seam; only the jackknife covariance still comes from the raw patched measurement, where the patches live. papers/bmodes' integration ξ± path said npatch=FIDUCIAL['npatch']; the integration grid (XI_GRIDS, twopoint.smk) is unpatched, so pin it to 1. --- papers/bmodes/rules/claims.smk | 8 +++- src/sp_validation/cosmo_val/cosebis.py | 22 ++++++++--- src/sp_validation/tests/test_sacc_writers.py | 39 ++++++++++++++++++++ workflow/rules/cosmo_val.smk | 5 ++- workflow/scripts/cv_cosebis.py | 11 ++++-- 5 files changed, 73 insertions(+), 12 deletions(-) diff --git a/papers/bmodes/rules/claims.smk b/papers/bmodes/rules/claims.smk index 43962627..a41833ed 100644 --- a/papers/bmodes/rules/claims.smk +++ b/papers/bmodes/rules/claims.smk @@ -86,10 +86,14 @@ def _xi_reporting_path(version): def _xi_integration_path(version): - """Path to fine-binned 2PCF integration file.""" + """Path to fine-binned 2PCF integration file. + + The integration grid is unpatched (npatch=1, matching XI_GRIDS): it supplies + values only, never a jackknife covariance. + """ return ( f"{COSMO_VAL_OUTPUT}/{version}_xi_minsep={FIDUCIAL['min_sep_int']}" - f"_maxsep={FIDUCIAL['max_sep_int']}_nbins={FIDUCIAL['nbins_int']}_npatch={FIDUCIAL['npatch']}.txt" + f"_maxsep={FIDUCIAL['max_sep_int']}_nbins={FIDUCIAL['nbins_int']}_npatch=1.txt" ) diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index 2f472fce..d9d61582 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -165,7 +165,13 @@ def _fiducial_cosebis_result(results, fiducial_scale_cut): return results[key], tuple(key) def cosebis_to_sacc_part( - self, version, out_path, results, fiducial_scale_cut=None, en_override=None + self, + version, + out_path, + results, + fiducial_scale_cut=None, + en_override=None, + bn_override=None, ): """Write the COSEBIs SACC part at the fiducial scale cut. @@ -174,12 +180,18 @@ def cosebis_to_sacc_part( part: the covariance must cover every stored point and the cuts overlap in mode space, so the non-fiducial cuts stay in the ``.npz`` sidecar. - ``en_override`` replaces ``result["En"]`` with En derived from the - integration ξ± part; Bn and the covariance stay from ``result``. + ``en_override``/``bn_override`` replace ``result["En"]``/``result["Bn"]`` + with values derived from the integration ξ± part; the covariance stays + from ``result`` (patches exist only in the raw patched measurement). """ result, scale_cut = self._fiducial_cosebis_result(results, fiducial_scale_cut) - if en_override is not None: - result = {**result, "En": np.asarray(en_override)} + overrides = { + key: np.asarray(value) + for key, value in (("En", en_override), ("Bn", bn_override)) + if value is not None + } + if overrides: + result = {**result, **overrides} s = cosebis_to_sacc( self.sacc_nz(version), self.sacc_metadata(version), diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index 52f612eb..6dc3ef1b 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -316,3 +316,42 @@ def test_assemble_from_reloaded_parts(tmp_path): s = sw.assemble_analysis_sacc(reloaded) assert type(s.covariance).__name__ == "BlockDiagonalCovariance" assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) + + +def test_cosebis_part_overrides_en_and_bn(tmp_path): + """En/Bn overrides reach the part; the covariance stays from ``results``. + + Pins the cv_cosebis provenance split: values come from the (blindable) + integration ξ± part, the jackknife covariance from the raw patched run. + """ + from sp_validation.cosmo_val.cosebis import CosebisMixin + + raw = { + "En": np.zeros(10), + "Bn": np.zeros(10), + "cov": _spd(20, 11), + "scale_cut": (1.0, 100.0), + } + en_part, bn_part = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 + + class _Stub(CosebisMixin): + sacc_nz = staticmethod(lambda version: {0: _nz()}) + sacc_metadata = staticmethod(lambda version: META) + + out = tmp_path / "part.sacc" + _Stub().cosebis_to_sacc_part( + "vSYNTH", + str(out), + raw, + en_override=en_part, + bn_override=bn_part, + ) + # written with type="data", so reading it back needs the unblinded escape + s = sio.load(str(out), allow_unblinded=True) + n, E, B = sio.get_cosebis(s, (0, 0)) + assert np.array_equal(n, np.arange(1, 11)) + assert np.array_equal(E, en_part) and np.array_equal(B, bn_part) + assert np.array_equal(s.covariance.dense, raw["cov"]) + # the caller's results dict is not mutated + assert np.array_equal(raw["En"], np.zeros(10)) + assert np.array_equal(raw["Bn"], np.zeros(10)) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index afcdbac5..072c26d3 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -368,8 +368,9 @@ rule cv_pure_eb: rule cv_cosebis: """COSEBIs E/B decomposition for one version (config-space, fine binning). - Mixed provenance by design: En derives from the (blindable) integration ξ± - part, while Bn and the jackknife covariance need the patched raw measurement. + Mixed provenance by design: En and Bn derive from the (blindable) + integration ξ± part, while the jackknife covariance needs the patched raw + measurement (patches exist only there). """ input: xi=lambda w: cv_xi_txt(w.version), diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index 726e16ba..342550e0 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -30,13 +30,16 @@ fiducial_scale_cut=fiducial_scale_cut, ) -# Re-derive En from the integration ξ± part via the same kernel; Bn and cov stay raw. +# Re-derive En and Bn from the integration ξ± part via the same kernel; only the +# jackknife covariance stays from the raw patched measurement. from sp_validation import sacc_io from sp_validation.b_modes import cosebis_from_xi integ = sacc_io.load(snakemake.input["xi_integration"]) theta, xip, xim = sacc_io.get_xi(integ, (0, 0), grid="integration") -en_part, _ = cosebis_from_xi(theta, xip, xim, p["nmodes"], scale_cut=fiducial_scale_cut) +en_part, bn_part = cosebis_from_xi( + theta, xip, xim, p["nmodes"], scale_cut=fiducial_scale_cut +) # Born-as-SACC COSEBIs part at the fiducial scale cut. cv.cosebis_to_sacc_part( @@ -45,12 +48,14 @@ cv._cosebis_results[version], fiducial_scale_cut=fiducial_scale_cut, en_override=en_part, + bn_override=bn_part, ) -# Sync the npz's En with the part-derived values; Bn / cov / PTE untouched. +# Sync the npz's En/Bn with the part-derived values; cov / PTE untouched. npz_path = snakemake.output["npz"] data = dict(np.load(npz_path, allow_pickle=True)) data["En"] = np.asarray(en_part) +data["Bn"] = np.asarray(bn_part) np.savez(npz_path, **data) verify_outputs(snakemake) From a88bf7f8ad9179617c5825f05f5855e601bd4c10 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 02:10:31 +0200 Subject: [PATCH 043/160] =?UTF-8?q?Split=20the=20analysis=20pseudo-C?= =?UTF-8?q?=E2=84=93=20into=20its=20own=20rule=20so=20its=20plaintext=20ca?= =?UTF-8?q?n=20be=20temp()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generic `pseudo_cl` rule serves every variant (bmodes claims, mocks, the fine COSEBIs binning), so its output could not be temp()'d and the unblinded analysis part persisted on data runs. The analysis variant now has its own rule and its own name (pseudo_cl_analysis_{version}_{tag}.sacc, built by common.pseudo_cl_analysis_stem — one authority for producer, assembler and the blindable-stem regex), whose plaintext is maybe_temp'd: on a data run only the blinded sibling persists. The generic rule is unchanged for the other workflows. --- .../tests/test_blinding_wiring.py | 11 +++- .../tests/test_bmodes_workflow_dry_run.py | 2 +- workflow/common.py | 12 +++++ workflow/rules/blinding.smk | 2 +- workflow/rules/cosmo_val.smk | 10 ++-- workflow/rules/twopoint.smk | 50 +++++++++++++++---- 6 files changed, 71 insertions(+), 16 deletions(-) diff --git a/src/sp_validation/tests/test_blinding_wiring.py b/src/sp_validation/tests/test_blinding_wiring.py index a5e8ccf6..103a965d 100644 --- a/src/sp_validation/tests/test_blinding_wiring.py +++ b/src/sp_validation/tests/test_blinding_wiring.py @@ -10,6 +10,7 @@ import importlib.util import json import os +import re import subprocess import sys import types @@ -48,8 +49,8 @@ def _load_module(rel_path, name): _STEMS = [ "SP_v1.4.6.3_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=100", "SP_v1.4.6.3_leak_corr_xi_minsep=0.08_maxsep=300_nbins=1000_npatch=1", - "pseudo_cl_SP_v1.4.6.3_blind=A_powspace_nbins=32", - "pseudo_cl_SP_v1.4.6.3_leak_corr_blind=A_powspace_nbins=32", + "pseudo_cl_analysis_SP_v1.4.6.3_blind=A_powspace_nbins=32", + "pseudo_cl_analysis_SP_v1.4.6.3_leak_corr_blind=A_powspace_nbins=32", ] @@ -358,3 +359,9 @@ def test_blinding_subgraph_in_cosmo_val_dry_run(): assert "_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=100_blinded.sacc" in out assert "_xi_minsep=0.08_maxsep=300_nbins=1000_npatch=1_blinded.sacc" in out assert "_blind=A_powspace_nbins=32_blinded.sacc" in out + # Every blindable plaintext part is temp() on a data run, the analysis + # pseudo-Cℓ included (its own rule exists so it can be). + assert "Would remove temporary output" in out, out + assert re.search( + r"Would remove temporary output \S*pseudo_cl_analysis_\S+\.sacc", out + ), out diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index 3b7eecae..c4488cbb 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -82,7 +82,7 @@ def test_cosmo_val_workflow_assemble_dry_runs(): # covariance (not the untagged cv_pseudo_cl diagnostic), plus every part. out = result.stdout assert "rule assemble_sacc:" in out, out - assert f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" in out, out + assert f"pseudo_cl_analysis_{version}_blind=A_powspace_nbins=32.sacc" in out, out assert f"pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits" in out, out for part in ("_xi_minsep=", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): assert part in out, f"missing {part} part in assemble DAG:\n{out}" diff --git a/workflow/common.py b/workflow/common.py index 0878289f..43ed3890 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -213,6 +213,18 @@ def pseudo_cl_tag(config): return f"blind={fiducial['blind']}_{fiducial['binning']}_nbins={fiducial['nbins']}" +def pseudo_cl_analysis_stem(config, version): + """Stem of the analysis pseudo-Cℓ part for a version. + + Its own name (and its own producing rule, twopoint.smk), distinct from the + generic `pseudo_cl` variants the bmodes and mock workflows request: only + this part is blindable, so only it can be temp()'d on a data run. Single + definition shared by the producer, the assembler (cosmo_val.smk) and the + blindable-stem regex (blinding.smk). + """ + return f"pseudo_cl_analysis_{version}_{pseudo_cl_tag(config)}" + + def get_shear_catalog(wildcards): """Resolve shear catalog path from config for a given version.""" cat_config = CATALOG_CONFIG[wildcards.version.replace("_leak_corr", "")] diff --git a/workflow/rules/blinding.smk b/workflow/rules/blinding.smk index d43a7ae8..d951ab80 100644 --- a/workflow/rules/blinding.smk +++ b/workflow/rules/blinding.smk @@ -16,7 +16,7 @@ BLINDABLE_STEM = "(?:{})".format( re.escape(stem) for stem in ( [f"{_VERSION_SLOT}_xi_{xi_binning(grid)}" for grid in XI_GRIDS] - + [f"pseudo_cl_{_VERSION_SLOT}_{pseudo_cl_tag(config)}"] + + [pseudo_cl_analysis_stem(config, _VERSION_SLOT)] ) ).replace(_VERSION_SLOT, WILDCARD_CONSTRAINTS["version"]) ) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 939fb4b4..4189c78f 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -101,7 +101,7 @@ def cv_cosebis_npz(version): def cv_pseudo_cl_sacc(version): """Untagged pseudo-Cl SACC part cv_pseudo_cl writes (B-mode diagnostic). - The analysis file carries the tagged inference product instead (see + The analysis file carries the analysis part instead (see cv_pseudo_cl_analysis_sacc). """ return str(COSMO_VAL / f"pseudo_cl_{version}.sacc") @@ -111,8 +111,12 @@ _PSEUDO_CL_TAG = pseudo_cl_tag(config) def cv_pseudo_cl_analysis_sacc(version): - """Tagged pseudo-Cl SACC part the analysis file carries.""" - return str(COSMO_VAL / f"pseudo_cl_{version}_{_PSEUDO_CL_TAG}.sacc") + """Analysis pseudo-Cl SACC part the analysis file carries. + + Written by the dedicated `pseudo_cl_analysis` rule (twopoint.smk), the only + blindable pseudo-Cℓ variant. + """ + return str(COSMO_VAL / f"{pseudo_cl_analysis_stem(config, version)}.sacc") def cv_pseudo_cl_cov(version): diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 21c3a8c3..bd716416 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -146,25 +146,57 @@ wildcard_constraints: binning="linear|logspace|powspace", +HARMONIC_FIDUCIAL = config["harmonic"]["fiducial"] +PSEUDO_CL_PARAMS = dict( + cat_config=CAT_CONFIG, + nside=1024, + npatch=1, + cosmo_params=PLANCK18, + power=0.5, +) + + rule pseudo_cl: - """Generate pseudo-Cl data vector (born as SACC) with configurable binning.""" + """Generate pseudo-Cl data vector (born as SACC) with configurable binning. + + The diagnostic variants (bmodes claims, mocks, the fine COSEBIs binning); + the analysis part has its own rule below. + """ output: - # One rule for every pseudo-Cℓ variant, only one of which (the analysis - # part) is blindable — so this output cannot be temp()'d and a data run - # blinds it through a requested _blinded sibling instead. pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.sacc"), wildcard_constraints: blind="[ABC]", params: version="{version}", blind="{blind}", - cat_config=CAT_CONFIG, - nside=1024, - npatch=1, - cosmo_params=PLANCK18, binning="{binning}", nbins=lambda w: int(w.nbins), - power=0.5, + **PSEUDO_CL_PARAMS, + resources: + mem_mb=32000, + runtime=120, + threads: 12 + script: + "../scripts/generate_pseudo_cl.py" + + +rule pseudo_cl_analysis: + """The analysis pseudo-Cℓ part, at the fiducial harmonic binning. + + Split from the generic `pseudo_cl` rule because this variant alone is + blindable: on a data run the plaintext part is temp(), consumed only by + blind_part, so only the blinded sibling persists. + """ + output: + pseudo_cl=maybe_temp( + str(COSMO_VAL / f"{pseudo_cl_analysis_stem(config, '{version}')}.sacc") + ), + params: + version="{version}", + blind=HARMONIC_FIDUCIAL["blind"], + binning=HARMONIC_FIDUCIAL["binning"], + nbins=int(HARMONIC_FIDUCIAL["nbins"]), + **PSEUDO_CL_PARAMS, resources: mem_mb=32000, runtime=120, From f5f086bb03987dd6108b2d0f82744d8642b40852 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 02:14:27 +0200 Subject: [PATCH 044/160] test: make the COSEBIs override stub work under the blinded-part writer too --- src/sp_validation/tests/test_sacc_writers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index 6dc3ef1b..798ffbd2 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -335,8 +335,12 @@ def test_cosebis_part_overrides_en_and_bn(tmp_path): en_part, bn_part = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 class _Stub(CosebisMixin): + # run_type / commitment_path are what the blinded-part writer reads; + # a mock part with no commitment keeps this test off the custody gate. + run_type = "mock" sacc_nz = staticmethod(lambda version: {0: _nz()}) sacc_metadata = staticmethod(lambda version: META) + commitment_path = staticmethod(lambda version: None) out = tmp_path / "part.sacc" _Stub().cosebis_to_sacc_part( @@ -346,7 +350,6 @@ class _Stub(CosebisMixin): en_override=en_part, bn_override=bn_part, ) - # written with type="data", so reading it back needs the unblinded escape s = sio.load(str(out), allow_unblinded=True) n, E, B = sio.get_cosebis(s, (0, 0)) assert np.array_equal(n, np.arange(1, 11)) From 6bd65b46c613bfb62d923754d21aedc4fc018e7f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 02:21:39 +0200 Subject: [PATCH 045/160] =?UTF-8?q?Drop=20the=20legacy=20blind=3DA=20field?= =?UTF-8?q?=20from=20the=20analysis=20pseudo-C=E2=84=93=20stem=20(#312)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The A/B/C blind is n(z) vocabulary, not this part's concealment, so the analysis part is now pseudo_cl_analysis_{version}_{binning}_nbins={n}.sacc. pseudo_cl_binning_tag() is the shared half; pseudo_cl_tag() (blind= included) still names the generic pseudo_cl / pseudo_cl_cov products. --- src/sp_validation/tests/test_blinding_wiring.py | 6 +++--- .../tests/test_bmodes_workflow_dry_run.py | 2 +- workflow/common.py | 12 ++++++++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/sp_validation/tests/test_blinding_wiring.py b/src/sp_validation/tests/test_blinding_wiring.py index 103a965d..abeec346 100644 --- a/src/sp_validation/tests/test_blinding_wiring.py +++ b/src/sp_validation/tests/test_blinding_wiring.py @@ -49,8 +49,8 @@ def _load_module(rel_path, name): _STEMS = [ "SP_v1.4.6.3_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=100", "SP_v1.4.6.3_leak_corr_xi_minsep=0.08_maxsep=300_nbins=1000_npatch=1", - "pseudo_cl_analysis_SP_v1.4.6.3_blind=A_powspace_nbins=32", - "pseudo_cl_analysis_SP_v1.4.6.3_leak_corr_blind=A_powspace_nbins=32", + "pseudo_cl_analysis_SP_v1.4.6.3_powspace_nbins=32", + "pseudo_cl_analysis_SP_v1.4.6.3_leak_corr_powspace_nbins=32", ] @@ -358,7 +358,7 @@ def test_blinding_subgraph_in_cosmo_val_dry_run(): # the subgraph for it too. assert "_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=100_blinded.sacc" in out assert "_xi_minsep=0.08_maxsep=300_nbins=1000_npatch=1_blinded.sacc" in out - assert "_blind=A_powspace_nbins=32_blinded.sacc" in out + assert "pseudo_cl_analysis_SP_v1.4.6.3_powspace_nbins=32_blinded.sacc" in out # Every blindable plaintext part is temp() on a data run, the analysis # pseudo-Cℓ included (its own rule exists so it can be). assert "Would remove temporary output" in out, out diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index c4488cbb..fa963b55 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -82,7 +82,7 @@ def test_cosmo_val_workflow_assemble_dry_runs(): # covariance (not the untagged cv_pseudo_cl diagnostic), plus every part. out = result.stdout assert "rule assemble_sacc:" in out, out - assert f"pseudo_cl_analysis_{version}_blind=A_powspace_nbins=32.sacc" in out, out + assert f"pseudo_cl_analysis_{version}_powspace_nbins=32.sacc" in out, out assert f"pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits" in out, out for part in ("_xi_minsep=", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): assert part in out, f"missing {part} part in assemble DAG:\n{out}" diff --git a/workflow/common.py b/workflow/common.py index 43ed3890..b920a309 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -209,8 +209,13 @@ def pseudo_cl_tag(config): Single definition shared by the producer (twopoint.smk) and the consumers (cosmo_val.smk, inference.smk), which reconstruct the name from config. """ + return f"blind={config['harmonic']['fiducial']['blind']}_{pseudo_cl_binning_tag(config)}" + + +def pseudo_cl_binning_tag(config): + """The `{binning}_nbins={n}` half of the tag — the binning alone.""" fiducial = config["harmonic"]["fiducial"] - return f"blind={fiducial['blind']}_{fiducial['binning']}_nbins={fiducial['nbins']}" + return f"{fiducial['binning']}_nbins={fiducial['nbins']}" def pseudo_cl_analysis_stem(config, version): @@ -221,8 +226,11 @@ def pseudo_cl_analysis_stem(config, version): this part is blindable, so only it can be temp()'d on a data run. Single definition shared by the producer, the assembler (cosmo_val.smk) and the blindable-stem regex (blinding.smk). + + Carries the binning but no `blind=` field: the A/B/C blind is the legacy + n(z) vocabulary (#312), not this part's concealment. """ - return f"pseudo_cl_analysis_{version}_{pseudo_cl_tag(config)}" + return f"pseudo_cl_analysis_{version}_{pseudo_cl_binning_tag(config)}" def get_shear_catalog(wildcards): From e37aeaa3a993544a7e250d36d0f62d79c317df75 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 02:43:29 +0200 Subject: [PATCH 046/160] Leave inference.smk untouched: it is #255's to rewrite The dormant inference subsystem was being half-edited here (inference_prep deleted, comments trimmed, pseudo_cl_tag adopted) while #255 rewrites it wholesale. Restored to its merge-base-with-develop state; both paper workflows still dry-run. --- workflow/rules/inference.smk | 102 ++++++++++++++++++++++++++++++----- 1 file changed, 90 insertions(+), 12 deletions(-) diff --git a/workflow/rules/inference.smk b/workflow/rules/inference.smk index 0366420f..d2847976 100644 --- a/workflow/rules/inference.smk +++ b/workflow/rules/inference.smk @@ -1,7 +1,9 @@ # Imports from Snakefile: FIDUCIAL, COSMO_INFERENCE, COSMO_VAL, covariance_path, build_redshift_path, fiducial_binning_suffix -# NOTE: dormant subsystem, not run end-to-end. Reviving it needs the FITS -# content reconciled: cosmosis_fitting.py reads ELL/EE/BB + COVAR_FULL, while -# the producers write PSEUDO_CELL/ELL + COVAR_BB_BB. +# NOTE: dormant subsystem. The file-name plumbing (config-driven paths + the +# producer-tagged pseudo-Cl names) is fixed and the DAG is valid, but it has not +# been run end-to-end. Reviving it still needs the FITS-CONTENT plumbing +# reconciled: cosmosis_fitting.py reads ELL/EE/BB + COVAR_FULL, while the +# producers write PSEUDO_CELL/ELL + COVAR_BB_BB. # Output root for CosmoSIS data products + configs. COSMO_INFERENCE (common.py) # already resolves to THIS repo's cosmo_inference dir, so the products land @@ -12,6 +14,7 @@ COSMO_INFERENCE_RUNDIR = str(COSMO_INFERENCE) # External chain/mock locations are deployment-specific, so they live in config. INFERENCE = config["inference"] +CHAINS_DIR = INFERENCE["chains_dir"] # CosmoSIS chain output root (real data) GLASS_MOCK_DATA_DIR = INFERENCE["glass_mock_data_dir"] # precomputed mock xi/Cl products GLASS_MOCK_CHAINS_DIR = INFERENCE["glass_mock_chains_dir"] # mock chain output root @@ -26,26 +29,99 @@ GLASS_MOCK_FITS_PATTERN = str( ) GLASS_MOCK_CONFIG_PATTERN = str( COSMO_INFERENCE_PROD - / f"cosmosis_config/cosmosis_pipeline_glass_mocks_{GLASS_MOCK_VERSION}_glass_mock_{{mock_id}}.ini" + / f"cosmosis_config/output/cosmosis_pipeline_glass_mocks_{GLASS_MOCK_VERSION}_glass_mock_{{mock_id}}.ini" ) -PSEUDO_CL_TAG = pseudo_cl_tag(config) +# Fiducial harmonic-binning tag the pseudo-Cl producer (twopoint.smk) stamps +# into the filename. These are NOT inference_prep wildcards, so the consumer +# reads them from config to reconstruct the exact name the producer emits +# (canonical: blind=A, powspace, nbins=32 — see twopoint.smk pseudo_cl_all). +HARMONIC_FIDUCIAL = config["harmonic"]["fiducial"] +PSEUDO_CL_TAG = ( + f"blind={HARMONIC_FIDUCIAL['blind']}" + f"_{HARMONIC_FIDUCIAL['binning']}" + f"_nbins={HARMONIC_FIDUCIAL['nbins']}" +) def pseudo_cl_assets(version): - """Pseudo-Cl and covariance paths for a catalog version. + """Return pseudo-Cl and covariance paths for the requested catalog version. - The producer (twopoint.smk) writes wildcard-tagged names; the consumer - reconstructs them from the fiducial harmonic-binning config. + The producer (twopoint.smk rules pseudo_cl / pseudo_cl_cov) writes + wildcard-tagged names; the consumer reconstructs them from the fiducial + harmonic-binning config so the requested path matches byte-for-byte. """ cl_path = PSEUDO_CL_DIR / f"pseudo_cl_{version}_{PSEUDO_CL_TAG}.fits" cov_path = PSEUDO_CL_DIR / f"pseudo_cl_cov_{version}_{PSEUDO_CL_TAG}.fits" return str(cl_path), str(cov_path) -# DORMANT: the SACC migration removed the data products rule inference_prep -# (and its inference_fiducial target) named, so its DAG no longer resolves; the -# rules are dropped rather than left red. Rewiring inference onto the assembled -# {version}.sacc is tracked separately. +rule inference_prep: + input: + # Processed covariance matrix - use centralized covariance_path() + cov_matrix=lambda w: covariance_path(w.version, w.blind, min_sep=w.min_sep, max_sep=w.max_sep, nbins=w.nbins), + # Xi FITS files + xi_plus=str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + xi_minus=str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + # n(z) file (using new location with base version mapping) + nz_file=lambda w: build_redshift_path(w.version, w.blind), + # rho/tau stats + rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + # tau covariance (tracked as dependency) + tau_cov=str(COSMO_VAL / "rho_tau_stats/cov_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}_th.npy"), + pseudo_cl=lambda w: pseudo_cl_assets(w.version)[0], + pseudo_cl_cov=lambda w: pseudo_cl_assets(w.version)[1], + output: + fits_file=str( + COSMO_INFERENCE_PROD + / "data/{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}/cosmosis_{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits" + ), + config_file=str( + COSMO_INFERENCE_PROD + / "cosmosis_config/output/cosmosis_pipeline_{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.ini" + ) + params: + cosmosis_root="{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}", + data_dir=f"{CHAINS_DIR}/{{version}}_{{blind}}_minsep={{min_sep}}_maxsep={{max_sep}}_nbins={{nbins}}_npatch={{npatch}}", + output_root=str(COSMO_INFERENCE_PROD), + threads: 1 + resources: + mem_mb=8000, + runtime=10, + shell: + """ + cd {COSMO_INFERENCE_RUNDIR} + + # Run inference preparation step with cosmosis_fitting.py + python scripts/cosmosis_fitting.py \ + --cosmosis-root {params.cosmosis_root} \ + --nz-file {input.nz_file} \ + --data-dir {params.data_dir} \ + --output-root {params.output_root} \ + --xi {input.xi_plus} {input.xi_minus} \ + --cov-xi {input.cov_matrix} \ + --use-rho-tau \ + --rho-stats {input.rho_stats} \ + --tau-stats {input.tau_stats} \ + --cov-tau {input.tau_cov} \ + --cl-file {input.pseudo_cl} \ + --cov-cl {input.pseudo_cl_cov} + """ + + +rule inference_fiducial: + input: + # Use the same output patterns as inference_prep with FIDUCIAL params + rules.inference_prep.output.fits_file.format( + version=FIDUCIAL["version"], blind=FIDUCIAL["blind"], + min_sep=FIDUCIAL["min_sep"], max_sep=FIDUCIAL["max_sep"], + nbins=FIDUCIAL["nbins"], npatch=FIDUCIAL["npatch"] + ), + rules.inference_prep.output.config_file.format( + version=FIDUCIAL["version"], blind=FIDUCIAL["blind"], + min_sep=FIDUCIAL["min_sep"], max_sep=FIDUCIAL["max_sep"], + nbins=FIDUCIAL["nbins"], npatch=FIDUCIAL["npatch"] + ) rule inference_glass_mocks: @@ -103,5 +179,7 @@ rule inference_prep_glass_mock: """ localrules: + inference_prep, inference_prep_glass_mock, + inference_fiducial, inference_glass_mocks, From f11670c9b21fa760b3dc2f1ee0a4cfb855a4fed6 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 02:47:50 +0200 Subject: [PATCH 047/160] =?UTF-8?q?Assemble=20against=20the=20real=20Cosmo?= =?UTF-8?q?Cov=20=CE=BE=C2=B1=20covariance;=20drop=20the=20placeholder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ξ± block now comes from the CosmoCov-processed covariance on the reporting binning, wired as a DAG input of assemble_sacc (cv_xi_cov), so requesting the terminal file builds the covariance chain. With a real block always present, the allow_placeholder_cov switch and the diagonal stand-in are gone: a missing covariance is a missing Snakemake input, and a cov-less part with nothing to inject still raises. The tests use a CosmoCov-format .txt fixture written with the same writer the seam reads. --- src/sp_validation/tests/test_assemble_sacc.py | 57 ++++++++++++------- workflow/rules/cosmo_val.smk | 29 ++++++++-- workflow/scripts/assemble_sacc.py | 36 ++++-------- 3 files changed, 69 insertions(+), 53 deletions(-) diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py index 323357f5..25da7375 100644 --- a/src/sp_validation/tests/test_assemble_sacc.py +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -3,8 +3,8 @@ The pure assembler (``sacc_writers.assemble_analysis_sacc``) is covered in ``test_sacc_writers.py``. This file exercises the *script seam* the DAG uses: ``assemble_sacc.assemble_sacc`` loads per-statistic ``.sacc`` part *files* in -CANONICAL order, injects the born-cov-less ξ± / pseudo-Cℓ blocks (real CosmoCov -/ NaMaster covariance, or a flagged diagonal placeholder), and writes one +CANONICAL order, injects the born-cov-less ξ± / pseudo-Cℓ blocks from the real +CosmoCov ``.txt`` and NaMaster covariance FITS, and writes one ``{version}.sacc`` whose points and covariance blocks land in canonical order. The script lives under ``workflow/scripts`` (off the package path); it is loaded @@ -53,6 +53,19 @@ def _theta(n=6): META = {"catalogue_version": "vSYNTH", "npatch": 1} +def _xi_cov_txt(tmp_path, n=12, seed=21): + """A CosmoCov-format ξ± covariance: the dense ``_processed.txt`` matrix. + + Same writer/format ``covariance_process`` emits and ``--xi-cov`` reads, at + the synthetic parts' 12-point ([ξ+; ξ−] over 6 θ) size. Returns + ``(path, matrix)``. + """ + cov = _spd(n, seed) + path = tmp_path / "xi_cov_processed.txt" + np.savetxt(str(path), cov) + return str(path), cov + + def _write_parts(tmp_path, *, with_pseudo_cl=True, cov_less=("xi_reporting",)): """Write per-statistic parts to disk; return the ``{name: path}`` mapping. @@ -136,12 +149,13 @@ def get_bandpower_windows(self): return paths -def test_assemble_sacc_placeholder_canonical_order(tmp_path): - """The cov-less ξ± part gets a placeholder; every point is covered and the - blocks land in canonical order (ξ±, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ).""" +def test_assemble_sacc_canonical_order(tmp_path): + """Every point is covered and the blocks land in canonical order + (ξ±, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ).""" paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) + cov_path, xi_cov = _xi_cov_txt(tmp_path) out = tmp_path / "vSYNTH.sacc" - s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=cov_path) assert out.exists() assert type(s.covariance).__name__ == "BlockDiagonalCovariance" assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) @@ -157,11 +171,11 @@ def test_assemble_sacc_placeholder_canonical_order(tmp_path): assert first[sio.PURE_TYPES["xip_E"]] < first[sio.RHO_PLUS.format(k=0)] assert first[sio.RHO_PLUS.format(k=0)] < first[sio.TAU_PLUS.format(k=0)] - # The ξ± block is the placeholder diagonal (variance 1.0 on its own points). + # The ξ± block is the injected CosmoCov matrix on its own points. tr = ("source_0", "source_0") xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) dense = s.covariance.dense - assert np.allclose(np.diag(dense[np.ix_(xi_idx, xi_idx)]), 1.0) + assert np.allclose(dense[np.ix_(xi_idx, xi_idx)], xi_cov) # ...and it does not bleed into the neighbouring COSEBIs block (cross zero). co_idx = np.concatenate( [s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)] @@ -172,13 +186,10 @@ def test_assemble_sacc_placeholder_canonical_order(tmp_path): def test_assemble_sacc_injects_real_xi_covariance(tmp_path): """A CosmoCov ξ covariance .txt is loaded into the cov-less ξ± block.""" paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) - # ξ± part has 12 points ([ξ+; ξ−] over 6 θ); supply a matching cov .txt. - xi_cov = _spd(12, 21) - cov_path = tmp_path / "xi_cov.txt" - np.savetxt(str(cov_path), xi_cov) + cov_path, xi_cov = _xi_cov_txt(tmp_path) out = tmp_path / "vSYNTH.sacc" - s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=str(cov_path)) + s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=cov_path) tr = ("source_0", "source_0") xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) assert np.allclose(s.covariance.dense[np.ix_(xi_idx, xi_idx)], xi_cov) @@ -186,10 +197,11 @@ def test_assemble_sacc_injects_real_xi_covariance(tmp_path): def test_assemble_sacc_injects_pseudo_cl_covariance(tmp_path): """The NaMaster cov FITS (COVAR_EE_EE/BB_BB/EB_EB) → block-diagonal pseudo-Cℓ - block (the live default: ξ± placeholder + real pseudo-Cℓ cov).""" + block, beside the injected CosmoCov ξ± block (the live default).""" from astropy.io import fits paths = _write_parts(tmp_path, cov_less=("xi_reporting", "pseudo_cl")) + cov_path, xi_cov = _xi_cov_txt(tmp_path) # pseudo-Cℓ part is 3 ell × {EE, BB, EB} = 9 points; per-spectrum 3×3 blocks. ee, bb, eb = _spd(3, 31), _spd(3, 32), _spd(3, 33) cov_fits = tmp_path / "pseudo_cl_cov.fits" @@ -204,7 +216,7 @@ def test_assemble_sacc_injects_pseudo_cl_covariance(tmp_path): out = tmp_path / "vSYNTH.sacc" s = asm.assemble_sacc( - "vSYNTH", paths, str(out), pseudo_cl_cov=str(cov_fits), placeholder_var=1.0 + "vSYNTH", paths, str(out), xi_cov=cov_path, pseudo_cl_cov=str(cov_fits) ) tr = ("source_0", "source_0") cl_idx = np.concatenate( @@ -214,14 +226,14 @@ def test_assemble_sacc_injects_pseudo_cl_covariance(tmp_path): expected = np.zeros((9, 9)) expected[0:3, 0:3], expected[3:6, 3:6], expected[6:9, 6:9] = ee, bb, eb assert np.allclose(dense[np.ix_(cl_idx, cl_idx)], expected) - # ξ± stays the placeholder; the two blocks don't bleed into each other. + # ξ± carries its own CosmoCov block; the two don't bleed into each other. xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) - assert np.allclose(np.diag(dense[np.ix_(xi_idx, xi_idx)]), 1.0) + assert np.allclose(dense[np.ix_(xi_idx, xi_idx)], xi_cov) assert np.allclose(dense[np.ix_(xi_idx, cl_idx)], 0.0) def test_assemble_sacc_missing_cov_raises(tmp_path): - """A cov-less part with no injected block and no placeholder fails loudly.""" + """A cov-less part with no injected block fails loudly.""" paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) out = tmp_path / "vSYNTH.sacc" with pytest.raises(ValueError, match="carries no covariance"): @@ -232,8 +244,9 @@ def test_assemble_sacc_respects_pseudo_cl_toggle(tmp_path): """With pseudo_cl absent, assembly still succeeds and omits the Cℓ points.""" paths = _write_parts(tmp_path, with_pseudo_cl=False, cov_less=("xi_reporting",)) assert "pseudo_cl" not in paths + cov_path, _xi_cov = _xi_cov_txt(tmp_path) out = tmp_path / "vSYNTH.sacc" - s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=cov_path) tr = ("source_0", "source_0") assert len(s.indices(sio.CL_EE, tr)) == 0 # Round-trips as a valid BlockDiagonalCovariance over the remaining points. @@ -248,6 +261,7 @@ def test_assemble_sacc_expected_part_missing_raises(tmp_path): paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) # Simulate a rule-input typo: cosebis wired under the wrong key. paths["cosebi"] = paths.pop("cosebis") + cov_path, _xi_cov = _xi_cov_txt(tmp_path) out = tmp_path / "vSYNTH.sacc" with pytest.raises(ValueError, match="expected parts \\['cosebis'\\] missing"): asm.assemble_sacc( @@ -255,15 +269,16 @@ def test_assemble_sacc_expected_part_missing_raises(tmp_path): paths, str(out), expected=["xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau"], - placeholder_var=1.0, + xi_cov=cov_path, ) def test_assemble_sacc_expected_rejects_unknown_name(tmp_path): """A typo in the expected list itself is rejected (not a valid statistic).""" paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) + cov_path, _xi_cov = _xi_cov_txt(tmp_path) out = tmp_path / "vSYNTH.sacc" with pytest.raises(ValueError, match="not assemblable statistics"): asm.assemble_sacc( - "vSYNTH", paths, str(out), expected=["cosebi"], placeholder_var=1.0 + "vSYNTH", paths, str(out), expected=["cosebi"], xi_cov=cov_path ) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 072c26d3..ef426edd 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -120,6 +120,25 @@ def cv_pseudo_cl_cov(version): return str(COSMO_VAL / f"pseudo_cl_cov_{version}_{_PSEUDO_CL_TAG}.fits") +def cv_xi_cov(version): + """CosmoCov-processed ξ± covariance for the reporting grid. + + The real analysis covariance for the ξ± block, produced by + covariance_process (covariance.smk) on the same binning the reporting grid + measures, so requesting it builds it. OneCovariance replaces CosmoCov + upstream of this path (#256) in the same format. + """ + return covariance_path( + version, + FIDUCIAL["blind"], + gaussian="ng", + min_sep=CV["theta_min"], + max_sep=CV["theta_max"], + nbins=CV["nbins"], + mask_suffix=DEFAULT_MASK_SUFFIX, + ) + + def cv_cosebis_sacc(version): """COSEBIs SACC part (fiducial scale cut) the cv_cosebis rule writes.""" return str(COSMO_VAL / f"{version}_cosebis.sacc") @@ -453,6 +472,7 @@ def cv_assemble_inputs(version): """ parts = dict( xi_reporting=cv_xi_sacc(version, "reporting"), + xi_cov=cv_xi_cov(version), cosebis=cv_cosebis_sacc(version), pure_eb=cv_pure_eb_sacc(version), rho_tau=cv_rho_tau_sacc(version), @@ -477,13 +497,10 @@ rule assemble_sacc: # Statistics this rule wired; the script validates part_paths against it # so a typo'd input keyword can't silently drop one. expected=lambda w: [ - k for k in cv_assemble_inputs(w.version) if k != "pseudo_cl_cov" + k + for k in cv_assemble_inputs(w.version) + if k not in ("xi_cov", "pseudo_cl_cov") ], - # Without a real ξ± covariance, assembly raises by default rather than - # ship {version}.sacc with a var=1.0 leading block — ~20 orders off the - # real variance, i.e. silently catastrophic χ²/PTE for any consumer. The - # opt-in is for dry-run and test configs only. - placeholder_var=(1.0 if CV.get("allow_placeholder_cov", False) else None), resources: mem_mb=8000, runtime=20, diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 97a86f43..410bc019 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -12,9 +12,8 @@ with one; ξ± reporting and pseudo-Cℓ are not, so their blocks are injected here from the CosmoCov ``.txt`` (``--xi-cov``) and the NaMaster covariance FITS (``--pseudo-cl-cov``). The pseudo-Cℓ cross-spectrum blocks (EE↔BB, …) are -dropped — matching what the B-mode PTE reads today. ``--allow-placeholder VAR`` -attaches a flagged diagonal stand-in instead, so a dry run or the fast test can -still build a structurally valid covariance. +dropped — matching what the B-mode PTE reads today. Both are DAG inputs of the +assemble rule, so a missing one is a missing input, never a stand-in. """ import argparse @@ -48,11 +47,10 @@ def _pseudo_cl_cov_block(cov_fits): return full -def _attach_cov(part, name, xi_cov, pseudo_cl_cov, placeholder_var): +def _attach_cov(part, name, xi_cov, pseudo_cl_cov): """Ensure ``part`` (mutated in place) carries a covariance block. - Raises if a required ξ± / pseudo-Cℓ block is missing and no placeholder was - requested. + Raises if a required ξ± / pseudo-Cℓ block was not supplied. """ if part.covariance is not None: return part @@ -62,14 +60,10 @@ def _attach_cov(part, name, xi_cov, pseudo_cl_cov, placeholder_var): if name == "pseudo_cl" and pseudo_cl_cov is not None: part.add_covariance(_pseudo_cl_cov_block(pseudo_cl_cov)) return part - if placeholder_var is None: - raise ValueError( - f"the {name!r} part carries no covariance and no covariance input was " - f"given (--xi-cov / --pseudo-cl-cov). Supply the block, or pass " - "--allow-placeholder to attach a documented diagonal placeholder." - ) - part.add_covariance(np.full(len(part.mean), float(placeholder_var))) - return part + raise ValueError( + f"the {name!r} part carries no covariance and no covariance input was " + "given (--xi-cov / --pseudo-cl-cov); supply the block" + ) def assemble_sacc( @@ -80,7 +74,6 @@ def assemble_sacc( expected=None, xi_cov=None, pseudo_cl_cov=None, - placeholder_var=None, allow_unblinded=False, ): """Assemble ``{version}.sacc`` from the per-statistic ``part_paths`` mapping. @@ -95,7 +88,7 @@ def assemble_sacc( expected : sequence of str, optional Statistics that must be present, from the caller's config toggles. A typo'd input keyword would otherwise silently drop a statistic. - xi_cov, pseudo_cl_cov, placeholder_var + xi_cov, pseudo_cl_cov Covariance sourcing — see the module docstring. allow_unblinded : bool, optional Passed to :func:`sacc_io.load` for every part; ``True`` only for mocks. @@ -120,7 +113,7 @@ def assemble_sacc( if path is None: continue part = sacc_io.load(path, allow_unblinded=allow_unblinded) - parts.append(_attach_cov(part, name, xi_cov, pseudo_cl_cov, placeholder_var)) + parts.append(_attach_cov(part, name, xi_cov, pseudo_cl_cov)) if not parts: raise ValueError(f"no parts found for {version}: {part_paths}") s = assemble_analysis_sacc(parts) @@ -144,7 +137,6 @@ def _from_snakemake(smk): expected=list(p["expected"]), xi_cov=getattr(inp, "xi_cov", None), pseudo_cl_cov=getattr(inp, "pseudo_cl_cov", None), - placeholder_var=p.get("placeholder_var", None), allow_unblinded=(p.get("type", "data") == "mock"), ) @@ -170,13 +162,6 @@ def _from_cli(argv=None): ap.add_argument( "--pseudo-cl-cov", default=None, help="NaMaster pseudo-Cℓ covariance FITS" ) - ap.add_argument( - "--allow-placeholder", - type=float, - default=None, - metavar="VAR", - help="Attach a diagonal placeholder (variance VAR) to cov-less parts", - ) a = ap.parse_args(argv) part_paths = {name: getattr(a, name) for name in CANONICAL if getattr(a, name)} assemble_sacc( @@ -185,7 +170,6 @@ def _from_cli(argv=None): out_path=a.out, xi_cov=a.xi_cov, pseudo_cl_cov=a.pseudo_cl_cov, - placeholder_var=a.allow_placeholder, allow_unblinded=(a.type == "mock"), ) From cc6dce5bd2431e09314193b90d12be3ff9c31c1c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 02:48:57 +0200 Subject: [PATCH 048/160] test: drop the removed placeholder_var kwarg from the blinding assembly tests --- .../tests/test_blinding_wiring.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/sp_validation/tests/test_blinding_wiring.py b/src/sp_validation/tests/test_blinding_wiring.py index abeec346..989b5545 100644 --- a/src/sp_validation/tests/test_blinding_wiring.py +++ b/src/sp_validation/tests/test_blinding_wiring.py @@ -155,9 +155,7 @@ def test_data_assemble_fails_closed_on_unblinded_part(tmp_path): """A data run refuses to assemble an unconcealed real part (fail closed).""" paths = _data_parts(tmp_path, conceal=False) with pytest.raises(ValueError, match="refusing to load an unblinded"): - asm.assemble_sacc( - "vSYNTH", paths, str(tmp_path / "vSYNTH.sacc"), placeholder_var=1.0 - ) + asm.assemble_sacc("vSYNTH", paths, str(tmp_path / "vSYNTH.sacc")) def test_data_assemble_passes_on_blinded_parts(tmp_path): @@ -165,7 +163,7 @@ def test_data_assemble_passes_on_blinded_parts(tmp_path): and stamps the shared commitment on the terminal file.""" paths = _data_parts(tmp_path, conceal=True) out = tmp_path / "vSYNTH.sacc" - s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + s = asm.assemble_sacc("vSYNTH", paths, str(out)) assert s.metadata["concealed"] is True assert s.metadata["blind_commitment"] == _COMMIT assert s.metadata["blind_config_digest"] == _DIGEST @@ -179,9 +177,7 @@ def test_data_assemble_refuses_blinded_plaintext_mix(tmp_path): # The plaintext ξ± reporting part fails the load gate first (data + not # concealed), so the mix can never even reach assembly. with pytest.raises(ValueError, match="refusing to load an unblinded"): - asm.assemble_sacc( - "vSYNTH", paths, str(tmp_path / "vSYNTH.sacc"), placeholder_var=1.0 - ) + asm.assemble_sacc("vSYNTH", paths, str(tmp_path / "vSYNTH.sacc")) def test_data_assemble_runs_behind_the_custody_guard(tmp_path, monkeypatch): @@ -199,16 +195,14 @@ def test_data_assemble_runs_behind_the_custody_guard(tmp_path, monkeypatch): paths = _data_parts(tmp_path, conceal=True) monkeypatch.setattr(blinding, "draw_scheme", lambda: 99) with pytest.raises(ValueError, match="DRAW_SCHEME"): - asm.assemble_sacc( - "vSYNTH", paths, str(tmp_path / "vSYNTH.sacc"), placeholder_var=1.0 - ) + asm.assemble_sacc("vSYNTH", paths, str(tmp_path / "vSYNTH.sacc")) def test_data_assemble_stamps_the_draw_scheme_on_the_terminal_file(tmp_path): """The assembled file carries the blind's draw scheme, like its parts.""" paths = _data_parts(tmp_path, conceal=True) out = tmp_path / "vSYNTH.sacc" - asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + asm.assemble_sacc("vSYNTH", paths, str(out)) assert sio.load(str(out)).metadata["blind_draw_scheme"] == blinding.draw_scheme() @@ -224,7 +218,7 @@ def test_mock_assemble_succeeds_without_a_blind(tmp_path): """ paths = _data_parts(tmp_path, conceal=False, run_type="mock") out = tmp_path / "vSYNTH.sacc" - s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + s = asm.assemble_sacc("vSYNTH", paths, str(out)) assert "concealed" not in s.metadata # No escape hatch: a mock part is not gated by the fail-closed loader. assert sio.load(str(out)).metadata["type"] == "mock" From 726a0caef58c3eaee70a5f1587a6ab81f391b0ce Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 02:58:32 +0200 Subject: [PATCH 049/160] covariance_process: run the launched checkout's cosmocov_process.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rule shelled out to an absolute path through the deprecated pure_eb symlink, i.e. a different checkout on an unrelated branch — now on the critical path, since assemble_sacc depends on this rule. REPO_ROOT (common.py) anchors on the module's realpath, the same way the rest of the workflow resolves the running checkout under `module` composition. --- workflow/common.py | 6 ++++++ workflow/rules/covariance.smk | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/workflow/common.py b/workflow/common.py index 63081e4e..bc7d1c71 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -11,6 +11,12 @@ # composition basedir reflects the composing paper, not the running checkout. WORKFLOW_SCRIPTS = os.path.join(os.path.dirname(os.path.realpath(__file__)), "scripts") +# The running checkout, for rules that shell out to a repo script directly +# rather than through Snakemake's `script:` directive. Anchored on this module's +# own location, not workflow.basedir — under `module` composition basedir +# reflects the composing paper, not the running checkout. +REPO_ROOT = Path(os.path.realpath(__file__)).parents[1] + # Output roots are env-overridable so a reproduction run can write into a # fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL = Path( diff --git a/workflow/rules/covariance.smk b/workflow/rules/covariance.smk index 49e40201..97feedbe 100644 --- a/workflow/rules/covariance.smk +++ b/workflow/rules/covariance.smk @@ -253,7 +253,7 @@ rule covariance_process: threads: 1 shell: """ - python /n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/scripts/cosmocov_process.py {input} {params.output_stub} + python {REPO_ROOT}/cosmo_inference/scripts/cosmocov_process.py {input} {params.output_stub} """ From 6fdad562256acb3d137dd54839eb2d5dee7290df Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 02:59:15 +0200 Subject: [PATCH 050/160] Derive WORKFLOW_SCRIPTS / REPO_SCRIPTS from REPO_ROOT --- workflow/common.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/workflow/common.py b/workflow/common.py index 4bd0432c..a70fef54 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -11,21 +11,15 @@ # conventions without importing numpy + smokescreen. from sp_validation.blinding_paths import init_paths, part_paths -# Absolute path to the generic workflow's scripts, for rules that shell out to a -# script directly rather than through Snakemake's `script:` directive. Anchored -# on this module's own location, not workflow.basedir — under `module` -# composition basedir reflects the composing paper, not the running checkout. -WORKFLOW_SCRIPTS = os.path.join(os.path.dirname(os.path.realpath(__file__)), "scripts") -# The repo's top-level scripts/ (user-facing CLIs), anchored the same way. -REPO_SCRIPTS = os.path.join( - os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "scripts" -) - # The running checkout, for rules that shell out to a repo script directly # rather than through Snakemake's `script:` directive. Anchored on this module's # own location, not workflow.basedir — under `module` composition basedir # reflects the composing paper, not the running checkout. REPO_ROOT = Path(os.path.realpath(__file__)).parents[1] +# The generic workflow's scripts, and the repo's top-level scripts/ (user-facing +# CLIs). +WORKFLOW_SCRIPTS = str(REPO_ROOT / "workflow" / "scripts") +REPO_SCRIPTS = str(REPO_ROOT / "scripts") # Output roots are env-overridable so a reproduction run can write into a # fresh tree without clobbering (or silently reusing) prior products. From 34db82f7602ceee9f83c44a324461f97d45e9946 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 03:07:25 +0200 Subject: [PATCH 051/160] Leave cosmo_inference/README.md untouched (inference is out of scope here) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NmGA86b7YyQn54JFj78ssM --- cosmo_inference/README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cosmo_inference/README.md b/cosmo_inference/README.md index aa52e57f..5d753010 100644 --- a/cosmo_inference/README.md +++ b/cosmo_inference/README.md @@ -7,9 +7,17 @@ This folder contains the files neccessary to run the cosmological inference pipe To run the pipeline, one would need to have installed [CosmoSIS](https://cosmosis.readthedocs.io/en/latest/). To sample the PSF leakage parameters, the fork of [cosmosis-standard-library](https://github.com/sachaguer/cosmosis-standard-library/) of Sacha Guerrini has to be used. ### To Run -The `inference_fiducial` Snakemake target is dormant: the SACC migration -removed the per-product FITS its `inference_prep` rule consumed, and rewiring -inference onto the assembled `{version}.sacc` is tracked separately. +The inference pipeline is now orchestrated through Python. Run the main Snakemake workflow from the parent directory: + +```bash +snakemake -j inference_fiducial +``` + +This will automatically execute all steps: +1. Calculate 2PCF ($\xi_{pm}$) via `cosmo_val.py` +2. Compute covariance matrices using CosmoCov +3. Prepare CosmoSIS data (FITS) via `cosmosis_fitting.py` +4. Run CosmoSIS inference For standalone FITS data preparation (real-space inputs plus optional pseudo-$C_\ell$ data), you can also use the Python script directly: From dfe3a5ba2af5dbf4f96c0c7e2110f5c34c448c6a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 03:14:00 +0200 Subject: [PATCH 052/160] Restore image_sims include + README section (out of scope here) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NmGA86b7YyQn54JFj78ssM --- workflow/README.md | 33 +++++++++++++++++++++++++++++++++ workflow/Snakefile | 5 +++++ 2 files changed, 38 insertions(+) diff --git a/workflow/README.md b/workflow/README.md index 47015158..f20ffe8f 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -24,3 +24,36 @@ What does *not* belong here: Runs stay modular, not monolithic: a paper or run composes these rules with Snakemake's `module` directive under its own config and an output `prefix`, so each namespaces cleanly under `results//`. + +## Running on the cluster — the candide profile + +`profiles/candide/config.yaml` is the committed SLURM profile: it hands +Snakemake the candide executor, account, partition, node excludes, and per-job +floor, so scheduling is repo state rather than an operator's shell. Drive any +target with one command: + +```bash +snakemake --profile workflow/profiles/candide \ + -s workflow/image_sims/Snakefile \ + --configfile +``` + +For example, the image-sim m-bias chain end to end (`im_mbias` fans out one +SLURM job per branch × tile, MPI-free): + +```bash +snakemake --profile workflow/profiles/candide \ + -s workflow/image_sims/Snakefile \ + im_mbias --configfile my_run.yaml +``` + +Give the target *before* `--configfile`: `--configfile` takes one-or-more +paths, so a target after it is read as a config file ("No such file: +im_mbias"). Always dry-run first with `-n`. + +The profile carries only cluster policy — no container settings (the image-sims +rules own their `apptainer exec` call) and no `OMP_NUM_THREADS` (pinned to 1 at +that same `apptainer exec` line, since the slurm executor's `--export=ALL` +propagates the driver's env, not a profile flag). Per-rule `mem_mb` / `runtime` +stay on the rules. Off-cluster, drop `--profile` and add `-j N`. See the +profile's own comments for the full rationale. diff --git a/workflow/Snakefile b/workflow/Snakefile index 6f59827b..7d91db81 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -47,3 +47,8 @@ include: "rules/glass_mock.smk" # guarded so paper configs without it (e.g. bmodes) don't trip on the lookups. if "cosmo_val" in config: include: "rules/cosmo_val.smk" + +# Image-simulation m/c-bias chain (image_sims.smk). Active only when the config +# carries an `image_sims` block; standalone runs use workflow/image_sims/Snakefile. +if "image_sims" in config: + include: "rules/image_sims.smk" From 9abf08d0e7891dd2b3227d7191488d6415105805 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 03:14:51 +0200 Subject: [PATCH 053/160] README: state the 3.12 requirement, not its rationale Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NmGA86b7YyQn54JFj78ssM --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index ef9e601b..36d5589e 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,7 @@ We do not currently build images for Apple Silicon/arm64; however the amd64 imag ## Local Installation -Requires Python ≥ 3.12 (the floor is set by the blinding stack; the container -already runs 3.12). With [uv](https://docs.astral.sh/uv/): +Requires Python ≥ 3.12. With [uv](https://docs.astral.sh/uv/): ```bash uv venv --python 3.12 From bfa4819edf7ed630f316607afdfa8b55fa575d6c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 03:19:43 +0200 Subject: [PATCH 054/160] Comment sweep: each concept once, at the thing itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kills repeats (the same grid/naming/canonical-order explanation restated at producer, rule and script) and comments that narrated what downstream code does — consumer names, rule wiring, pipeline shape — which rot as soon as the consumer moves. What stays is each thing's own contract plus the invariants a reader would violate: the inclusive scale-cut selection, the strict sub-range the pure-E/B integration grid must satisfy, why the integration grid carries no covariance, why an already-written part may be read back unblinded, and the covariance ordering assembly depends on. Also drops the stale rename note in generate_pseudo_cl (parts are born at their final path) and the duplicated REPO_ROOT anchoring rationale in common.py. --- papers/bmodes/rules/claims.smk | 6 +- papers/bmodes/scripts/run_xi_sweep.py | 2 - papers/cosmo_val/config/config.yaml | 4 +- pyproject.toml | 3 +- src/sp_validation/b_modes.py | 7 +-- src/sp_validation/cosmo_val/core.py | 4 +- src/sp_validation/cosmo_val/cosebis.py | 20 +++---- src/sp_validation/cosmo_val/pseudo_cl.py | 14 +---- src/sp_validation/cosmo_val/pure_eb.py | 6 +- src/sp_validation/cosmo_val/real_space.py | 4 +- src/sp_validation/cosmo_val/sacc_writers.py | 29 ++++----- src/sp_validation/tests/test_sacc_writers.py | 9 +-- workflow/common.py | 21 ++----- workflow/rules/cosmo_val.smk | 62 +++++--------------- workflow/rules/twopoint.smk | 17 ++---- workflow/scripts/assemble_sacc.py | 19 +++--- workflow/scripts/cv_cosebis.py | 12 +--- workflow/scripts/cv_pseudo_cl.py | 6 +- workflow/scripts/cv_pure_eb.py | 11 +--- workflow/scripts/generate_pseudo_cl.py | 14 ++--- workflow/scripts/run_2pcf.py | 19 +++--- workflow/scripts/run_rho_tau.py | 2 +- 22 files changed, 90 insertions(+), 201 deletions(-) diff --git a/papers/bmodes/rules/claims.smk b/papers/bmodes/rules/claims.smk index a41833ed..c6b5b0e7 100644 --- a/papers/bmodes/rules/claims.smk +++ b/papers/bmodes/rules/claims.smk @@ -86,11 +86,7 @@ def _xi_reporting_path(version): def _xi_integration_path(version): - """Path to fine-binned 2PCF integration file. - - The integration grid is unpatched (npatch=1, matching XI_GRIDS): it supplies - values only, never a jackknife covariance. - """ + """Path to fine-binned 2PCF integration file. Unpatched: values only, no covariance.""" return ( f"{COSMO_VAL_OUTPUT}/{version}_xi_minsep={FIDUCIAL['min_sep_int']}" f"_maxsep={FIDUCIAL['max_sep_int']}_nbins={FIDUCIAL['nbins_int']}_npatch=1.txt" diff --git a/papers/bmodes/scripts/run_xi_sweep.py b/papers/bmodes/scripts/run_xi_sweep.py index a4674cb5..43233474 100644 --- a/papers/bmodes/scripts/run_xi_sweep.py +++ b/papers/bmodes/scripts/run_xi_sweep.py @@ -72,8 +72,6 @@ def _from_cli(argv=None): versions = a.versions or nonfiducial_versions(config) for ver in versions: for grid in a.grids: - # The sweep consumes only the .txt dump; the SACC part's default - # name carries the binning, so the two grids land in distinct files. run_2pcf( ver=ver, cat_config=a.cat_config, diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 1e528850..cbbe33cf 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -58,9 +58,7 @@ cosmo_val: kmax: 20 kmax_extrapolate: 500 - # Integration-grid ξ±: the shared fine grid both B-mode estimators consume. - # Pure-E/B uses the full range, which must strictly contain the reporting grid - # [1, 250]; COSEBIs scale-cuts it up to 0.9. + # The fine ξ± grid the B-mode integrals run over. integration: min_sep: 0.08 max_sep: 300 diff --git a/pyproject.toml b/pyproject.toml index a7385d3e..d278d3f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,8 +79,7 @@ dependencies = [ "pymaster", "regions", "reproject", - # sacc_io uses BlockDiagonalCovariance, from sacc's 2.x rewrite; the lock - # already resolves to 2.4, this just makes the floor honest. + # Floor matches the resolved lock; needs sacc's 2.x rewrite (BlockDiagonalCovariance). "sacc>=2.4,<3", # scipy 1.18 ported FITPACK from Fortran to C, changing the return shape of # RectBivariateSpline(scalar, scalar, grid=False) from 0-d `array(x)` to diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index f843007d..f05c84a4 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -259,11 +259,10 @@ def cosebis_from_xi(theta, xip, xim, nmodes, scale_cut=None): """COSEBIs (Eₙ, Bₙ) from ξ± arrays through the pipeline kernel (values only). The values-only seam of :func:`calculate_cosebis`, for callers holding ξ± - arrays rather than a TreeCorr ``GGCorrelation``; the covariance/χ² machinery - stays with :func:`calculate_cosebis`. + arrays rather than a TreeCorr ``GGCorrelation``. - ``scale_cut`` follows the :func:`sacc_io.add_cosebis` contract: ``(theta_min, - theta_max)`` are min/max of the *retained bin centres*, selected inclusively. + ``scale_cut`` is ``(theta_min, theta_max)``, the min/max of the retained bin + centres, selected inclusively. """ from cosmo_numba.B_modes.cosebis import COSEBIS diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 0c9273d8..4f7f99c2 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -384,9 +384,7 @@ def _output_path(self, *parts): def sacc_nz(self, version): """Single-bin ``nz`` mapping ``{0: (z, nz)}`` for the SACC writers. - The tomography-native writer interface (``sacc_writers``) takes an nz - dict keyed by 0-based source bin; the round is single-bin, so the whole - survey n(z) is bin 0. ``get_redshift`` returns ``(z, nz)``. + The round is single-bin, so the whole survey n(z) is bin 0. """ return {0: tuple(self.get_redshift(version))} diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index d9d61582..53a1dd48 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -146,11 +146,10 @@ def calculate_cosebis( def _fiducial_cosebis_result(results, fiducial_scale_cut): """Select the fiducial scale cut's result dict + its ``(min, max)`` cut. - ``calculate_cosebis`` returns either a single result dict (full range) or - a multi-cut mapping keyed by ``(theta_min, theta_max)`` tuples. Only the - fiducial cut is a SACC data product: pick it via - ``find_conservative_scale_cut_key`` when a fiducial cut is given, else the - widest cut — mirroring ``plot_cosebis``. + ``results`` is a single result dict (full range) or a multi-cut mapping + keyed by ``(theta_min, theta_max)`` tuples; picks via + ``find_conservative_scale_cut_key`` when ``fiducial_scale_cut`` is given, + else the widest cut. """ multi_cut = isinstance(results, dict) and all( isinstance(k, tuple) for k in results @@ -176,13 +175,12 @@ def cosebis_to_sacc_part( """Write the COSEBIs SACC part at the fiducial scale cut. ``results`` is what ``calculate_cosebis`` returned (single dict or - multi-cut mapping). Only the fiducial cut's ``{En, Bn, cov}`` becomes the - part: the covariance must cover every stored point and the cuts overlap - in mode space, so the non-fiducial cuts stay in the ``.npz`` sidecar. + multi-cut mapping); only the fiducial cut's ``{En, Bn, cov}`` becomes + the part. - ``en_override``/``bn_override`` replace ``result["En"]``/``result["Bn"]`` - with values derived from the integration ξ± part; the covariance stays - from ``result`` (patches exist only in the raw patched measurement). + ``en_override``/``bn_override`` replace ``result["En"]``/``result["Bn"]``; + the covariance stays from ``result`` (patches exist only in the raw + patched measurement). """ result, scale_cut = self._fiducial_cosebis_result(results, fiducial_scale_cut) overrides = { diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index ab871448..b327e37b 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -458,17 +458,8 @@ def calculate_pseudo_cl(self, out_path=None): """ Compute the pseudo-Cl of given catalogs. - Each version's spectra are born as a SACC part via - :func:`sacc_writers.pseudo_cl_to_sacc` — EE/BB/EB carrying the shared - NaMaster bandpower window, with this instance's (blinded) n(z) stamped - in. The in-memory ``self._pseudo_cls[ver]`` ``"pseudo_cl"`` entry keeps - the ``ELL``/``EE``/``EB``/``BB`` arrays the plotting and B-mode-summary - consumers read by column name. - ``out_path`` is the exact destination the part is born at (one version only); ``None`` defaults each part to ``pseudo_cl_{ver}.sacc``. - Skip-if-exists keys on this final path, so no two rules ever share an - undeclared native basename. """ self.print_start("Computing pseudo-Cl's") @@ -508,7 +499,7 @@ def calculate_pseudo_cl(self, out_path=None): @staticmethod def _load_pseudo_cl_sacc(out_path): - """Read a pseudo-Cl SACC part into the ELL/EE/EB/BB dict consumers use.""" + """Read a pseudo-Cl SACC part into the ELL/EE/EB/BB dict.""" # Readback of a part this producer just wrote — a legitimate pre-blind # consumer, so the fail-closed load is opted out of. s = sacc_io.load(out_path, allow_unblinded=True) @@ -691,8 +682,7 @@ def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); the writer takes the shared bandpower window from ``wsp``. No covariance - is attached here — the analysis file's pseudo-Cl block is supplied at - assembly (``assemble_sacc``) from the NaMaster / OneCovariance product. + is attached here. """ s = pseudo_cl_to_sacc( self.sacc_nz(version), diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index d3126af7..cf5fc7e6 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -140,10 +140,10 @@ def pure_eb_to_sacc_part(self, version, out_path, results, eb_override=None): ``results`` is the dict ``calculate_pure_eb`` returned: the six pure-mode arrays under ``sacc_io.PURE_KEYS``, the ``"cov"`` block (in ``PURE_KEYS`` order), and the reporting-grid TreeCorr object ``"gg"`` whose ``meanr`` - is the shared ``theta``. The covariance must cover every stored point. + is the shared ``theta``. - ``eb_override`` replaces the six arrays with ones derived from the - reporting + integration ξ± parts; the covariance stays from ``results``. + ``eb_override`` replaces the six arrays; the covariance stays from + ``results``. """ theta = results["gg"].meanr source = eb_override if eb_override is not None else results diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index e12ddbbe..3400991d 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -45,9 +45,7 @@ def calculate_2pcf(self, ver, npatch=None, **treecorr_config): calculation is skipped, and the results are loaded from the file. - If a patch file for the given configuration does not exist, it is created during the process. - - The ``.txt`` TreeCorr dump is the only raw byproduct written here - (read back by the covariance machinery and the skip-if-exists); the - ξ± data product is born as SACC in ``run_2pcf.py``. + - The ``.txt`` TreeCorr dump is the only raw byproduct written here. """ self.print_magenta(f"Computing {ver} ξ±") diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index de8e425c..6378d2e1 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -4,10 +4,8 @@ TreeCorr / NaMaster / b_modes arrays) and :mod:`sp_validation.sacc_io` (which knows the file layout). Each ``*_to_sacc`` function turns one already-computed statistic into a single-statistic SACC — a *part* — carrying that statistic's -own covariance as its one block. The Snakemake DAG writes one part per rule; -:func:`assemble_analysis_sacc` then rebuilds the single ``{version}.sacc`` -analysis file with a ``BlockDiagonalCovariance`` over the per-part blocks in -canonical order. +own covariance as its one block. :func:`assemble_analysis_sacc` rebuilds the +single ``{version}.sacc`` analysis file from these parts. Everything here is single-bin today (``bins=(0, 0)``); the interface is tomography-native so a future round supplies real bin pairs unchanged. @@ -26,7 +24,7 @@ # NaMaster spin-2 × spin-2 decoupled-spectrum row order (EE, EB, BE, BB). _NMT_EE, _NMT_EB, _NMT_BB = 0, 1, 3 -BIN = (0, 0) # single-bin default until the round goes tomographic +BIN = (0, 0) def xi_to_sacc( @@ -44,9 +42,8 @@ def xi_to_sacc( ): """One ξ± part (``bins=(0, 0)``) on the reporting or integration grid. - ``variances`` (the concatenated ``[varxip; varxim]``) attaches a - ``DiagonalCovariance`` — used for the integration-grid part, where npatch=1 - leaves TreeCorr shot-noise variance as the only covariance estimate. + ``variances`` is the concatenated ``[varxip; varxim]``; when given, attaches + a ``DiagonalCovariance``. """ s = sio.new_sacc(nz, metadata) sio.add_xi( @@ -94,10 +91,7 @@ def cosebis_to_sacc(nz, metadata, result, scale_cut): ``result`` is a single scale-cut result dict from ``b_modes.calculate_cosebis`` — ``{"En", "Bn", "cov", ...}`` — where ``cov`` - is the ``[En; Bn]``-ordered COSEBIs covariance. Non-fiducial scale cuts are - a diagnostic (the PTE scan) and stay in the sidecar ``.npz``; only the - fiducial cut is a data product, because the analysis covariance must cover - every stored point and the cuts overlap in mode space. + is the ``[En; Bn]``-ordered COSEBIs covariance. """ s = sio.new_sacc(nz, metadata) sio.add_cosebis(s, BIN, result["En"], scale_cut, Bn=result["Bn"]) @@ -124,12 +118,10 @@ def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov_th=None): ``rho_stats`` / ``tau_stats`` are the ``shear_psf_leakage`` handler tables (columns ``theta``, ``rho_{k}_p``, ``varrho_{k}_p``, … and the τ analogue). - ρ carries a ``varrho`` diagonal (a diagnostic, not consumed by inference); - τ carries a ``vartau`` diagonal with ``tau_cov_th`` scattered into the τ-plus - rows/columns. ``CovTauTh.build_cov`` returns a ``(3·nbin, 3·nbin)`` k-major - matrix over ``{τ0, τ2, τ5}`` with plus/minus folded into one component per k, - so it aligns to the τ-plus points in k-major order; the τ-minus points have - no theory covariance. ``tau_cov_th=None`` leaves the τ block fully diagonal. + ρ carries a ``varrho`` diagonal; τ carries a ``vartau`` diagonal, with + ``tau_cov_th`` — a ``(3·nbin, 3·nbin)`` k-major matrix over the τ-plus points + only — scattered into the τ-plus rows/columns when given. ``tau_cov_th=None`` + leaves the τ block fully diagonal. """ s = sio.new_sacc(nz, metadata) theta_rho = np.asarray(rho_stats["theta"]) @@ -165,7 +157,6 @@ def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov_th=None): ] ) if tau_cov_th is None: - # Compact DiagonalCovariance; assembly reads it back via .dense. s.add_covariance(np.concatenate([rho_var, tau_var])) return s tau_cov_th = np.asarray(tau_cov_th) diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index 798ffbd2..09926b88 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -319,11 +319,7 @@ def test_assemble_from_reloaded_parts(tmp_path): def test_cosebis_part_overrides_en_and_bn(tmp_path): - """En/Bn overrides reach the part; the covariance stays from ``results``. - - Pins the cv_cosebis provenance split: values come from the (blindable) - integration ξ± part, the jackknife covariance from the raw patched run. - """ + """En/Bn overrides reach the part; the covariance stays from ``results``.""" from sp_validation.cosmo_val.cosebis import CosebisMixin raw = { @@ -335,8 +331,7 @@ def test_cosebis_part_overrides_en_and_bn(tmp_path): en_part, bn_part = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 class _Stub(CosebisMixin): - # run_type / commitment_path are what the blinded-part writer reads; - # a mock part with no commitment keeps this test off the custody gate. + # A mock part with no commitment keeps this off the custody gate. run_type = "mock" sacc_nz = staticmethod(lambda version: {0: _nz()}) sacc_metadata = staticmethod(lambda version: META) diff --git a/workflow/common.py b/workflow/common.py index bc7d1c71..70003dcf 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -5,17 +5,12 @@ import re from pathlib import Path -# Absolute path to the generic workflow's scripts, for rules that shell out to a -# script directly rather than through Snakemake's `script:` directive. Anchored -# on this module's own location, not workflow.basedir — under `module` -# composition basedir reflects the composing paper, not the running checkout. -WORKFLOW_SCRIPTS = os.path.join(os.path.dirname(os.path.realpath(__file__)), "scripts") - -# The running checkout, for rules that shell out to a repo script directly -# rather than through Snakemake's `script:` directive. Anchored on this module's -# own location, not workflow.basedir — under `module` composition basedir -# reflects the composing paper, not the running checkout. +# The running checkout, for rules that shell out to a script directly rather +# than through Snakemake's `script:` directive. Anchored on this module's own +# location, not workflow.basedir — under `module` composition basedir reflects +# the composing paper, not the running checkout. REPO_ROOT = Path(os.path.realpath(__file__)).parents[1] +WORKFLOW_SCRIPTS = os.path.join(os.path.dirname(os.path.realpath(__file__)), "scripts") # Output roots are env-overridable so a reproduction run can write into a # fresh tree without clobbering (or silently reusing) prior products. @@ -190,11 +185,7 @@ def build_redshift_path(version, blind): def pseudo_cl_tag(config): - """Fiducial harmonic-binning tag the pseudo-Cl producers stamp into filenames. - - Single definition shared by the producer (twopoint.smk) and the consumers - (cosmo_val.smk, inference.smk), which reconstruct the name from config. - """ + """Fiducial harmonic-binning tag stamped into pseudo-Cl filenames.""" fiducial = config["harmonic"]["fiducial"] return f"blind={fiducial['blind']}_{fiducial['binning']}_nbins={fiducial['nbins']}" diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index ef426edd..fcd9a304 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -99,11 +99,7 @@ def cv_cosebis_npz(version): def cv_pseudo_cl_sacc(version): - """Untagged pseudo-Cl SACC part cv_pseudo_cl writes (B-mode diagnostic). - - The analysis file carries the tagged inference product instead (see - cv_pseudo_cl_analysis_sacc). - """ + """Untagged pseudo-Cl SACC part: the B-mode diagnostic, not a data product.""" return str(COSMO_VAL / f"pseudo_cl_{version}.sacc") @@ -111,7 +107,7 @@ _PSEUDO_CL_TAG = pseudo_cl_tag(config) def cv_pseudo_cl_analysis_sacc(version): - """Tagged pseudo-Cl SACC part the analysis file carries.""" + """Tagged pseudo-Cl SACC part: the harmonic block of the analysis file.""" return str(COSMO_VAL / f"pseudo_cl_{version}_{_PSEUDO_CL_TAG}.sacc") @@ -121,13 +117,7 @@ def cv_pseudo_cl_cov(version): def cv_xi_cov(version): - """CosmoCov-processed ξ± covariance for the reporting grid. - - The real analysis covariance for the ξ± block, produced by - covariance_process (covariance.smk) on the same binning the reporting grid - measures, so requesting it builds it. OneCovariance replaces CosmoCov - upstream of this path (#256) in the same format. - """ + """CosmoCov-processed ξ± covariance, on the reporting grid's own binning.""" return covariance_path( version, FIDUCIAL["blind"], @@ -140,33 +130,24 @@ def cv_xi_cov(version): def cv_cosebis_sacc(version): - """COSEBIs SACC part (fiducial scale cut) the cv_cosebis rule writes.""" + """COSEBIs SACC part, at the fiducial scale cut.""" return str(COSMO_VAL / f"{version}_cosebis.sacc") def cv_pure_eb_sacc(version): - """Pure-E/B SACC part the cv_pure_eb rule writes.""" + """Pure-E/B SACC part.""" return str(COSMO_VAL / f"{version}_pure_eb.sacc") def cv_rho_tau_sacc(version): - """ρ/τ SACC part calculate_rho_tau_stats writes (rho_tau_{base}.sacc).""" + """ρ/τ SACC part.""" return str( COSMO_VAL / "rho_tau_stats" / f"rho_tau_{cv_basename(version, CV_FIDUCIAL)}.sacc" ) def cv_xi_sacc(version, grid): - """ξ± SACC part the `xi` rule writes for a version on a named grid. - - Named by its binning (xi_binning, twopoint.smk), which is what binds the xi - job's wildcards; the rule resolves the grid label from that binning. - - grid='reporting' is the analysis part (its covariance is injected at - assembly); grid='integration' is the fine-grid part COSEBIs and pure-E/B - consume. The integration part stays standalone — it is not folded into the - terminal {version}.sacc. - """ + """ξ± SACC part for a version on a named grid, named by that grid's binning.""" return str(COSMO_VAL / f"{version}_xi_{xi_binning(grid)}.sacc") @@ -385,12 +366,7 @@ rule cv_pure_eb: rule cv_cosebis: - """COSEBIs E/B decomposition for one version (config-space, fine binning). - - Mixed provenance by design: En and Bn derive from the (blindable) - integration ξ± part, while the jackknife covariance needs the patched raw - measurement (patches exist only there). - """ + """COSEBIs E/B decomposition for one version (config-space, fine binning).""" input: xi=lambda w: cv_xi_txt(w.version), xi_integration=lambda w: cv_xi_sacc(w.version, "integration"), @@ -451,24 +427,16 @@ rule cv_summarize_bmodes: # --------------------------------------------------------------------------- # Terminal analysis file: assemble the per-statistic SACC parts into {version}.sacc # --------------------------------------------------------------------------- -# assemble_sacc.py loads the five parts in canonical order (xi_reporting, -# pseudo_cl, cosebis, pure_eb, rho_tau) and rebuilds one {version}.sacc with a -# single BlockDiagonalCovariance. The integration-grid ξ± is deliberately not -# gathered: it stays an intermediate consumed by COSEBIs/pure-E/B. -# -# The pseudo-Cℓ part is the tagged inference product, and its NaMaster covariance -# is injected here from the matching pseudo_cl_cov FITS. The ξ± reporting block -# has no real covariance wired yet — the CosmoCov theory .txt is blind/gaussian/ -# mask-keyed and lives deep in the inference tree, so sourcing it couples -# cosmo_val to the whole inference covariance DAG; it plugs in via --xi-cov. +# The terminal file carries the analysis vector only. The integration-grid ξ± is +# deliberately not gathered: it stays a per-part intermediate. The two blocks +# born without a covariance (ξ± reporting, pseudo-Cℓ) get theirs injected from +# the covariance inputs below. def cv_assemble_inputs(version): """The per-statistic SACC parts + covariance inputs assemble_sacc consumes. Each part's filename carries enough to bind its producing rule's wildcards. - pseudo_cl (+ its cov) is included only when the config toggles the - harmonic-space BB into the analysis. """ parts = dict( xi_reporting=cv_xi_sacc(version, "reporting"), @@ -491,11 +459,9 @@ rule assemble_sacc: sacc=cv_analysis_sacc("{version}"), params: version="{version}", - # Run type gates unblinded loading: a 'data' run fails closed on - # unblinded parts, a 'mock' run loads freely. type=CV.get("type", "data"), - # Statistics this rule wired; the script validates part_paths against it - # so a typo'd input keyword can't silently drop one. + # The statistics this rule wired, so a typo'd input keyword cannot + # silently drop one. expected=lambda w: [ k for k in cv_assemble_inputs(w.version) diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 76f321d0..37884e52 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -4,11 +4,9 @@ # ξ± angular grids # --------------------------------------------------------------------------- # A grid is a binning: (min_sep, max_sep, nbins, npatch). `reporting` is the -# analysis grid; `integration` is the fine grid COSEBIs and pure-E/B integrate -# over. Both are measured by the single `xi` rule below, whose files are named -# by binning, so the grid label is resolved from the wildcards rather than -# duplicated into a second rule. Workflows carrying no cosmo_val block (e.g. -# papers/bmodes) fall back to their fiducial grids. +# analysis grid, `integration` the fine one the B-mode integrals run over. +# Workflows carrying no cosmo_val block (e.g. papers/bmodes) fall back to their +# fiducial grids. def _xi_grids(): cv = config.get("cosmo_val", {}) reporting = ( @@ -63,10 +61,8 @@ def xi_grid_of(wildcards): rule xi: """TreeCorr ξ±(θ) for one version on one angular grid. - Binning-agnostic: the reporting and integration measurements are the same - job with different wildcards. The raw TreeCorr .txt byproduct and the - born-as-SACC part are both named by that binning, so a request for either - binds unambiguously; the grid label comes from XI_GRIDS. + One rule for every grid: outputs are named by their binning, so a request + binds the wildcards and `xi_grid_of` resolves the grid label from them. """ input: catalog=get_shear_catalog, @@ -114,8 +110,7 @@ rule rho_tau_stats: output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - # Born-as-SACC ρ/τ part the assemble_sacc rule consumes, written - # alongside the FITS by calculate_rho_tau_stats. + # Born-as-SACC ρ/τ part, written alongside the FITS. rho_tau=str(COSMO_VAL / "rho_tau_stats/rho_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 48 params: diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 410bc019..bc9049c4 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -3,17 +3,14 @@ Dual-mode: under Snakemake (``script:``) the injected ``snakemake`` object supplies the inputs; as a standalone CLI the same assembly runs from flags. -Each part is a single-statistic SACC written by the cosmo_val mixins, run_2pcf -or generate_pseudo_cl. Parts load in canonical order (ξ± reporting, pseudo-Cℓ, -COSEBIs, pure-E/B, ρ/τ) and are rebuilt into one Sacc with a single -``BlockDiagonalCovariance``. - -Every part must carry a covariance block. COSEBIs, pure-E/B and ρ/τ are born -with one; ξ± reporting and pseudo-Cℓ are not, so their blocks are injected here -from the CosmoCov ``.txt`` (``--xi-cov``) and the NaMaster covariance FITS -(``--pseudo-cl-cov``). The pseudo-Cℓ cross-spectrum blocks (EE↔BB, …) are -dropped — matching what the B-mode PTE reads today. Both are DAG inputs of the -assemble rule, so a missing one is a missing input, never a stand-in. +Each part is a single-statistic SACC; they load in CANONICAL order and are +rebuilt into one Sacc with a single ``BlockDiagonalCovariance``. + +Every part must carry a covariance block. ξ± reporting and pseudo-Cℓ are born +without one, so theirs are injected here from the CosmoCov ``.txt`` +(``--xi-cov``) and the NaMaster covariance FITS (``--pseudo-cl-cov``); the +pseudo-Cℓ cross-spectrum blocks (EE↔BB, …) are dropped, matching what the +B-mode PTE reads today. """ import argparse diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index 342550e0..b4e0cafd 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -1,13 +1,8 @@ """Rule cv_cosebis: COSEBIs E/B decomposition for one version. -Compute + plot rule (per version). plot_cosebis calls calculate_cosebis over a -fine integration binning (the 2000-bin TreeCorr is the dominant cost) and -evaluates the configured scale cuts. Writes the {version}_eb_..._data.npz -COSEBIs data product plus figures, and the per-version COSEBIs PTE that -cv_summarize_bmodes collects. It also writes the born-as-SACC COSEBIs part -({version}_cosebis.sacc, the fiducial scale cut's {En,Bn,cov}) that the -assemble_sacc rule consumes — the multi-cut .npz sidecar stays the diagnostic -PTE scan. +plot_cosebis calls calculate_cosebis over the fine integration binning and +evaluates the configured scale cuts. Writes the SACC part at the fiducial cut, +plus the multi-cut .npz sidecar (the diagnostic PTE scan) and figures. """ import numpy as np @@ -41,7 +36,6 @@ theta, xip, xim, p["nmodes"], scale_cut=fiducial_scale_cut ) -# Born-as-SACC COSEBIs part at the fiducial scale cut. cv.cosebis_to_sacc_part( version, snakemake.output["sacc"], diff --git a/workflow/scripts/cv_pseudo_cl.py b/workflow/scripts/cv_pseudo_cl.py index 3f6374bf..22e01621 100644 --- a/workflow/scripts/cv_pseudo_cl.py +++ b/workflow/scripts/cv_pseudo_cl.py @@ -1,9 +1,7 @@ """Rule cv_pseudo_cl: harmonic-space pseudo-Cl B-mode spectra. -plot_pseudo_cl triggers calculate_pseudo_cl, which writes the born-as-SACC -pseudo_cl_{version}.sacc part for every version (EE/BB/EB with the shared -bandpower window) and the cell_ee.png figure. The per-version SACC parts are -the declared outputs, read by cv_summarize_bmodes. +plot_pseudo_cl triggers calculate_pseudo_cl, which writes one SACC part per +version (EE/BB/EB with the shared bandpower window) and the cell_ee.png figure. """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index bff39ecb..a6e730b0 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -1,12 +1,8 @@ """Rule cv_pure_eb: pure E/B-mode decomposition for one version. -Compute + plot rule (per version). plot_pure_eb calls calculate_pure_eb, which -runs two TreeCorr correlations (reporting + integration binning); the reporting -binning reuses the cv_2pcf data vector via calculate_2pcf's skip-if-exists -path. Writes the {version}_eb_..._data.npz data product plus companion figures, -and the per-version E/B PTEs that cv_summarize_bmodes collects. It also writes -the born-as-SACC pure-E/B part ({version}_pure_eb.sacc, the six PURE_KEYS blocks -+ covariance) that the assemble_sacc rule consumes. +plot_pure_eb calls calculate_pure_eb, which runs two TreeCorr correlations +(reporting + integration binning). Writes the SACC part, the .npz data product +and the companion figures. """ import numpy as np @@ -40,7 +36,6 @@ ti, xpi, xmi = sacc_io.get_xi(integ, (0, 0), grid="integration") modes = pure_eb_from_xi(tr, xpr, xmr, ti, xpi, xmi, tmin, tmax) -# Born-as-SACC pure-E/B part; the six blocks come from the consumed parts. cv.pure_eb_to_sacc_part(version, snakemake.output["sacc"], results, eb_override=modes) # Sync the npz's pure modes with the part-derived values; theta / cov / PTE untouched. diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index 1299a36d..e25f6ed9 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -1,14 +1,11 @@ """Generate pseudo-Cls (data vector only, no covariance). Dual-mode. Under Snakemake (``script:`` directive) the injected ``snakemake`` -object supplies the parameters and the native product is renamed to the tagged -output filename the rule declares; as a standalone CLI (argparse) the same -compute runs from explicit flags and the primitive's native -``pseudo_cl_{ver}.sacc`` is left in place under ``--out`` (each lc/ASTRA recipe -gets its own output directory, so the untagged name is unambiguous). The C_ell -data vector is born as SACC (EE/BB/EB with a shared bandpower window). The CLI -form is what the lightcone/ASTRA recipe calls, so the measurement is driven -directly (no nested Snakemake) with lc handling orchestration: +object supplies the parameters; as a standalone CLI (argparse) the same compute +runs from explicit flags. Either way the part is born at its final path, as +SACC (EE/BB/EB with a shared bandpower window). The CLI form is what the +lightcone/ASTRA recipe calls, driving the measurement directly (no nested +Snakemake) with lc handling orchestration: python generate_pseudo_cl.py \ --ver SP_v1.4.6.3_leak_corr \ @@ -147,7 +144,6 @@ def generate_pseudo_cl( def _from_snakemake(smk): p = smk.params - # Born directly at the rule's declared (tagged) output — no rename step. generate_pseudo_cl( version=p["version"], out_path=smk.output.pseudo_cl, diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 19f50682..9480b4a9 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -16,10 +16,9 @@ grids are the same compute with different ``--min-sep/--max-sep/--nbins``. ``CosmologyValidation.calculate_2pcf`` writes the ``.txt`` dump (a raw byproduct); the ξ± data product is born as SACC here, a *part* named by its -binning and tagged with its ``--grid``. The reporting part carries no covariance -(its block is supplied at assembly from CosmoCov); the integration part carries -a ``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim``, the only estimate -available at npatch=1, which is what COSEBIs and pure-E/B consume. +binning and tagged with its ``--grid``. The reporting part carries no +covariance; the integration part carries a ``DiagonalCovariance`` from +TreeCorr ``varxip``/``varxim``, the only estimate available at npatch=1. ``output_dir`` is passed explicitly (rather than via the ``COSMO_VAL`` env hook) so lc can point each run at its own ``{output}`` tree. @@ -53,9 +52,9 @@ def run_2pcf( patches (1 for the paper fiducial). ``cat_config`` is an absolute path to the catalog configuration; ``output_dir`` overrides ``cat_config['paths']['output']`` so the ``.txt`` byproduct lands where lc - expects. ``sacc_out`` is the exact destination for the reporting ξ± SACC part - (the Snakemake-declared output); it defaults to ``{ver}_xi_reporting.sacc`` - under the resolved output directory for the CLI path. + expects. ``sacc_out`` is the exact destination for the SACC part (the + Snakemake-declared output); it defaults to a binning-derived name under + the resolved output directory for the CLI path. Returns ------- @@ -115,11 +114,9 @@ def _from_snakemake(smk): # class defaults (./cat_config.yaml, COSMO_VAL env) otherwise. cat_config=p.get("cat_config", "./cat_config.yaml"), output_dir=p.get("output_dir", None), - # Grid label is resolved by the rule from the binning wildcards - # (workflow/rules/twopoint.smk XI_GRIDS). grid=p.get("grid", "reporting"), - # Write the SACC part exactly where the rule declares it (the .txt - # byproduct still lands under the resolved output dir via _output_path). + # The SACC part goes exactly where the rule declares it; the .txt + # byproduct still lands under the resolved output dir. sacc_out=smk.output["sacc"], ) diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index bda566f8..71e3deb7 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -49,7 +49,7 @@ cv.calculate_rho_tau_stats() # Confirm CosmologyValidation produced the requested outputs: the rho/tau FITS -# and the born-as-SACC rho_tau part the assemble_sacc rule consumes. +# and the born-as-SACC rho_tau part. outputs = snakemake.output # type: ignore for label in ("rho_stats", "tau_stats", "rho_tau"): target = Path(outputs[label]) From f56bedebc5028257d040f296d1660941b90a9b4d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 03:42:13 +0200 Subject: [PATCH 055/160] =?UTF-8?q?Each=20grid=20carries=20its=20own=20cov?= =?UTF-8?q?ariance;=20COSEBIs=20derives=20from=20=CE=BE=C2=B1=20arrays?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A grid is now a binning plus how its covariance is estimated, so one TreeCorr run per grid produces a part complete enough to work from: the dense jackknife block where there are patches, the varxip/varxim diagonal, or nothing. The COSEBIs scan gains an arrays-and-covariance seam (cosebis_scan_from_xi): the COSEBIs covariance was already the ξ± covariance through the same linear kernel as the modes, so a caller holding a data vector needs no estimator re-run — only the realisation count behind the covariance, for Hartlap. calculate_cosebis delegates to it, and the scale-cut heatmap takes bin edges rather than a correlation object; log_bin_edges reconstructs those edges from a binning, which is what a part-based consumer has. cv_cosebis becomes such a consumer: one part in, and the SACC part, the .npz scan and the figures all out of those values. The en/bn_override machinery it needed while values and covariance came from different runs goes with it. --- src/sp_validation/b_modes.py | 122 +++++++++++++------ src/sp_validation/cosmo_val/cosebis.py | 60 +-------- src/sp_validation/cosmo_val/sacc_writers.py | 14 ++- src/sp_validation/tests/test_b_modes.py | 107 ++++++++++++++++ src/sp_validation/tests/test_sacc_writers.py | 64 +++++----- workflow/rules/twopoint.smk | 28 ++++- workflow/scripts/cv_cosebis.py | 82 ++++++++----- workflow/scripts/run_2pcf.py | 27 ++-- 8 files changed, 320 insertions(+), 184 deletions(-) diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index f05c84a4..dd5fcf3c 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -74,23 +74,34 @@ def scale_cut_to_bins(gg, min_scale=None, max_scale=None): stop_bin : int Last included bin index + 1 (for slicing notation) """ - nbins = len(gg.meanr) - - if min_scale is not None: - # Conservative: exclude bins whose left edge is below min_scale - start_bin = np.searchsorted(gg.left_edges, min_scale, side="left") - else: - start_bin = 0 + return bins_from_edges(gg.left_edges, gg.right_edges, min_scale, max_scale) - if max_scale is not None: - # Conservative: exclude bins whose right edge is above max_scale - stop_bin = np.searchsorted(gg.right_edges, max_scale, side="right") - else: - stop_bin = nbins +def bins_from_edges(left_edges, right_edges, min_scale=None, max_scale=None): + """:func:`scale_cut_to_bins` on bare bin edges.""" + start_bin = ( + np.searchsorted(left_edges, min_scale, side="left") + if min_scale is not None + else 0 + ) + stop_bin = ( + np.searchsorted(right_edges, max_scale, side="right") + if max_scale is not None + else len(right_edges) + ) return start_bin, stop_bin +def log_bin_edges(min_sep, max_sep, nbins): + """TreeCorr ``Log`` bin edges — the grid a binning defines. + + SACC ξ± parts store bin centres, not edges, so a consumer working from a + part reconstructs the edges from the binning it was measured on. + """ + edges = np.geomspace(float(min_sep), float(max_sep), int(nbins) + 1) + return edges[:-1], edges[1:] + + def correlation_from_covariance(covariance): """ Convert covariance matrix to correlation matrix. @@ -338,23 +349,52 @@ def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): Each results dictionary contains 'En', 'Bn', 'cov', 'chi2_E', 'chi2_B', 'pte_B', 'scale_cut', and 'mask' entries. """ - from cosmo_numba.B_modes.cosebis import COSEBIS + cov_xipm = np.loadtxt(cov_path) if cov_path is not None else gg.cov + return cosebis_scan_from_xi( + gg.meanr, + gg.xip, + gg.xim, + cov_xipm, + gg.left_edges, + gg.right_edges, + nmodes=nmodes, + scale_cuts=scale_cuts, + # A theory covariance has no jackknife realisations to debias. + npatch=None if cov_path is not None else gg.npatch1, + ) - # Default to full range if no scale cuts provided - if scale_cuts is None: - scale_cuts = [(gg.left_edges[0], gg.right_edges[-1])] - # Pre-compute values that don't change across scale cuts - nbins = len(gg.meanr) +def cosebis_scan_from_xi( + theta, + xip, + xim, + cov_xipm, + left_edges, + right_edges, + *, + nmodes=10, + scale_cuts=None, + npatch=None, +): + """COSEBIs over a set of scale cuts, from ξ± arrays and their covariance. - # Load covariance matrix and calculate Hartlap factor once - if cov_path is not None: - print(f"Loading theoretical covariance from {cov_path}") - cov_xipm = np.loadtxt(cov_path) - hartlap_factor = 1 # Not defined for analytic covariances - else: - cov_xipm = gg.cov - hartlap_factor = (gg.npatch1 - 2 * nmodes - 2) / (gg.npatch1 - 1) + The values-and-covariance seam of :func:`calculate_cosebis`, for callers + holding a ξ± data vector rather than a TreeCorr ``GGCorrelation``. The + COSEBIs covariance is the ξ± covariance carried through the same linear + kernel as the modes, so no estimator re-run is involved. ``npatch`` is the + jackknife realisation count behind ``cov_xipm``, which sets the Hartlap + debiasing; leave it ``None`` for a theory covariance, which needs none. + + Returns the ``{scale_cut: result}`` mapping :func:`calculate_cosebis` returns. + """ + from cosmo_numba.B_modes.cosebis import COSEBIS + + theta, xip, xim = (np.asarray(a) for a in (theta, xip, xim)) + cov_xipm = np.asarray(cov_xipm) + if scale_cuts is None: + scale_cuts = [(left_edges[0], right_edges[-1])] + nbins = len(theta) + hartlap_factor = 1 if npatch is None else (npatch - 2 * nmodes - 2) / (npatch - 1) all_results = {} @@ -362,11 +402,12 @@ def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): for scale_cut in tqdm.tqdm(scale_cuts, desc="COSEBIs scale cuts"): min_theta, max_theta = scale_cut - # Apply scale cuts using scale_cut_to_bins for consistency - start_bin, stop_bin = scale_cut_to_bins(gg, min_theta, max_theta) + start_bin, stop_bin = bins_from_edges( + left_edges, right_edges, min_theta, max_theta + ) inds = np.arange(start_bin, stop_bin) - theta_cut, xip_cut, xim_cut = [arr[inds] for arr in [gg.meanr, gg.xip, gg.xim]] + theta_cut, xip_cut, xim_cut = [arr[inds] for arr in [theta, xip, xim]] # Calculate COSEBIs E/B modes using actual theta range (per Axel's recommendation) # Use precision=120 (vs default 80) to avoid sympy root convergence failures @@ -819,7 +860,7 @@ def plot_pure_eb_correlations( def plot_cosebis_scale_cut_heatmap( - cosebis_results, gg, version, output_path, fiducial_scale_cut=None + cosebis_results, edges, version, output_path, fiducial_scale_cut=None ): """ Create 2D heatmaps showing how COSEBIs statistics vary across different scale cuts. @@ -828,8 +869,8 @@ def plot_cosebis_scale_cut_heatmap( ---------- cosebis_results : dict Dictionary with scale cut tuples as keys, containing 'chi2_E' and 'pte_B' values - gg : treecorr.GGCorrelation - Correlation function object for bin edges + edges : tuple of numpy.ndarray + ``(left_edges, right_edges)`` of the grid the scale cuts index version : str Version string for main title output_path : str @@ -837,7 +878,8 @@ def plot_cosebis_scale_cut_heatmap( fiducial_scale_cut : tuple, optional (min_scale, max_scale) for cross-hatching """ - nbins = gg.nbins + left_edges, right_edges = edges + nbins = len(left_edges) # Initialize matrices snrs = [np.sqrt(result["chi2_E"]) for result in cosebis_results.values()] @@ -853,9 +895,9 @@ def plot_cosebis_scale_cut_heatmap( pte_matrix[row, column] = pte # Fill matrices from COSEBIs results - for i in range(len(gg.left_edges)): - for j in range(i, len(gg.right_edges)): - scale_cut = (gg.left_edges[i], gg.right_edges[j]) + for i in range(len(left_edges)): + for j in range(i, len(right_edges)): + scale_cut = (left_edges[i], right_edges[j]) result = cosebis_results.get(scale_cut) if result is not None: @@ -916,7 +958,9 @@ def plot_cosebis_scale_cut_heatmap( # Add fiducial scale cut cross-hatching if provided if fiducial_scale_cut is not None: min_scale, max_scale = fiducial_scale_cut - start_bin, stop_bin = scale_cut_to_bins(gg, min_scale, max_scale) + start_bin, stop_bin = bins_from_edges( + left_edges, right_edges, min_scale, max_scale + ) if stop_bin > start_bin and start_bin < nbins and stop_bin > 0: # Add cross-hatching at the fiducial scale cut matrix element @@ -943,10 +987,10 @@ def plot_cosebis_scale_cut_heatmap( # Set angular scale ticks tick_indices = np.arange(0, nbins) x_tick_labels = [ - f"{gg.left_edges[i]:.1f}" for i in tick_indices if i < len(gg.left_edges) + f"{left_edges[i]:.1f}" for i in tick_indices if i < len(left_edges) ] y_tick_labels = [ - f"{gg.right_edges[i]:.1f}" for i in tick_indices if i < len(gg.right_edges) + f"{right_edges[i]:.1f}" for i in tick_indices if i < len(right_edges) ] x_tick_positions = tick_indices + 0.5 y_tick_positions = tick_indices + 0.5 diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index 53a1dd48..312aadcd 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -7,7 +7,6 @@ import numpy as np -from .. import sacc_io from ..b_modes import ( calculate_cosebis, find_conservative_scale_cut_key, @@ -16,7 +15,6 @@ plot_cosebis_scale_cut_heatmap, save_cosebis_results, ) -from .sacc_writers import cosebis_to_sacc class CosebisMixin: @@ -142,62 +140,6 @@ def calculate_cosebis( return results - @staticmethod - def _fiducial_cosebis_result(results, fiducial_scale_cut): - """Select the fiducial scale cut's result dict + its ``(min, max)`` cut. - - ``results`` is a single result dict (full range) or a multi-cut mapping - keyed by ``(theta_min, theta_max)`` tuples; picks via - ``find_conservative_scale_cut_key`` when ``fiducial_scale_cut`` is given, - else the widest cut. - """ - multi_cut = isinstance(results, dict) and all( - isinstance(k, tuple) for k in results - ) - if not multi_cut: - return results, tuple(results["scale_cut"]) - key = ( - find_conservative_scale_cut_key(results, fiducial_scale_cut) - if fiducial_scale_cut is not None - else max(results, key=lambda x: x[1] - x[0]) - ) - return results[key], tuple(key) - - def cosebis_to_sacc_part( - self, - version, - out_path, - results, - fiducial_scale_cut=None, - en_override=None, - bn_override=None, - ): - """Write the COSEBIs SACC part at the fiducial scale cut. - - ``results`` is what ``calculate_cosebis`` returned (single dict or - multi-cut mapping); only the fiducial cut's ``{En, Bn, cov}`` becomes - the part. - - ``en_override``/``bn_override`` replace ``result["En"]``/``result["Bn"]``; - the covariance stays from ``result`` (patches exist only in the raw - patched measurement). - """ - result, scale_cut = self._fiducial_cosebis_result(results, fiducial_scale_cut) - overrides = { - key: np.asarray(value) - for key, value in (("En", en_override), ("Bn", bn_override)) - if value is not None - } - if overrides: - result = {**result, **overrides} - s = cosebis_to_sacc( - self.sacc_nz(version), - self.sacc_metadata(version), - result, - scale_cut, - ) - sacc_io.save(s, out_path, type="data") - def plot_cosebis( self, version=None, @@ -329,7 +271,7 @@ def plot_cosebis( plot_cosebis_scale_cut_heatmap( results, - gg_temp, + (gg_temp.left_edges, gg_temp.right_edges), version, out_stub + "_scalecut_ptes.png", fiducial_scale_cut=fiducial_scale_cut, diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index 6378d2e1..8b7f57b7 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -39,11 +39,13 @@ def xi_to_sacc( npairs=None, weight=None, variances=None, + covariance=None, ): - """One ξ± part (``bins=(0, 0)``) on the reporting or integration grid. + """One ξ± part (``bins=(0, 0)``) on a named angular grid. - ``variances`` is the concatenated ``[varxip; varxim]``; when given, attaches - a ``DiagonalCovariance``. + The grid's covariance comes in one of two shapes: ``covariance``, the dense + ``[ξ+; ξ−]``-ordered block (a jackknife estimate), or ``variances``, the + concatenated ``[varxip; varxim]`` diagonal. At most one may be given. """ s = sio.new_sacc(nz, metadata) sio.add_xi( @@ -57,7 +59,11 @@ def xi_to_sacc( npairs=npairs, weight=weight, ) - if variances is not None: + if covariance is not None and variances is not None: + raise ValueError("give xi_to_sacc a dense covariance or variances, not both") + if covariance is not None: + s.add_covariance(np.asarray(covariance)) + elif variances is not None: sio.add_diagonal_covariance(s, np.asarray(variances)) return s diff --git a/src/sp_validation/tests/test_b_modes.py b/src/sp_validation/tests/test_b_modes.py index e8e05c39..62bd11e1 100644 --- a/src/sp_validation/tests/test_b_modes.py +++ b/src/sp_validation/tests/test_b_modes.py @@ -285,3 +285,110 @@ def test_calculate_eb_statistics_has_teeth(): loud_pte = pm_loud[key][0, nbins - 1] assert loud_pte < quiet_pte assert loud_pte < 0.05 # louder B-modes are clearly rejected + + +# --------------------------------------------------------------------------- +# 6. Grid edges and the COSEBIs covariance seam +# --------------------------------------------------------------------------- + + +def test_log_bin_edges_matches_the_grid_stub(): + """Edges reconstructed from a binning are the ones TreeCorr would report. + + A part stores bin centres only, so a consumer rebuilds the edges from the + binning it was measured on; the two must agree bin for bin. + """ + left, right = b_modes.log_bin_edges(1.0, 100.0, _NBINS_GRID) + gg = _grid_gg() + npt.assert_allclose(left, gg.left_edges) + npt.assert_allclose(right, gg.right_edges) + # ...and they index scale cuts identically. + assert b_modes.bins_from_edges(left, right, 2.0, 50.0) == (2, 8) + + +def test_cosebis_scan_propagates_the_supplied_covariance(monkeypatch): + """The COSEBIs covariance is the ξ± covariance through the same kernel. + + The kernel is stubbed, so what is pinned is the seam: which ξ± covariance + sub-block is handed to the transform (the scale cut's, in [ξ+; ξ−] order) + and that Hartlap uses the supplied npatch. + """ + nbins, nmodes = _NBINS_GRID, 3 + theta = np.geomspace(1.2, 90.0, nbins) + cov_xipm = np.diag(np.arange(1.0, 2 * nbins + 1)) + seen = {} + + class _StubCOSEBIS: + def __init__(self, **kwargs): + seen["init"] = kwargs + + def cosebis_from_xipm(self, theta_cut, xip_cut, xim_cut, parallel=True): + seen["n_theta"] = len(theta_cut) + return np.ones(nmodes), np.full(nmodes, 2.0) + + def cosebis_covariance_from_xipm_covariance(self, theta_cut, cov_cut): + seen["cov_cut"] = cov_cut + return np.eye(2 * nmodes) + + module = types.ModuleType("cosmo_numba.B_modes.cosebis") + module.COSEBIS = _StubCOSEBIS + monkeypatch.setitem( + __import__("sys").modules, "cosmo_numba.B_modes.cosebis", module + ) + + left, right = b_modes.log_bin_edges(1.0, 100.0, nbins) + results = b_modes.cosebis_scan_from_xi( + theta, + np.arange(nbins) * 1e-5, + np.arange(nbins) * 2e-5, + cov_xipm, + left, + right, + nmodes=nmodes, + scale_cuts=[(2.0, 50.0)], + npatch=100, + ) + + (result,) = results.values() + # The cut is bins 2..8, so the covariance sub-block is those rows/cols in + # both the ξ+ and the ξ− half. + inds = np.concatenate([np.arange(2, 8), np.arange(2, 8) + nbins]) + npt.assert_array_equal(seen["cov_cut"], cov_xipm[np.ix_(inds, inds)]) + assert seen["n_theta"] == 6 + npt.assert_allclose(result["hartlap_factor"], (100 - 2 * nmodes - 2) / 99) + # χ² carries the Hartlap factor: modes are 1, cov is the identity. + npt.assert_allclose(result["chi2_E"], nmodes * result["hartlap_factor"]) + + +def test_cosebis_scan_theory_covariance_skips_hartlap(monkeypatch): + """A theory covariance has no realisations to debias, so Hartlap is 1.""" + nbins, nmodes = _NBINS_GRID, 2 + + class _StubCOSEBIS: + def __init__(self, **kwargs): + pass + + def cosebis_from_xipm(self, theta_cut, xip_cut, xim_cut, parallel=True): + return np.ones(nmodes), np.ones(nmodes) + + def cosebis_covariance_from_xipm_covariance(self, theta_cut, cov_cut): + return np.eye(2 * nmodes) + + module = types.ModuleType("cosmo_numba.B_modes.cosebis") + module.COSEBIS = _StubCOSEBIS + monkeypatch.setitem( + __import__("sys").modules, "cosmo_numba.B_modes.cosebis", module + ) + + left, right = b_modes.log_bin_edges(1.0, 100.0, nbins) + (result,) = b_modes.cosebis_scan_from_xi( + np.geomspace(1.2, 90.0, nbins), + np.zeros(nbins), + np.zeros(nbins), + np.eye(2 * nbins), + left, + right, + nmodes=nmodes, + npatch=None, + ).values() + assert result["hartlap_factor"] == 1 diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index 09926b88..59ea333d 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -318,38 +318,34 @@ def test_assemble_from_reloaded_parts(tmp_path): assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) -def test_cosebis_part_overrides_en_and_bn(tmp_path): - """En/Bn overrides reach the part; the covariance stays from ``results``.""" - from sp_validation.cosmo_val.cosebis import CosebisMixin - - raw = { - "En": np.zeros(10), - "Bn": np.zeros(10), - "cov": _spd(20, 11), - "scale_cut": (1.0, 100.0), - } - en_part, bn_part = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 - - class _Stub(CosebisMixin): - # A mock part with no commitment keeps this off the custody gate. - run_type = "mock" - sacc_nz = staticmethod(lambda version: {0: _nz()}) - sacc_metadata = staticmethod(lambda version: META) - commitment_path = staticmethod(lambda version: None) - - out = tmp_path / "part.sacc" - _Stub().cosebis_to_sacc_part( - "vSYNTH", - str(out), - raw, - en_override=en_part, - bn_override=bn_part, +def test_xi_part_carries_a_dense_covariance(tmp_path): + """A grid measured with patches puts its jackknife block in the part.""" + theta = _theta() + cov = _spd(2 * len(theta), 41) + s = sw.xi_to_sacc( + {0: _nz()}, + META, + theta, + np.arange(6) * 1e-5, + np.arange(6) * 2e-5, + grid="cosebis", + covariance=cov, ) - s = sio.load(str(out), allow_unblinded=True) - n, E, B = sio.get_cosebis(s, (0, 0)) - assert np.array_equal(n, np.arange(1, 11)) - assert np.array_equal(E, en_part) and np.array_equal(B, bn_part) - assert np.array_equal(s.covariance.dense, raw["cov"]) - # the caller's results dict is not mutated - assert np.array_equal(raw["En"], np.zeros(10)) - assert np.array_equal(raw["Bn"], np.zeros(10)) + s2 = _roundtrip(s, tmp_path, "xi_cov") + assert np.allclose(s2.covariance.dense, cov) + + +def test_xi_part_rejects_two_covariances(): + """Dense block and variances are alternatives, not a merge.""" + theta = _theta() + with pytest.raises(ValueError, match="not both"): + sw.xi_to_sacc( + {0: _nz()}, + META, + theta, + np.zeros(6), + np.zeros(6), + grid="reporting", + covariance=_spd(12, 42), + variances=np.ones(12), + ) diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 37884e52..6d886a05 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -3,10 +3,12 @@ # --------------------------------------------------------------------------- # ξ± angular grids # --------------------------------------------------------------------------- -# A grid is a binning: (min_sep, max_sep, nbins, npatch). `reporting` is the -# analysis grid, `integration` the fine one the B-mode integrals run over. -# Workflows carrying no cosmo_val block (e.g. papers/bmodes) fall back to their -# fiducial grids. +# A grid is a binning plus how its covariance is estimated: (min_sep, max_sep, +# nbins, npatch, cov). `reporting` is the analysis grid, `integration` the fine +# one the B-mode integrals run over, `cosebis` the fine patched grid COSEBIs +# propagates its covariance from. cov is "jackknife" (dense, from the patches), +# "diagonal" (TreeCorr varxip/varxim) or "none". Workflows carrying no cosmo_val +# block (e.g. papers/bmodes) fall back to their fiducial grids. def _xi_grids(): cv = config.get("cosmo_val", {}) reporting = ( @@ -28,11 +30,24 @@ def _xi_grids(): } ) integration.setdefault("npatch", 1) - return {"reporting": reporting, "integration": integration} + grids = {"reporting": reporting, "integration": integration} + cb = cv.get("cosebis") + if cb: + grids["cosebis"] = { + "min_sep": cb["min_sep_int"], + "max_sep": cb["max_sep_int"], + "nbins": cb["nbins_int"], + "npatch": cb["npatch"], + } + for grid in grids.values(): + # A jackknife estimate needs patches; at npatch=1 TreeCorr's var_method + # is "shot" and the diagonal is all it can offer. + grid.setdefault("cov", "jackknife" if int(grid["npatch"]) > 1 else "none") + return grids XI_GRIDS = _xi_grids() -XI_KEYS = ("min_sep", "max_sep", "nbins", "npatch") +XI_KEYS = ("min_sep", "max_sep", "nbins", "npatch") # the binning; `cov` is not part of the name def xi_binning(grid): @@ -78,6 +93,7 @@ rule xi: npatch="{npatch}", cat_config=CAT_CONFIG, grid=lambda w: xi_grid_of(w), + cov=lambda w: XI_GRIDS[xi_grid_of(w)]["cov"], resources: # The fine integration grid needs more memory and wall time than the # ~20-bin reporting one; scale on nbins rather than splitting the rule. diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index b4e0cafd..170341a3 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -1,55 +1,71 @@ """Rule cv_cosebis: COSEBIs E/B decomposition for one version. -plot_cosebis calls calculate_cosebis over the fine integration binning and -evaluates the configured scale cuts. Writes the SACC part at the fiducial cut, -plus the multi-cut .npz sidecar (the diagnostic PTE scan) and figures. +A consumer of the ξ± part alone — values, covariance, PTEs and figures all +derive from it, so nothing here touches a catalogue. The part's ξ± covariance +goes through the same linear kernel as the modes to give the COSEBIs +covariance; its ``npatch`` metadata sets the Hartlap debiasing. """ -import numpy as np -from cv_runner import _unbuffer_streams, make_cv, verify_outputs +from cv_runner import _unbuffer_streams, verify_outputs from snakemake.script import snakemake +from sp_validation import sacc_io +from sp_validation.b_modes import ( + cosebis_scan_from_xi, + find_conservative_scale_cut_key, + log_bin_edges, + plot_cosebis_covariance_matrix, + plot_cosebis_modes, + plot_cosebis_scale_cut_heatmap, + save_cosebis_results, +) +from sp_validation.cosmo_val.sacc_writers import cosebis_to_sacc + _unbuffer_streams() -cv = make_cv(snakemake) p = snakemake.params version = p["version"] fiducial_scale_cut = tuple(p["fiducial_scale_cut"]) -cv.plot_cosebis( - version=version, - min_sep_int=p["min_sep_int"], - max_sep_int=p["max_sep_int"], - nbins_int=p["nbins_int"], - npatch=p["npatch"], + +part = sacc_io.load(snakemake.input["xi"]) +theta, xip, xim = sacc_io.get_xi(part, (0, 0), grid="cosebis") +edges = log_bin_edges(p["min_sep"], p["max_sep"], p["nbins"]) + +results = cosebis_scan_from_xi( + theta, + xip, + xim, + part.covariance.dense, + *edges, nmodes=p["nmodes"], scale_cuts=[tuple(sc) for sc in p["scale_cuts"]], - fiducial_scale_cut=fiducial_scale_cut, + npatch=part.metadata["npatch"], ) -# Re-derive En and Bn from the integration ξ± part via the same kernel; only the -# jackknife covariance stays from the raw patched measurement. -from sp_validation import sacc_io -from sp_validation.b_modes import cosebis_from_xi +fiducial_key = find_conservative_scale_cut_key(results, fiducial_scale_cut) +fiducial = results[fiducial_key] -integ = sacc_io.load(snakemake.input["xi_integration"]) -theta, xip, xim = sacc_io.get_xi(integ, (0, 0), grid="integration") -en_part, bn_part = cosebis_from_xi( - theta, xip, xim, p["nmodes"], scale_cut=fiducial_scale_cut +plot_cosebis_modes( + fiducial, + version, + snakemake.output["figure_modes"], + fiducial_scale_cut=fiducial_scale_cut, ) - -cv.cosebis_to_sacc_part( +plot_cosebis_covariance_matrix( + fiducial, version, "jackknife", snakemake.output["figure_covariance"] +) +plot_cosebis_scale_cut_heatmap( + results, + edges, version, - snakemake.output["sacc"], - cv._cosebis_results[version], + snakemake.output["figure_scalecut_ptes"], fiducial_scale_cut=fiducial_scale_cut, - en_override=en_part, - bn_override=bn_part, ) -# Sync the npz's En/Bn with the part-derived values; cov / PTE untouched. -npz_path = snakemake.output["npz"] -data = dict(np.load(npz_path, allow_pickle=True)) -data["En"] = np.asarray(en_part) -data["Bn"] = np.asarray(bn_part) -np.savez(npz_path, **data) +save_cosebis_results(results, snakemake.output["npz"], fiducial_scale_cut) + +# The part inherits the ξ± part's provenance; `type` is re-stamped on save. +metadata = {k: v for k, v in part.metadata.items() if k != "type"} +s = cosebis_to_sacc({0: sacc_io.get_nz(part, 0)}, metadata, fiducial, fiducial_key) +sacc_io.save(s, snakemake.output["sacc"], type="data") verify_outputs(snakemake) diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 9480b4a9..78281c73 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -16,9 +16,9 @@ grids are the same compute with different ``--min-sep/--max-sep/--nbins``. ``CosmologyValidation.calculate_2pcf`` writes the ``.txt`` dump (a raw byproduct); the ξ± data product is born as SACC here, a *part* named by its -binning and tagged with its ``--grid``. The reporting part carries no -covariance; the integration part carries a ``DiagonalCovariance`` from -TreeCorr ``varxip``/``varxim``, the only estimate available at npatch=1. +binning and tagged with its ``--grid``. The part carries the covariance its +grid configures (``--cov``): the dense jackknife estimate from the patches, the +TreeCorr ``varxip``/``varxim`` diagonal, or none. ``output_dir`` is passed explicitly (rather than via the ``COSMO_VAL`` env hook) so lc can point each run at its own ``{output}`` tree. @@ -44,6 +44,7 @@ def run_2pcf( output_dir, sacc_out=None, grid="reporting", + cov="none", ): """Measure ξ±(θ) for ``ver`` and write its reporting SACC part. @@ -76,6 +77,9 @@ def run_2pcf( nbins=nbins, ) + if cov == "jackknife" and int(npatch) < 2: + raise ValueError(f"cov='jackknife' needs patches; got npatch={npatch}") + # Born-as-SACC ξ± part. theta = meanr; theta_nom = rnom. s = xi_to_sacc( cv.sacc_nz(ver), @@ -87,8 +91,9 @@ def run_2pcf( theta_nom=gg.rnom, npairs=gg.npairs, weight=gg.weight, + covariance=gg.cov if cov == "jackknife" else None, variances=( - np.concatenate([gg.varxip, gg.varxim]) if grid == "integration" else None + np.concatenate([gg.varxip, gg.varxim]) if cov == "diagonal" else None ), ) out_path = sacc_out or os.path.join( @@ -115,6 +120,7 @@ def _from_snakemake(smk): cat_config=p.get("cat_config", "./cat_config.yaml"), output_dir=p.get("output_dir", None), grid=p.get("grid", "reporting"), + cov=p.get("cov", "none"), # The SACC part goes exactly where the rule declares it; the .txt # byproduct still lands under the resolved output dir. sacc_out=smk.output["sacc"], @@ -145,11 +151,13 @@ def _from_cli(argv=None): ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") ap.add_argument( - "--grid", - default="reporting", - choices=["reporting", "integration"], - help="SACC grid tag; 'integration' also attaches the varxip/varxim " - "DiagonalCovariance", + "--grid", default="reporting", help="SACC grid tag for the measured points" + ) + ap.add_argument( + "--cov", + default="none", + choices=["jackknife", "diagonal", "none"], + help="Covariance the part carries", ) a = ap.parse_args(argv) run_2pcf( @@ -161,6 +169,7 @@ def _from_cli(argv=None): cat_config=a.cat_config, output_dir=a.out, grid=a.grid, + cov=a.cov, ) From aaedb503aefb1ab10373dbe81425653917127530 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 03:42:20 +0200 Subject: [PATCH 056/160] cv_cosebis: bind the COSEBIs grid's part, and declare the figures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rule's inputs are now one ξ± part instead of a TreeCorr .txt plus the integration part, and the three companion figures are declared outputs rather than byproducts landing beside them. The COSEBIs grid (0.9–300 at 1000 bins, patched) is a row in the grid table, so the same xi rule measures it. --- workflow/rules/cosmo_val.smk | 46 +++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index fcd9a304..e036ccc3 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -80,24 +80,40 @@ def cv_pure_eb_npz(version): ) -def cv_cosebis_npz(version): +def _cosebis_stub(version): + """Shared stem of the COSEBIs diagnostic products (npz + figures). + + Also the stem ``plot_cosebis`` builds for its own byproducts, so the two + write one set of files rather than two competing schemas. + """ cb = CV["cosebis"] fsc = CV["fiducial_scale_cut"] - # Mirror calculate/plot_cosebis out_stub (cosmo_val.py): a distinct schema - # from pure_eb — _cosebis_ prefix, integration nbins, plus _nmodes= and - # _scalecut= segments. Must match save_cosebis_results exactly or - # verify_outputs raises and cv_summarize_bmodes deadlocks on this input. return str( COSMO_VAL / ( f"{version}_cosebis_minsep={cb['min_sep_int']}" f"_maxsep={cb['max_sep_int']}_nbins={cb['nbins_int']}" f"_npatch={cb['npatch']}_varmethod=jackknife_nmodes={cb['nmodes']}" - f"_scalecut={fsc[0]}-{fsc[1]}_data.npz" + f"_scalecut={fsc[0]}-{fsc[1]}" ) ) +def cv_cosebis_npz(version): + """COSEBIs multi-cut diagnostic .npz (the PTE scan).""" + return _cosebis_stub(version) + "_data.npz" + + +def cv_cosebis_figures(version): + """The COSEBIs companion figures, by output key.""" + stub = _cosebis_stub(version) + return { + "figure_modes": f"{stub}_cosebis.png", + "figure_covariance": f"{stub}_covariance.png", + "figure_scalecut_ptes": f"{stub}_scalecut_ptes.png", + } + + def cv_pseudo_cl_sacc(version): """Untagged pseudo-Cl SACC part: the B-mode diagnostic, not a data product.""" return str(COSMO_VAL / f"pseudo_cl_{version}.sacc") @@ -366,23 +382,25 @@ rule cv_pure_eb: rule cv_cosebis: - """COSEBIs E/B decomposition for one version (config-space, fine binning).""" + """COSEBIs E/B decomposition for one version, from its ξ± part. + + Values, covariance and PTEs all come from the part: the COSEBIs covariance + is the part's ξ± covariance through the same kernel as the modes. + """ input: - xi=lambda w: cv_xi_txt(w.version), - xi_integration=lambda w: cv_xi_sacc(w.version, "integration"), + xi=lambda w: cv_xi_sacc(w.version, "cosebis"), output: npz=cv_cosebis_npz("{version}"), sacc=cv_cosebis_sacc("{version}"), + **cv_cosebis_figures("{version}"), params: version="{version}", - min_sep_int=CV["cosebis"]["min_sep_int"], - max_sep_int=CV["cosebis"]["max_sep_int"], - nbins_int=CV["cosebis"]["nbins_int"], - npatch=CV["cosebis"]["npatch"], + min_sep=CV["cosebis"]["min_sep_int"], + max_sep=CV["cosebis"]["max_sep_int"], + nbins=CV["cosebis"]["nbins_int"], nmodes=CV["cosebis"]["nmodes"], scale_cuts=CV["cosebis"]["scale_cuts"], fiducial_scale_cut=CV["fiducial_scale_cut"], - cv_init=lambda w: cv_init_params(config, version_list=[w.version]), rundir=CV_RUNDIR, threads: 24 resources: From cabc582eab10a8db71a44845994f7d18bca2a9a2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 04:06:04 +0200 Subject: [PATCH 057/160] Pure E/B derives from its parts; per-patch vectors are never written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pure-E/B jackknife needed the patched correlation objects, which is the one thing a part cannot carry — and per Cail's ruling jackknife and blinding are a deprecated pair for derived statistics. So the covariance now comes from the Monte Carlo path calculate_pure_eb already had, extracted as pure_eb_covariance_mc: draws from the CosmoCov gaussian covariance on the integration grid, around a theory mean, through the same kernel as the modes. It depends on the covariance model and the grids, never on the measured vector, so it is blind-invariant by construction. cv_pure_eb consumes two parts plus that one covariance file and touches no catalogue. The results dict is now self-describing — the grids travel with the modes (theta, edges, the raw ξ± and their variances, n_eff) instead of a TreeCorr object riding along — so the statistics, plots and npz all work from values. That deletes the FakeGG stub the paper PTE script needed, and turns calculate_eb_statistics into a function of its results alone. calculate_2pcf no longer writes patch results into the .txt dump: a per-patch ξ± realisation is an unblinded data vector, nothing reads one back, and the covariance a consumer needs is the matrix the part carries. --- .../bmodes/scripts/calculate_pure_eb_ptes.py | 20 +- src/sp_validation/b_modes.py | 299 ++++++++++-------- src/sp_validation/cosmo_val/core.py | 6 +- src/sp_validation/cosmo_val/pure_eb.py | 39 +-- src/sp_validation/cosmo_val/real_space.py | 13 +- src/sp_validation/tests/test_b_modes.py | 20 +- src/sp_validation/tests/test_cosmo_val.py | 2 +- workflow/rules/cosmo_val.smk | 64 +++- workflow/scripts/cv_pure_eb.py | 138 +++++--- 9 files changed, 347 insertions(+), 254 deletions(-) diff --git a/papers/bmodes/scripts/calculate_pure_eb_ptes.py b/papers/bmodes/scripts/calculate_pure_eb_ptes.py index f8bc709d..bc5aaf82 100644 --- a/papers/bmodes/scripts/calculate_pure_eb_ptes.py +++ b/papers/bmodes/scripts/calculate_pure_eb_ptes.py @@ -22,15 +22,6 @@ from sp_validation.b_modes import calculate_eb_statistics -class FakeGG: - """Minimal GGCorrelation-like object for calculate_eb_statistics.""" - - def __init__(self, nbins, npatch): - self.nbins = nbins - self.npatch1 = npatch - self.npatch2 = npatch - - def calculate_ptes( version, blind, @@ -43,10 +34,11 @@ def calculate_ptes( dataset = np.load(pure_eb_data) theta = dataset["theta"] - nbins = len(theta) results = { - "gg": FakeGG(nbins, int(npatch)), + "theta": theta, + # The MC draws are the realisations behind this covariance. + "n_eff": int(n_samples), "xip_E": dataset["xip_E"], "xim_E": dataset["xim_E"], "xip_B": dataset["xip_B"], @@ -57,11 +49,7 @@ def calculate_ptes( } print(f"Calculating PTE matrices for {version}...") - results = calculate_eb_statistics( - results, - cov_path_int=cov_integration, - n_samples=int(n_samples), - ) + results = calculate_eb_statistics(results) pte_matrices = results["pte_matrices"] output_data = { diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index dd5fcf3c..fed51fb1 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -175,75 +175,40 @@ def pure_EB(corrs): parallel=True, ) - # Initialize results dictionary with basic E/B mode data - results = {"gg": gg, "gg_int": gg_int} + # The results dict is self-describing: the grids it was measured on travel + # with the modes, so every consumer downstream works from values alone. + results = { + "theta": gg.meanr, + "left_edges": gg.left_edges, + "right_edges": gg.right_edges, + "xip": gg.xip, + "xim": gg.xim, + "var_xip": gg.varxip, + "var_xim": gg.varxim, + "theta_int": gg_int.meanr, + "xip_int": gg_int.xip, + "xim_int": gg_int.xim, + "n_eff": n_samples if cov_path_int is not None else gg.npatch1, + } results.update(dict(zip(_EB_KEYS, pure_EB([gg, gg_int])))) if cov_path_int is not None: - # Use semi-analytical covariance propagation - print("Computing semi-analytical covariance for pure E/B modes") - if z_dist is None: - raise ValueError("z_dist must be provided for semi-analytical covariance") - if cosmo_cov is None: + if z_dist is None or cosmo_cov is None: raise ValueError( - "cosmo_cov must be provided for semi-analytical covariance" - ) - - # Load covariance matrix - cov_int = np.loadtxt(cov_path_int) - - # Set up integration binning and pre-compute binning matrix - nbins_int, theta_int = len(gg_int.meanr), gg_int.meanr - reporting_bin_edges = np.concatenate([gg.left_edges, [gg.right_edges[-1]]]) - bin_indices = np.digitize(theta_int, reporting_bin_edges) - 1 - - valid_mask = (bin_indices >= 0) & (bin_indices < len(gg.meanr)) - row_indices, col_indices = (bin_indices[valid_mask], np.where(valid_mask)[0]) - - binning_matrix = sparse.csr_matrix( - (np.ones(len(row_indices)), (row_indices, col_indices)), - shape=(len(gg.meanr), nbins_int), - ) - row_sums = np.array(binning_matrix.sum(axis=1)).flatten() - binning_matrix = sparse.diags(1 / row_sums) @ binning_matrix - - # Generate theoretical xi+/xi- predictions and sample - mean_int = np.concatenate( - get_theo_xi( - theta=theta_int, - z=z_dist[:, 0], - nz=z_dist[:, 1], - backend="ccl", - cosmo=cosmo_cov, + "semi-analytical covariance needs both z_dist and cosmo_cov" ) + cov, eb_samples = pure_eb_covariance_mc( + theta=gg.meanr, + left_edges=gg.left_edges, + right_edges=gg.right_edges, + theta_int=gg_int.meanr, + cov_int=np.loadtxt(cov_path_int), + z=z_dist[:, 0], + nz=z_dist[:, 1], + cosmo=cosmo_cov, + n_samples=n_samples, ) - - samples_int = np.random.multivariate_normal(mean_int, cov_int, size=n_samples) - samples_int_xip = samples_int[:, :nbins_int] - samples_int_xim = samples_int[:, nbins_int:] - samples_rep_xip = (binning_matrix @ samples_int_xip.T).T - samples_rep_xim = (binning_matrix @ samples_int_xim.T).T - - transformed_samples = [ - np.concatenate( - get_pure_EB_modes( - theta=gg.meanr, - theta_int=gg_int.meanr, - xip=samples_rep_xip[i], - xim=samples_rep_xim[i], - xip_int=samples_int_xip[i], - xim_int=samples_int_xim[i], - tmin=min_sep, - tmax=max_sep, - parallel=True, - ) - ) - for i in tqdm.tqdm(range(n_samples), desc="MC samples") - ] - - # Store semi-analytical covariance results - eb_samples = np.array(transformed_samples) - results.update({"cov": np.cov(eb_samples.T), "eb_samples": eb_samples}) + results.update({"cov": cov, "eb_samples": eb_samples}) else: # Use existing treecorr covariance estimation results["cov"] = treecorr.estimate_multi_cov( @@ -325,6 +290,78 @@ def pure_eb_from_xi( return dict(zip(_EB_KEYS, (np.asarray(m) for m in modes))) +def pure_eb_covariance_mc( + *, + theta, + left_edges, + right_edges, + theta_int, + cov_int, + z, + nz, + cosmo, + n_samples=1000, +): + """Pure-E/B covariance by Monte Carlo through the same kernel as the modes. + + ξ± draws come from ``cov_int``, a ξ± covariance on the integration grid, + around the theory mean for ``(z, nz)`` under ``cosmo``; each draw is binned + down to the reporting grid and pushed through ``get_pure_EB_modes``. The + covariance of the transformed draws is the result, so it depends on the + covariance model and the grids, never on the measured data vector. + + Returns ``(cov, eb_samples)`` — the covariance in ``_EB_KEYS`` order and + the draws behind it. + """ + from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes + + theta, theta_int = np.asarray(theta), np.asarray(theta_int) + nbins_int = len(theta_int) + + # Each reporting bin averages the integration bins that fall inside it. + reporting_bin_edges = np.concatenate([left_edges, [right_edges[-1]]]) + bin_indices = np.digitize(theta_int, reporting_bin_edges) - 1 + valid_mask = (bin_indices >= 0) & (bin_indices < len(theta)) + row_indices, col_indices = (bin_indices[valid_mask], np.where(valid_mask)[0]) + binning_matrix = sparse.csr_matrix( + (np.ones(len(row_indices)), (row_indices, col_indices)), + shape=(len(theta), nbins_int), + ) + row_sums = np.array(binning_matrix.sum(axis=1)).flatten() + binning_matrix = sparse.diags(1 / row_sums) @ binning_matrix + + mean_int = np.concatenate( + get_theo_xi(theta=theta_int, z=z, nz=nz, backend="ccl", cosmo=cosmo) + ) + samples_int = np.random.multivariate_normal(mean_int, cov_int, size=n_samples) + samples_int_xip, samples_int_xim = ( + samples_int[:, :nbins_int], + samples_int[:, nbins_int:], + ) + samples_rep_xip = (binning_matrix @ samples_int_xip.T).T + samples_rep_xim = (binning_matrix @ samples_int_xim.T).T + + eb_samples = np.array( + [ + np.concatenate( + get_pure_EB_modes( + theta=theta, + theta_int=theta_int, + xip=samples_rep_xip[i], + xim=samples_rep_xim[i], + xip_int=samples_int_xip[i], + xim_int=samples_int_xim[i], + tmin=left_edges[0], + tmax=right_edges[-1], + parallel=True, + ) + ) + for i in tqdm.tqdm(range(n_samples), desc="MC samples") + ] + ) + return np.cov(eb_samples.T), eb_samples + + def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): """ Calculate COSEBIs modes from a correlation function for multiple scale cuts. @@ -454,11 +491,7 @@ def cosebis_scan_from_xi( return all_results -def calculate_eb_statistics( - results, - cov_path_int=None, - n_samples=1000, -): +def calculate_eb_statistics(results): """ Calculate E/B mode statistics using 2D PTE analysis for all scale cut combinations. @@ -469,23 +502,17 @@ def calculate_eb_statistics( Parameters ---------- results : dict - Dictionary containing pure E/B mode results from calculate_pure_eb_correlation - cov_path_int : str, optional - Path to integration covariance matrix for semi-analytical calculation - n_samples : int, optional - Number of Monte Carlo samples used for semi-analytical covariance - min_bins : int, optional - Minimum number of bins required for valid PTE calculation + Pure E/B results: the six mode arrays, the ``cov`` block, the reporting + ``theta``, and ``n_eff`` — the realisation count behind the covariance + (jackknife patches or MC draws), which sets the Hartlap debiasing Returns ------- dict Updated results dictionary with PTE matrices and statistics """ - gg = results["gg"] - nbins = gg.nbins - npatch = gg.npatch1 - n_eff = n_samples if cov_path_int is not None else npatch + nbins = len(results["theta"]) + n_eff = results["n_eff"] # Extract covariance blocks and standard deviations cov = results["cov"] @@ -549,16 +576,16 @@ def calculate_eb_statistics( return results -def plot_integration_vs_reporting(gg, gg_int, output_path, version): +def plot_integration_vs_reporting(results, output_path, version): """ Plot integration vs reporting scale comparison. Parameters ---------- - gg : treecorr.GGCorrelation - Reporting scale correlation function - gg_int : treecorr.GGCorrelation - Integration scale correlation function + results : dict + Pure E/B results carrying both grids (``theta``/``xip``/``xim`` and the + ``theta_int``/``xip_int``/``xim_int`` counterparts), plus the reporting + ``var_xip``/``var_xim`` the error bars use output_path : str Output file path for the plot version : str @@ -568,26 +595,24 @@ def plot_integration_vs_reporting(gg, gg_int, output_path, version): # Configure plot data for both xi+ and xi- in a consolidated loop plot_configs = [ - ("+", "xip", "varxip", r"$\theta \xi_+(\theta) \times 10^4$"), - ("-", "xim", "varxim", r"$\theta \xi_-(\theta) \times 10^4$"), + ("+", "xip", r"$\theta \xi_+(\theta) \times 10^4$"), + ("-", "xim", r"$\theta \xi_-(\theta) \times 10^4$"), ] data_configs = [ - (gg_int, "k.", 3, 0.3, "Integration"), - (gg, ".", 12, 1, "Reporting"), + ("_int", "k.", 3, 0.3, "Integration"), + ("", ".", 12, 1, "Reporting"), ] - for ax_idx, (xi_label, xi_attr, var_attr, ylabel) in enumerate(plot_configs): - for data, fmt, ms, alpha, label_type in data_configs: - xi_val = getattr(data, xi_attr) - yerr = ( - data.meanr * np.sqrt(getattr(data, var_attr)) / 1e-4 - if hasattr(data, var_attr) and label_type == "Reporting" - else None - ) + for ax_idx, (xi_label, xi_attr, ylabel) in enumerate(plot_configs): + for suffix, fmt, ms, alpha, label_type in data_configs: + theta = results[f"theta{suffix}"] + xi_val = results[f"{xi_attr}{suffix}"] + var = results.get(f"var_{xi_attr}{suffix}") + yerr = theta * np.sqrt(var) / 1e-4 if var is not None else None axs[ax_idx].errorbar( - data.meanr, - data.meanr * xi_val / 1e-4, + theta, + theta * xi_val / 1e-4, yerr=yerr, fmt=fmt, ms=ms, @@ -596,8 +621,8 @@ def plot_integration_vs_reporting(gg, gg_int, output_path, version): ls="" if label_type == "Reporting" else None, label=( rf"$\xi_{{{xi_label}}}$, {label_type}: " - rf"${data.min_sep} < \theta < {data.max_sep}$, " - rf"{data.nbins} bins" + rf"${theta[0]:.2g} < \theta < {theta[-1]:.4g}$, " + rf"{len(theta)} bins" ), ) axs[ax_idx].set( @@ -613,7 +638,7 @@ def plot_integration_vs_reporting(gg, gg_int, output_path, version): plt.savefig(output_path, dpi=300, bbox_inches="tight") -def _get_pte_from_scale_cut(pte_matrix, gg, scale_cut): +def _get_pte_from_scale_cut(pte_matrix, edges, scale_cut): """ Extract PTE value from matrix based on scale cut range using conservative logic. @@ -621,8 +646,8 @@ def _get_pte_from_scale_cut(pte_matrix, gg, scale_cut): ---------- pte_matrix : numpy.ndarray 2D PTE matrix - gg : treecorr.GGCorrelation - Correlation function object with bin edges + edges : tuple of numpy.ndarray + ``(left_edges, right_edges)`` of the grid the matrix is indexed on scale_cut : tuple (min_scale, max_scale) angular range for scale cut @@ -631,7 +656,8 @@ def _get_pte_from_scale_cut(pte_matrix, gg, scale_cut): float PTE value for the given scale cut, or full-range PTE if scale_cut is None """ - nbins = len(gg.meanr) + left_edges, right_edges = edges + nbins = len(left_edges) if scale_cut is None: # Return full-range PTE (first row, last column) @@ -639,8 +665,7 @@ def _get_pte_from_scale_cut(pte_matrix, gg, scale_cut): min_scale, max_scale = scale_cut - # Use conservative scale_cut_to_bins helper - start_bin, stop_bin = scale_cut_to_bins(gg, min_scale, max_scale) + start_bin, stop_bin = bins_from_edges(left_edges, right_edges, min_scale, max_scale) # Ensure valid range, otherwise fallback to full range if stop_bin <= start_bin or start_bin >= nbins or stop_bin <= 0: @@ -671,19 +696,20 @@ def plot_pure_eb_correlations( fiducial_xim_scale_cut : tuple, optional (min_scale, max_scale) for xi- fiducial analysis, shown as gray regions """ - gg = results["gg"] - nbins = gg.nbins + theta = results["theta"] + edges = (results["left_edges"], results["right_edges"]) + nbins = len(theta) cov = results["cov"] # Calculate combined PTE using off-diagonal covariance blocks # Get scale cuts for both xi+ and xi- if fiducial_xip_scale_cut is not None: - xip_start_bin, xip_stop_bin = scale_cut_to_bins(gg, *fiducial_xip_scale_cut) + xip_start_bin, xip_stop_bin = bins_from_edges(*edges, *fiducial_xip_scale_cut) else: xip_start_bin, xip_stop_bin = 0, nbins if fiducial_xim_scale_cut is not None: - xim_start_bin, xim_stop_bin = scale_cut_to_bins(gg, *fiducial_xim_scale_cut) + xim_start_bin, xim_stop_bin = bins_from_edges(*edges, *fiducial_xim_scale_cut) else: xim_start_bin, xim_stop_bin = 0, nbins @@ -718,7 +744,7 @@ def plot_pure_eb_correlations( if "eb_samples" in results: # Semi-analytical case n_eff = results["eb_samples"].shape[0] else: # Jackknife case - n_eff = gg.npatch1 + n_eff = results["n_eff"] hartlap_factor = (n_eff - nbins_eff - 2) / (n_eff - 1) chi2_combined = hartlap_factor * ( @@ -728,10 +754,10 @@ def plot_pure_eb_correlations( # Extract PTE values for fiducial scale cuts (or full range) xip_B_pte = _get_pte_from_scale_cut( - results["pte_matrices"]["xip_B"], gg, fiducial_xip_scale_cut + results["pte_matrices"]["xip_B"], edges, fiducial_xip_scale_cut ) xim_B_pte = _get_pte_from_scale_cut( - results["pte_matrices"]["xim_B"], gg, fiducial_xim_scale_cut + results["pte_matrices"]["xim_B"], edges, fiducial_xim_scale_cut ) fig, axs = plt.subplots(1, 2, figsize=(14, 6), sharex=True, sharey=True) @@ -743,14 +769,14 @@ def plot_pure_eb_correlations( ( "xip", "+", - "varxip", + "var_xip", r"$\xi_{+}=\xi_{+}^{E}+\xi_{+}^{B}+\xi_{+}^{\mathrm{amb}}$", xip_B_pte, ), ( "xim", "-", - "varxim", + "var_xim", r"$\xi_{-}=\xi_{-}^{E}-\xi_{-}^{B}+\xi_{-}^{\mathrm{amb}}$", xim_B_pte, ), @@ -760,11 +786,11 @@ def plot_pure_eb_correlations( plot_configs ): # Plot main correlation function - xi_val = getattr(gg, xi_type) + xi_val = results[xi_type] axs[ax_idx].errorbar( - gg.meanr, - gg.meanr * xi_val / scale_factor, - yerr=gg.meanr * np.sqrt(getattr(gg, var_attr)) / scale_factor, + theta, + theta * xi_val / scale_factor, + yerr=theta * np.sqrt(results[var_attr]) / scale_factor, fmt="k.", capsize=3, label=main_label, @@ -790,9 +816,9 @@ def plot_pure_eb_correlations( for key, color, alpha, label in plot_data: axs[ax_idx].errorbar( - gg.meanr, - gg.meanr * results[key] / scale_factor, - yerr=gg.meanr * results[f"std_{key}"] / scale_factor, + theta, + theta * results[key] / scale_factor, + yerr=theta * results[f"std_{key}"] / scale_factor, color=color, ls="", marker=".", @@ -822,12 +848,12 @@ def plot_pure_eb_correlations( xlim = original_xlims[ax_idx] # Use conservative scale_cut_to_bins helper for consistency - start_bin, stop_bin = scale_cut_to_bins(gg, min_scale, max_scale) + start_bin, stop_bin = bins_from_edges(*edges, min_scale, max_scale) # Show excluded regions based on bin edges used in PTE calculation # Lower exclusion: bins 0 to start_bin-1 are excluded if start_bin > 0: - lower_exclusion_edge = gg.right_edges[start_bin - 1] + lower_exclusion_edge = edges[1][start_bin - 1] axs[ax_idx].axvspan( xlim[0], lower_exclusion_edge, @@ -837,8 +863,8 @@ def plot_pure_eb_correlations( ) # Upper exclusion: bins stop_bin to end are excluded - if stop_bin < len(gg.left_edges): - upper_exclusion_edge = gg.left_edges[stop_bin] + if stop_bin < len(edges[0]): + upper_exclusion_edge = edges[0][stop_bin] axs[ax_idx].axvspan( upper_exclusion_edge, xlim[1], @@ -1042,8 +1068,9 @@ def plot_pte_2d_heatmaps( fiducial_xim_scale_cut : tuple, optional (min_scale, max_scale) for xi- fiducial analysis, shown as cross-hatched """ - gg = results["gg"] - nbins = gg.nbins + theta = results["theta"] + edges = (results["left_edges"], results["right_edges"]) + nbins = len(theta) pte_xip_B = results["pte_matrices"]["xip_B"] pte_xim_B = results["pte_matrices"]["xim_B"] @@ -1105,7 +1132,7 @@ def plot_pte_2d_heatmaps( for ax_idx, fiducial_scale_cut in enumerate(fiducial_scale_cuts): if fiducial_scale_cut is not None: min_scale, max_scale = fiducial_scale_cut - start_bin, stop_bin = scale_cut_to_bins(gg, min_scale, max_scale) + start_bin, stop_bin = bins_from_edges(*edges, min_scale, max_scale) if stop_bin > start_bin and start_bin < nbins and stop_bin > 0: rect_x = start_bin @@ -1129,12 +1156,8 @@ def plot_pte_2d_heatmaps( # Set angular scale ticks tick_indices = np.arange(0, nbins) - x_tick_labels = [ - f"{gg.left_edges[i]:.1f}" for i in tick_indices if i < len(gg.left_edges) - ] - y_tick_labels = [ - f"{gg.right_edges[i]:.1f}" for i in tick_indices if i < len(gg.right_edges) - ] + x_tick_labels = [f"{edges[0][i]:.1f}" for i in tick_indices if i < len(edges[0])] + y_tick_labels = [f"{edges[1][i]:.1f}" for i in tick_indices if i < len(edges[1])] x_tick_positions = tick_indices + 0.5 y_tick_positions = tick_indices + 0.5 @@ -1274,10 +1297,8 @@ def save_pure_eb_results(results, output_path): output_path : str Output .npz file path """ - gg = results["gg"] - # Data vectors and covariance - save_dict = {"theta": gg.meanr, "cov": results["cov"]} + save_dict = {"theta": results["theta"], "cov": results["cov"]} for key in _EB_KEYS: save_dict[key] = results[key] @@ -1286,7 +1307,7 @@ def save_pure_eb_results(results, output_path): save_dict[f"pte_matrices_{key}"] = matrix # Metadata - save_dict["npatch"] = np.array(gg.npatch1) + save_dict["n_eff"] = np.array(results["n_eff"]) if "eb_samples" in results: save_dict["var_method"] = np.array("semi-analytic") save_dict["n_samples"] = np.array(results["eb_samples"].shape[0]) diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 4f7f99c2..addd7309 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -579,18 +579,18 @@ def summarize_bmodes(self, fiducial_scale_cut=(12, 83), versions=None): # Pure E/B PTEs from stored results if ver in self._pure_eb_results: res = self._pure_eb_results[ver] - gg = res["gg"] + edges = (res["left_edges"], res["right_edges"]) try: for stat in ("xip_B", "xim_B", "combined"): row[stat] = _get_pte_from_scale_cut( - res["pte_matrices"][stat], gg, fiducial_scale_cut + res["pte_matrices"][stat], edges, fiducial_scale_cut ) except (KeyError, RuntimeError): pass cov_methods.add( "semi-analytic" if "eb_samples" in res - else f"jackknife ({gg.npatch1} patches)" + else f"jackknife ({res['n_eff']} patches)" ) # COSEBIs PTE from stored results diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index cf5fc7e6..934eaa42 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -7,7 +7,6 @@ import numpy as np -from .. import sacc_io from ..b_modes import ( calculate_eb_statistics, calculate_pure_eb_correlation, @@ -17,7 +16,6 @@ plot_pure_eb_correlations, save_pure_eb_results, ) -from .sacc_writers import pure_eb_to_sacc class PureEBMixin: @@ -134,29 +132,6 @@ def calculate_pure_eb( return results - def pure_eb_to_sacc_part(self, version, out_path, results, eb_override=None): - """Write the pure-E/B SACC part (six ``PURE_KEYS`` blocks + covariance). - - ``results`` is the dict ``calculate_pure_eb`` returned: the six pure-mode - arrays under ``sacc_io.PURE_KEYS``, the ``"cov"`` block (in ``PURE_KEYS`` - order), and the reporting-grid TreeCorr object ``"gg"`` whose ``meanr`` - is the shared ``theta``. - - ``eb_override`` replaces the six arrays; the covariance stays from - ``results``. - """ - theta = results["gg"].meanr - source = eb_override if eb_override is not None else results - eb = {key: source[key] for key in sacc_io.PURE_KEYS} - s = pure_eb_to_sacc( - self.sacc_nz(version), - self.sacc_metadata(version), - theta, - eb, - covariance=results["cov"], - ) - sacc_io.save(s, out_path, type="data") - def plot_pure_eb( self, versions=None, @@ -294,19 +269,13 @@ def plot_pure_eb( ) # Calculate E/B statistics for all bin combinations - version_results = calculate_eb_statistics( - version_results, - cov_path_int=cov_path_int, - n_samples=n_samples, - **kwargs, - ) - - # Generate all plots using specialized plotting functions - gg, gg_int = version_results["gg"], version_results["gg_int"] + version_results = calculate_eb_statistics(version_results, **kwargs) # Integration vs Reporting comparison plot plot_integration_vs_reporting( - gg, gg_int, out_stub + "_integration_vs_reporting.png", version + version_results, + out_stub + "_integration_vs_reporting.png", + version, ) # E/B/Ambiguous correlation functions plot diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 3400991d..1521ecc7 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -94,12 +94,13 @@ def calculate_2pcf(self, ver, npatch=None, **treecorr_config): # Process the catalog & write the correlation functions gg.process(cat_gal) - # Patch results and the jackknife covariance need npatch > 1: at - # npatch=1 var_method is "shot", so the covariance adds nothing over - # the varxip/varxim columns while the dense (2*nbins)^2 block would - # dominate the file on the fine integration grid (nbins ~ 1000). - write_cov = int(npatch) > 1 - gg.write(out_fname, write_patch_results=write_cov, write_cov=write_cov) + # Never write_patch_results: a per-patch ξ± realisation is an + # unblinded data vector, and nothing downstream reads one — the + # covariance a consumer needs is the matrix, which the SACC part + # carries. The .txt keeps the matrix only where there are patches to + # estimate it from; at npatch=1 var_method is "shot" and it would add + # nothing over the varxip/varxim columns. + gg.write(out_fname, write_patch_results=False, write_cov=int(npatch) > 1) # Add correlation object to class if not hasattr(self, "cat_ggs"): diff --git a/src/sp_validation/tests/test_b_modes.py b/src/sp_validation/tests/test_b_modes.py index 62bd11e1..fe5e02e2 100644 --- a/src/sp_validation/tests/test_b_modes.py +++ b/src/sp_validation/tests/test_b_modes.py @@ -50,6 +50,7 @@ def _eb_inputs(): nbins=4, npatch=50 so the Hartlap factor (n_eff - nbins_eff - 2)/(n_eff-1) is well-defined and strictly positive for every scale-cut combination. + n_eff is the jackknife patch count, as it is for a jackknife covariance. The covariance is built SPD via A @ A.T + I; the B-mode vectors are O(1) so the chi-squared (and hence PTE) lands in a meaningful range rather than being saturated at 1.0. @@ -60,8 +61,13 @@ def _eb_inputs(): cov = A @ A.T + np.eye(6 * nbins) xip_B = rng.standard_normal(nbins) xim_B = rng.standard_normal(nbins) - gg = types.SimpleNamespace(nbins=nbins, npatch1=npatch) - return {"gg": gg, "cov": cov, "xip_B": xip_B, "xim_B": xim_B}, nbins + return { + "theta": np.geomspace(1.0, 100.0, nbins), + "n_eff": npatch, + "cov": cov, + "xip_B": xip_B, + "xim_B": xim_B, + }, nbins # --------------------------------------------------------------------------- @@ -229,8 +235,8 @@ def test_calculate_eb_statistics_pte_matrices(): """Pin representative PTE-matrix entries from the full 2D E/B analysis. Inputs are fixed (seed 12345, nbins=4, npatch=50, SPD cov = A@A.T + I, - O(1) B-mode vectors). With cov_path_int=None the Hartlap correction uses - n_eff = npatch = 50. For each of xip_B, xim_B and combined we pin the + O(1) B-mode vectors). The Hartlap correction uses n_eff = 50, the patch + count behind a jackknife covariance. For each of xip_B, xim_B and combined we pin the full-range entry [0, nbins-1] (start=0, stop=nbins) and an interior entry [0, 2] (start=0, stop=3). These chi2->sf PTE values are deterministic functions of the seeded input. @@ -240,7 +246,7 @@ def test_calculate_eb_statistics_pte_matrices(): arithmetic shifts them past tolerance. """ results, nbins = _eb_inputs() - out = b_modes.calculate_eb_statistics(results, cov_path_int=None) + out = b_modes.calculate_eb_statistics(results) pm = out["pte_matrices"] # Full-range entries [0, nbins-1]. @@ -271,13 +277,13 @@ def test_calculate_eb_statistics_has_teeth(): combined 0.99999 -> 0.0031. """ results, nbins = _eb_inputs() - out = b_modes.calculate_eb_statistics(results, cov_path_int=None) + out = b_modes.calculate_eb_statistics(results) pm = out["pte_matrices"] loud, _ = _eb_inputs() loud["xip_B"] = loud["xip_B"] * 10.0 loud["xim_B"] = loud["xim_B"] * 10.0 - out_loud = b_modes.calculate_eb_statistics(loud, cov_path_int=None) + out_loud = b_modes.calculate_eb_statistics(loud) pm_loud = out_loud["pte_matrices"] for key in ("xip_B", "xim_B", "combined"): diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index f50992d4..f25ed522 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -697,4 +697,4 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path): # shape is pinned, not the values. cov = np.asarray(results["cov"]) assert cov.shape == (6 * nbins, 6 * nbins) - assert results["gg"].npatch1 == npatch + assert results["n_eff"] == npatch diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index e036ccc3..e6c29c8e 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -67,7 +67,8 @@ def cv_tau_stats(version): ) -def cv_pure_eb_npz(version): +def _pure_eb_stub(version): + """Shared stem of the pure-E/B diagnostic products (npz + figures).""" eb = CV["integration"] return str( COSMO_VAL @@ -75,16 +76,51 @@ def cv_pure_eb_npz(version): f"{version}_eb_minsep={CV['theta_min']}_maxsep={CV['theta_max']}" f"_nbins={CV['nbins']}_minsepint={eb['min_sep']}" f"_maxsepint={eb['max_sep']}_nbinsint={eb['nbins']}" - f"_npatch={CV['npatch']}_varmethod=jackknife_data.npz" + f"_npatch={CV['npatch']}_varmethod=semi-analytic" ) ) +def cv_pure_eb_npz(version): + """Pure-E/B data vectors + covariance .npz.""" + return _pure_eb_stub(version) + "_data.npz" + + +def cv_pure_eb_figures(version): + """The pure-E/B companion figures, by output key.""" + stub = _pure_eb_stub(version) + return { + "figure_integration_vs_reporting": f"{stub}_integration_vs_reporting.png", + "figure_xis": f"{stub}_xis.png", + "figure_ptes": f"{stub}_ptes.png", + "figure_covariance": f"{stub}_covariance.png", + } + + +def cv_xi_cov_integration(version): + """CosmoCov gaussian ξ± covariance on the integration grid. + + The covariance model the pure-E/B Monte Carlo draws from; gaussian because + the draws only need the scatter a Gaussian field would give. + """ + integ = CV["integration"] + return covariance_path( + version, + FIDUCIAL["blind"], + gaussian="g", + min_sep=integ["min_sep"], + max_sep=integ["max_sep"], + nbins=integ["nbins"], + mask_suffix=DEFAULT_MASK_SUFFIX, + ) + + def _cosebis_stub(version): """Shared stem of the COSEBIs diagnostic products (npz + figures). - Also the stem ``plot_cosebis`` builds for its own byproducts, so the two - write one set of files rather than two competing schemas. + varmethod names where the covariance came from, and these products are the + propagated one — which also keeps them clear of the paths plot_cosebis + builds for its own byproducts, so nothing overwrites a declared output. """ cb = CV["cosebis"] fsc = CV["fiducial_scale_cut"] @@ -93,7 +129,7 @@ def _cosebis_stub(version): / ( f"{version}_cosebis_minsep={cb['min_sep_int']}" f"_maxsep={cb['max_sep_int']}_nbins={cb['nbins_int']}" - f"_npatch={cb['npatch']}_varmethod=jackknife_nmodes={cb['nmodes']}" + f"_npatch={cb['npatch']}_varmethod=propagated_nmodes={cb['nmodes']}" f"_scalecut={fsc[0]}-{fsc[1]}" ) ) @@ -357,21 +393,27 @@ rule cv_pseudo_cl: # --------------------------------------------------------------------------- rule cv_pure_eb: - """Pure E/B-mode decomposition for one version (config-space).""" + """Pure E/B-mode decomposition for one version, from its ξ± parts. + + The modes come from the two parts; the covariance is Monte Carlo from the + integration-grid covariance model, so no patched estimator run is involved. + """ input: - xi=lambda w: cv_xi_txt(w.version), xi_reporting=lambda w: cv_xi_sacc(w.version, "reporting"), xi_integration=lambda w: cv_xi_sacc(w.version, "integration"), + cov_integration=lambda w: cv_xi_cov_integration(w.version), output: npz=cv_pure_eb_npz("{version}"), sacc=cv_pure_eb_sacc("{version}"), + **cv_pure_eb_figures("{version}"), params: version="{version}", - min_sep_int=CV["integration"]["min_sep"], - max_sep_int=CV["integration"]["max_sep"], - nbins_int=CV["integration"]["nbins"], + min_sep=CV["theta_min"], + max_sep=CV["theta_max"], + nbins=CV["nbins"], + n_samples=CV.get("n_mc_samples", 1000), + cosmo_params=CV["cosmo_params"], fiducial_scale_cut=CV["fiducial_scale_cut"], - cv_init=lambda w: cv_init_params(config, version_list=[w.version]), rundir=CV_RUNDIR, threads: 24 resources: diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index a6e730b0..d2acdbfe 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -1,48 +1,114 @@ """Rule cv_pure_eb: pure E/B-mode decomposition for one version. -plot_pure_eb calls calculate_pure_eb, which runs two TreeCorr correlations -(reporting + integration binning). Writes the SACC part, the .npz data product -and the companion figures. +A consumer of the two ξ± parts plus one covariance file — nothing here touches +a catalogue. The modes come from the reporting and integration parts through +the pipeline kernel; the covariance is Monte Carlo through that same kernel, +drawn from the CosmoCov integration-grid ξ± covariance around a theory mean, so +it depends on the covariance model and the grids rather than on the measured +vector. A jackknife of the transformed modes would need per-patch realisations, +which are never persisted. """ import numpy as np -from cv_runner import _unbuffer_streams, make_cv, verify_outputs +from cs_util.cosmo import get_cosmo +from cv_runner import _unbuffer_streams, verify_outputs from snakemake.script import snakemake +from sp_validation import sacc_io +from sp_validation.b_modes import ( + calculate_eb_statistics, + log_bin_edges, + plot_eb_covariance_matrix, + plot_integration_vs_reporting, + plot_pte_2d_heatmaps, + plot_pure_eb_correlations, + pure_eb_covariance_mc, + pure_eb_from_xi, + save_pure_eb_results, +) +from sp_validation.cosmo_val.sacc_writers import pure_eb_to_sacc + _unbuffer_streams() -cv = make_cv(snakemake) p = snakemake.params version = p["version"] -cv.plot_pure_eb( - versions=[version], - min_sep_int=p["min_sep_int"], - max_sep_int=p["max_sep_int"], - nbins_int=p["nbins_int"], - fiducial_xip_scale_cut=tuple(p["fiducial_scale_cut"]), - fiducial_xim_scale_cut=tuple(p["fiducial_scale_cut"]), -) -results = cv._pure_eb_results[version] - -# Re-derive the pure modes from the ξ± parts via the same kernel; cov stays raw. -# tmin/tmax are the reporting grid's TreeCorr bin edges (add_xi stores no edges). -from sp_validation import sacc_io -from sp_validation.b_modes import pure_eb_from_xi - -gg = results["gg"] -tmin, tmax = float(gg.left_edges[0]), float(gg.right_edges[-1]) -rep = sacc_io.load(snakemake.input["xi_reporting"]) -integ = sacc_io.load(snakemake.input["xi_integration"]) -tr, xpr, xmr = sacc_io.get_xi(rep, (0, 0), grid="reporting") -ti, xpi, xmi = sacc_io.get_xi(integ, (0, 0), grid="integration") -modes = pure_eb_from_xi(tr, xpr, xmr, ti, xpi, xmi, tmin, tmax) - -cv.pure_eb_to_sacc_part(version, snakemake.output["sacc"], results, eb_override=modes) - -# Sync the npz's pure modes with the part-derived values; theta / cov / PTE untouched. -npz_path = snakemake.output["npz"] -data = dict(np.load(npz_path, allow_pickle=True)) -for key, arr in modes.items(): - data[key] = np.asarray(arr) -np.savez(npz_path, **data) +fiducial_scale_cut = tuple(p["fiducial_scale_cut"]) + +reporting = sacc_io.load(snakemake.input["xi_reporting"]) +integration = sacc_io.load(snakemake.input["xi_integration"]) +theta, xip, xim = sacc_io.get_xi(reporting, (0, 0), grid="reporting") +theta_int, xip_int, xim_int = sacc_io.get_xi(integration, (0, 0), grid="integration") +left_edges, right_edges = log_bin_edges(p["min_sep"], p["max_sep"], p["nbins"]) + +# The reporting grid must sit strictly inside the integration grid: a reporting +# point on the boundary has no interior support and comes back NaN. +modes = pure_eb_from_xi( + theta, xip, xim, theta_int, xip_int, xim_int, left_edges[0], right_edges[-1] +) + +z, nz = sacc_io.get_nz(reporting, 0) +cov, eb_samples = pure_eb_covariance_mc( + theta=theta, + left_edges=left_edges, + right_edges=right_edges, + theta_int=theta_int, + cov_int=np.loadtxt(snakemake.input["cov_integration"]), + z=z, + nz=nz, + cosmo=get_cosmo(**p["cosmo_params"]), + n_samples=p["n_samples"], +) + +variances = reporting.covariance.dense.diagonal() +results = { + "theta": theta, + "left_edges": left_edges, + "right_edges": right_edges, + "xip": xip, + "xim": xim, + "var_xip": variances[: len(theta)], + "var_xim": variances[len(theta) :], + "theta_int": theta_int, + "xip_int": xip_int, + "xim_int": xim_int, + "n_eff": p["n_samples"], + "cov": cov, + "eb_samples": eb_samples, + **modes, +} +results = calculate_eb_statistics(results) + +plot_integration_vs_reporting( + results, snakemake.output["figure_integration_vs_reporting"], version +) +plot_pure_eb_correlations( + results, + snakemake.output["figure_xis"], + version, + fiducial_xip_scale_cut=fiducial_scale_cut, + fiducial_xim_scale_cut=fiducial_scale_cut, +) +plot_pte_2d_heatmaps( + results, + version, + snakemake.output["figure_ptes"], + fiducial_xip_scale_cut=fiducial_scale_cut, + fiducial_xim_scale_cut=fiducial_scale_cut, +) +plot_eb_covariance_matrix( + cov, "semi-analytic", snakemake.output["figure_covariance"], version +) + +save_pure_eb_results(results, snakemake.output["npz"]) + +# The part inherits the ξ± part's provenance; `type` is re-stamped on save. +metadata = {k: v for k, v in reporting.metadata.items() if k != "type"} +s = pure_eb_to_sacc( + {0: (z, nz)}, + metadata, + theta, + {key: results[key] for key in sacc_io.PURE_KEYS}, + covariance=cov, +) +sacc_io.save(s, snakemake.output["sacc"], type="data") verify_outputs(snakemake) From 770619d351800cd79ec05603a935d3fc6efaf7e4 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 04:15:36 +0200 Subject: [PATCH 058/160] cv_summarize_bmodes reads the products instead of recomputing them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It re-ran plot_pure_eb / plot_cosebis / plot_pseudo_cl in-process to repopulate the in-memory result dicts, which meant the terminal diagnostic reached the catalogue — and rewrote the two consumers' figures from unblinded values on the way past. It now reads what those rules wrote: the pure-E/B PTE matrices and the COSEBIs B-mode PTE from their .npz products, and the pseudo-Cℓ BB spectrum from its part against the NaMaster covariance, both declared inputs. The summary can no longer disagree with the products it summarises. The table itself moves to print_bmode_summary, so the in-memory path (summarize_bmodes, for notebooks) and the file path print the same thing. A test pins the .npz key contract the two rules meet on. --- src/sp_validation/cosmo_val/core.py | 70 +++++++++---------- src/sp_validation/tests/test_b_modes.py | 34 ++++++++++ workflow/rules/cosmo_val.smk | 24 +++---- workflow/scripts/cv_summarize_bmodes.py | 90 ++++++++++++++----------- 4 files changed, 130 insertions(+), 88 deletions(-) diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index addd7309..bff3581a 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -23,8 +23,42 @@ from .pure_eb import PureEBMixin from .real_space import RealSpaceMixin - # %% +BMODE_COLUMNS = { + "xip_B": r"xi+B", + "xim_B": r"xi-B", + "combined": "Combined", + "COSEBIS": "COSEBIS", + "C_l_BB": "C_l^BB", +} + + +def print_bmode_summary(summary, fiducial_scale_cut, cov_methods=()): + """Print the B-mode PTE table for ``{version: {statistic: pte}}``. + + Statistics absent from a row print as ``--``. + """ + sc_label = f"[{fiducial_scale_cut[0]}-{fiducial_scale_cut[1]} arcmin]" + sep = "\u2500" * 70 + header = f"{'Version':<28s}" + "".join( + f"{label:>10s}" for label in BMODE_COLUMNS.values() + ) + + print(f"\nB-mode summary {sc_label}") + print(sep) + print(header) + print(sep) + for ver, row in summary.items(): + cells = "".join( + f"{row[s]:>10.4f}" if s in row else f"{'--':>10s}" for s in BMODE_COLUMNS + ) + print(f"{ver:<28s}{cells}") + print(sep) + if cov_methods: + print(f"Covariance: {', '.join(sorted(cov_methods))}") + print() + + class CosmologyValidation( CosebisMixin, PureEBMixin, @@ -618,37 +652,5 @@ def summarize_bmodes(self, fiducial_scale_cut=(12, 83), versions=None): summary[ver] = row - # Print summary table - col_labels = { - "xip_B": r"xi+B", - "xim_B": r"xi-B", - "combined": "Combined", - "COSEBIS": "COSEBIS", - "C_l_BB": "C_l^BB", - } - stats_order = list(col_labels) - - sc_label = f"[{fiducial_scale_cut[0]}-{fiducial_scale_cut[1]} arcmin]" - sep = "\u2500" * 70 - header = f"{'Version':<28s}" + "".join( - f"{label:>10s}" for label in col_labels.values() - ) - - print(f"\nB-mode summary {sc_label}") - print(sep) - print(header) - print(sep) - - for ver in versions: - row = summary[ver] - cells = "".join( - f"{row[s]:>10.4f}" if s in row else f"{'--':>10s}" for s in stats_order - ) - print(f"{ver:<28s}{cells}") - - print(sep) - if cov_methods: - print(f"Covariance: {', '.join(sorted(cov_methods))}") - print() - + print_bmode_summary(summary, fiducial_scale_cut, cov_methods) return summary diff --git a/src/sp_validation/tests/test_b_modes.py b/src/sp_validation/tests/test_b_modes.py index fe5e02e2..f51913fc 100644 --- a/src/sp_validation/tests/test_b_modes.py +++ b/src/sp_validation/tests/test_b_modes.py @@ -398,3 +398,37 @@ def cosebis_covariance_from_xipm_covariance(self, theta_cut, cov_cut): npatch=None, ).values() assert result["hartlap_factor"] == 1 + + +def test_pure_eb_npz_carries_what_the_summary_reads(tmp_path): + """The .npz keys cv_summarize_bmodes reads are the ones the writer emits. + + The two live in different rules, so the contract between them — the PTE + matrices under ``pte_matrices_{stat}`` and the realisation count under + ``n_eff`` — is pinned here rather than discovered on a cluster run. + """ + results, nbins = _eb_inputs() + results.update( + {key: np.zeros(nbins) for key in b_modes._EB_KEYS if key not in results} + ) + results = b_modes.calculate_eb_statistics(results) + + out = tmp_path / "pure_eb_data.npz" + b_modes.save_pure_eb_results(results, str(out)) + saved = np.load(out) + + for stat in ("xip_B", "xim_B", "combined"): + assert f"pte_matrices_{stat}" in saved + assert saved[f"pte_matrices_{stat}"].shape == (nbins, nbins) + assert saved["n_eff"] == results["n_eff"] + npt.assert_allclose(saved["theta"], results["theta"]) + for key in b_modes._EB_KEYS: + assert key in saved + + # The summary reads the fiducial cut out of those matrices through the same + # helper the plots use, so a valid cut must resolve to a finite PTE. + edges = b_modes.log_bin_edges(1.0, 100.0, nbins) + pte = b_modes._get_pte_from_scale_cut( + saved["pte_matrices_xip_B"], edges, (1.0, 100.0) + ) + assert np.isfinite(pte) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index e6c29c8e..ebf54464 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -461,25 +461,23 @@ rule cv_summarize_bmodes: [cv_pseudo_cl_sacc(v) for v in CV_VERSIONS] if CV.get("include_pseudo_cl", False) else [] ), + pseudo_cl_cov=( + [cv_pseudo_cl_cov(v) for v in CV_VERSIONS] + if CV.get("include_pseudo_cl", False) else [] + ), output: summary_json=str(COSMO_VAL / "bmode_summary.json"), params: + versions=CV_VERSIONS, fiducial_scale_cut=CV["fiducial_scale_cut"], - pure_eb_min_sep_int=CV["integration"]["min_sep"], - pure_eb_max_sep_int=CV["integration"]["max_sep"], - pure_eb_nbins_int=CV["integration"]["nbins"], - cosebis_min_sep_int=CV["cosebis"]["min_sep_int"], - cosebis_max_sep_int=CV["cosebis"]["max_sep_int"], - cosebis_nbins_int=CV["cosebis"]["nbins_int"], - cosebis_npatch=CV["cosebis"]["npatch"], - cosebis_nmodes=CV["cosebis"]["nmodes"], - cosebis_scale_cuts=CV["cosebis"]["scale_cuts"], + min_sep=CV["theta_min"], + max_sep=CV["theta_max"], + nbins=CV["nbins"], include_pseudo_cl=CV.get("include_pseudo_cl", False), - **cv_params(), - threads: 24 + rundir=CV_RUNDIR, resources: - mem_mb=48000, - runtime=600, + mem_mb=8000, + runtime=20, script: "../scripts/cv_summarize_bmodes.py" diff --git a/workflow/scripts/cv_summarize_bmodes.py b/workflow/scripts/cv_summarize_bmodes.py index 0fe7b95a..1200ed4d 100644 --- a/workflow/scripts/cv_summarize_bmodes.py +++ b/workflow/scripts/cv_summarize_bmodes.py @@ -1,56 +1,64 @@ """Rule cv_summarize_bmodes: collect B-mode PTEs across all statistics. -The terminal diagnostic. summarize_bmodes reads the in-memory -_pure_eb_results / _cosebis_results / _pseudo_cls dicts, which are populated by -plot_pure_eb / plot_cosebis / plot_pseudo_cl. The per-version E/B and COSEBIs -npz products and the pseudo-Cl SACC parts are declared as inputs (so the DAG -forces those rules first), but the summary still needs the live result objects (it -reads each version's TreeCorr `gg`, which the npz cannot hold). So this rule -re-runs the three B-mode methods in-process: they reload the existing 2pcf / -data-vector files via their skip-if-exists paths and recompute only the cheap -PTE statistics, exactly as the original linear driver did on its shared cv. - -Writes the summary table to bmode_summary.txt (declared output) — the original -driver only printed it. +The terminal diagnostic, and a reader of what the three B-mode rules already +wrote: the pure-E/B PTE matrices and the COSEBIs B-mode PTE from their .npz +products, and the pseudo-Cℓ BB spectrum from its SACC part against the NaMaster +covariance. Nothing is recomputed and no catalogue is touched, so the summary +cannot disagree with the products it summarises. """ import json -from cv_runner import _unbuffer_streams, make_cv, verify_outputs +import numpy as np +from cv_runner import _unbuffer_streams, verify_outputs from snakemake.script import snakemake +from sp_validation import sacc_io +from sp_validation.b_modes import _get_pte_from_scale_cut, log_bin_edges +from sp_validation.cosmo_val.core import print_bmode_summary +from sp_validation.statistics import chi2_and_pte + _unbuffer_streams() -cv = make_cv(snakemake) p = snakemake.params fiducial_scale_cut = tuple(p["fiducial_scale_cut"]) +edges = log_bin_edges(p["min_sep"], p["max_sep"], p["nbins"]) + +summary = {} +cov_methods = set() + +for i, version in enumerate(p["versions"]): + row = {} + + pure_eb = np.load(snakemake.input["pure_eb"][i]) + for stat in ("xip_B", "xim_B", "combined"): + try: + row[stat] = _get_pte_from_scale_cut( + pure_eb[f"pte_matrices_{stat}"], edges, fiducial_scale_cut + ) + except (KeyError, RuntimeError): + pass + cov_methods.add(f"pure-E/B: semi-analytic ({int(pure_eb['n_eff'])} draws)") + + # The COSEBIs .npz is written at the fiducial cut, so its PTE is the one + # this table wants. + cosebis = np.load(snakemake.input["cosebis"][i]) + row["COSEBIS"] = float(cosebis["pte_B"]) + cov_methods.add("COSEBIs: propagated from the ξ± covariance") + + if p["include_pseudo_cl"]: + from astropy.io import fits + + part = sacc_io.load(snakemake.input["pseudo_cl"][i]) + _ell, _ee, bb, _eb, _window = sacc_io.get_pseudo_cl(part, (0, 0)) + with fits.open(snakemake.input["pseudo_cl_cov"][i]) as hdul: + cov_bb = np.asarray(hdul["COVAR_BB_BB"].data, float) + _chi2, _red, row["C_l_BB"] = chi2_and_pte(bb, cov_bb) + cov_methods.add("pseudo-Cℓ: Gaussian (NaMaster)") + + summary[version] = row + +print_bmode_summary(summary, fiducial_scale_cut, cov_methods) -# Repopulate the in-memory B-mode result dicts from existing data products. -cv.plot_pure_eb( - min_sep_int=p["pure_eb_min_sep_int"], - max_sep_int=p["pure_eb_max_sep_int"], - nbins_int=p["pure_eb_nbins_int"], - fiducial_xip_scale_cut=fiducial_scale_cut, - fiducial_xim_scale_cut=fiducial_scale_cut, -) -for version in cv.versions: - cv.plot_cosebis( - version=version, - min_sep_int=p["cosebis_min_sep_int"], - max_sep_int=p["cosebis_max_sep_int"], - nbins_int=p["cosebis_nbins_int"], - npatch=p["cosebis_npatch"], - nmodes=p["cosebis_nmodes"], - scale_cuts=[tuple(sc) for sc in p["cosebis_scale_cuts"]], - fiducial_scale_cut=fiducial_scale_cut, - ) -if p.get("include_pseudo_cl", False): - cv.plot_pseudo_cl() - -summary = cv.summarize_bmodes(fiducial_scale_cut=fiducial_scale_cut) - -# summarize_bmodes prints its table and returns {version: {stat: pte}}. Persist -# the returned dict (the table itself is reproducible from it) so downstream -# tooling and the all-rule have a real, machine-readable artifact to depend on. with open(snakemake.output["summary_json"], "w") as f: json.dump(summary, f, indent=2, default=str) From 40d20a8cfbd1de95de7beb46a1a52e8938cb1b90 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 04:44:09 +0200 Subject: [PATCH 059/160] Review round: one pseudo-Cl producer, analytic covariance wins, grid tags canonical MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six findings from the review. One pseudo-Cl producer. The untagged diagnostic part and its rule are gone; the summary, the terminal file and now the figures all read the analysis part with its matching NaMaster covariance, so the three cannot disagree. The figures become a plot-only ingest rule like the other B-mode ones, over a plot_pseudo_cl_spectrum that draws one spectrum for every version — the three near-identical EE/EB/BB blocks were one function all along. The analytic covariance wins, loudly. A part whose analysis covariance is external (ξ± reporting, pseudo-Cl) now always takes the supplied block, which replaces the estimate it was born with rather than losing a race with it, and says so on stdout; a missing injection raises instead of silently keeping the part's own. The reporting part keeps carrying its jackknife — it is a real diagnostic, just not the analysis covariance. Grid tags are canonical. xi_binning stamped raw YAML (maxsep=300) while the measurement wrote float-normalised names (maxsep=300.0), so producer and consumer would have asked for different paths on the first real run. The table coerces once, where it is built. It moved to common.py to be testable at all, and the tests pin the round trip a filename makes. Also: cosebis_from_xi deleted (no callers, and its centre-based cut diverged from the edge-based one in use); the paper PTE script's unread cov_integration and npatch dropped, with the rule docstring corrected to say plainly that nothing there varies by blind; and the header DAG comment rewritten to the SACC-part graph it actually describes. --- papers/bmodes/rules/claims.smk | 14 +- .../bmodes/scripts/calculate_pure_eb_ptes.py | 15 +- src/sp_validation/b_modes.py | 25 -- src/sp_validation/cosmo_val/pseudo_cl.py | 245 +++++------------- src/sp_validation/tests/test_assemble_sacc.py | 60 +++-- src/sp_validation/tests/test_xi_grids.py | 109 ++++++++ workflow/common.py | 89 +++++++ workflow/rules/cosmo_val.smk | 84 +++--- workflow/rules/twopoint.smk | 67 +---- workflow/scripts/assemble_sacc.py | 57 ++-- workflow/scripts/cv_plot_pseudo_cl.py | 40 +++ workflow/scripts/cv_pseudo_cl.py | 13 - 12 files changed, 436 insertions(+), 382 deletions(-) create mode 100644 src/sp_validation/tests/test_xi_grids.py create mode 100644 workflow/scripts/cv_plot_pseudo_cl.py delete mode 100644 workflow/scripts/cv_pseudo_cl.py diff --git a/papers/bmodes/rules/claims.smk b/papers/bmodes/rules/claims.smk index c6b5b0e7..388de007 100644 --- a/papers/bmodes/rules/claims.smk +++ b/papers/bmodes/rules/claims.smk @@ -355,25 +355,21 @@ rule pure_eb_covariance: rule calculate_pure_eb_ptes: - """Calculate PTE matrices for Pure E/B mode scale cut robustness. + """PTE matrices for pure E/B-mode scale-cut robustness. - Per-blind: Uses blind-specific integration covariance for PTE calculation. - The pure_eb_data vectors are identical across blinds; only covariance differs. - - In practice, BB covariance is blind-independent (validated by - bb_covariance_blind_independence), so downstream consumers (config_space_pte_matrices) - only request blind A. The per-blind wildcard is retained for the blind independence test. + Nothing here varies with the blind: the data vectors come from the blind-A + gather and the PTEs are Hartlap-debiased by the MC draw count, not by a + per-blind covariance. The wildcard survives as the filename slot the + consumer (config_space_pte_matrices) reads, and only blind A is ever built. """ input: pure_eb_data="results/paper_plots/intermediate/{version}_A_pure_eb_semianalytic.npz", - cov_integration=lambda w: _cov_integration_path(w.version, w.blind), output: "results/paper_plots/intermediate/{version}_{blind}_pure_eb_ptes.npz", wildcard_constraints: blind=r"[ABC]", params: version="{version}", - npatch=FIDUCIAL["npatch"], n_samples=config["covariance"]["n_samples"], resources: mem_mb=16000, diff --git a/papers/bmodes/scripts/calculate_pure_eb_ptes.py b/papers/bmodes/scripts/calculate_pure_eb_ptes.py index bc5aaf82..3a623669 100644 --- a/papers/bmodes/scripts/calculate_pure_eb_ptes.py +++ b/papers/bmodes/scripts/calculate_pure_eb_ptes.py @@ -4,14 +4,13 @@ pure-E/B ``semianalytic.npz`` (data vectors + MC covariance), evaluates the ξ_+^B / ξ_-^B / joint ξ_tot^B χ² PTE matrices over the scale-cut grid via ``sp_validation.b_modes.calculate_eb_statistics`` (Hartlap-corrected inverse -MC covariance), and writes the PTE matrices to +MC covariance, debiased by the draw count), and writes the PTE matrices to ``{out}/{version}_{blind}_pure_eb_ptes.npz``. python calculate_pure_eb_ptes.py \ --version SP_v1.4.6.3_leak_corr --blind A \ --pure-eb-data <..._pure_eb_semianalytic.npz> \ - --cov-integration \ - --npatch 1 --n-samples 2000 --out + --n-samples 2000 --out """ import argparse @@ -26,8 +25,6 @@ def calculate_ptes( version, blind, pure_eb_data, - cov_integration, - npatch, n_samples, output_dir, ): @@ -71,12 +68,6 @@ def _from_cli(argv=None): ap.add_argument("--version", required=True) ap.add_argument("--blind", default="A") ap.add_argument("--pure-eb-data", required=True, help="Gathered semianalytic .npz") - ap.add_argument( - "--cov-integration", - default=None, - help="Integration-grid covariance _processed.txt (optional)", - ) - ap.add_argument("--npatch", type=int, default=1) ap.add_argument("--n-samples", type=int, default=2000) ap.add_argument("--out", required=True, help="Output directory (lc {output})") a = ap.parse_args(argv) @@ -84,8 +75,6 @@ def _from_cli(argv=None): version=a.version, blind=a.blind, pure_eb_data=a.pure_eb_data, - cov_integration=a.cov_integration, - npatch=a.npatch, n_samples=a.n_samples, output_dir=a.out, ) diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index fed51fb1..c4b9a2b7 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -231,31 +231,6 @@ def pure_EB(corrs): return results -def cosebis_from_xi(theta, xip, xim, nmodes, scale_cut=None): - """COSEBIs (Eₙ, Bₙ) from ξ± arrays through the pipeline kernel (values only). - - The values-only seam of :func:`calculate_cosebis`, for callers holding ξ± - arrays rather than a TreeCorr ``GGCorrelation``. - - ``scale_cut`` is ``(theta_min, theta_max)``, the min/max of the retained bin - centres, selected inclusively. - """ - from cosmo_numba.B_modes.cosebis import COSEBIS - - theta, xip, xim = (np.asarray(a) for a in (theta, xip, xim)) - tmin, tmax = scale_cut if scale_cut is not None else (theta.min(), theta.max()) - cut = (theta >= tmin) & (theta <= tmax) - theta_cut, xip_cut, xim_cut = theta[cut], xip[cut], xim[cut] - cosebis = COSEBIS( - theta_min=np.min(theta_cut), - theta_max=np.max(theta_cut), - N_max=nmodes, - precision=120, - ) - En, Bn = cosebis.cosebis_from_xipm(theta_cut, xip_cut, xim_cut, parallel=True) - return np.asarray(En), np.asarray(Bn) - - def pure_eb_from_xi( theta_report, xip_report, xim_report, theta_int, xip_int, xim_int, tmin, tmax ): diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index b327e37b..cf74efc6 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -31,6 +31,45 @@ from .sacc_writers import pseudo_cl_to_sacc +def plot_pseudo_cl_spectrum(datasets, spectrum, output_path): + """Two-panel ℓC_ℓ / C_ℓ figure for one spectrum across catalogue versions. + + ``datasets`` maps a version to ``{"ell", "cl", "cov", "style"}``, where + ``style`` carries the ``marker`` and ``colour`` the version is drawn with. + """ + fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8)) + minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] + + for panel, scaled in ((ax[0], True), (ax[1], False)): + for version, data in datasets.items(): + ell, cl = np.asarray(data["ell"]), np.asarray(data["cl"]) + err = np.sqrt(np.diag(np.asarray(data["cov"]))) + style = data.get("style", {}) + panel.errorbar( + ell, + ell * cl if scaled else cl, + yerr=ell * err if scaled else err, + fmt=style.get("marker", "."), + color=style.get("colour"), + label=f"{version} {spectrum}", + capsize=2 if scaled else None, + ) + panel.set_ylabel(r"$\ell C_\ell$" if scaled else r"$C_\ell$") + panel.set_xlim(ell.min() - 10, ell.max() + 100) + panel.set_xscale("squareroot") + panel.set_xticks(np.array([100, 400, 900, 1600])) + panel.minorticks_on() + panel.tick_params(axis="x", which="minor", length=2, width=0.8) + panel.xaxis.set_ticks(minor_ticks, minor=True) + + ax[1].set_xlabel(r"$\ell$") + ax[1].set_yscale("log") + plt.suptitle(f"Pseudo-Cl {spectrum} (Gaussian covariance)") + plt.legend() + plt.savefig(output_path) + plt.close(fig) + + class PseudoClMixin: @property def pseudo_cls(self): @@ -694,201 +733,33 @@ def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): sacc_io.save(s, out_path, type="data") def plot_pseudo_cl(self): - """ - Plot pseudo-Cl's for given catalogs. - """ + """Plot the EE/EB/BB pseudo-Cl spectra for every version.""" self.print_cyan("Plotting pseudo-Cl's") - # Plotting EE - out_path = self._output_path("cell_ee.png") - fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8)) - - for ver in self.versions: - ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] - cov = self.pseudo_cls[ver]["cov"]["COVAR_EE_EE"].data - ax[0].errorbar( - ell, - ell * self.pseudo_cls[ver]["pseudo_cl"]["EE"], - yerr=ell * np.sqrt(np.diag(cov)), - fmt=self.cc[ver]["marker"], - label=ver + " EE", - color=self.cc[ver]["colour"], - capsize=2, - ) - - ax[0].set_ylabel(r"$\ell C_\ell$") - - ax[0].set_xlim(ell.min() - 10, ell.max() + 100) - ax[0].set_xscale("squareroot") - ax[0].set_xticks(np.array([100, 400, 900, 1600])) - ax[0].minorticks_on() - ax[0].tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax[0].xaxis.set_ticks(minor_ticks, minor=True) - - for ver in self.versions: - ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] - cov = self.pseudo_cls[ver]["cov"]["COVAR_EE_EE"].data - ax[1].errorbar( - ell, - self.pseudo_cls[ver]["pseudo_cl"]["EE"], - yerr=np.sqrt(np.diag(cov)), - fmt=self.cc[ver]["marker"], - label=ver + " EE", - color=self.cc[ver]["colour"], - ) - - ax[1].set_xlabel(r"$\ell$") - ax[1].set_ylabel(r"$C_\ell$") - - ax[1].set_xlim(ell.min() - 10, ell.max() + 100) - ax[1].set_xscale("squareroot") - ax[1].set_yscale("log") - ax[1].set_xticks(np.array([100, 400, 900, 1600])) - ax[1].minorticks_on() - ax[1].tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax[1].xaxis.set_ticks(minor_ticks, minor=True) - - plt.suptitle("Pseudo-Cl EE (Gaussian covariance)") - plt.legend() - plt.savefig(out_path) - - # Plotting EB - out_path = self._output_path("cell_eb.png") - - fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8)) - - for ver in self.versions: - ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] - cov = self.pseudo_cls[ver]["cov"]["COVAR_EB_EB"].data - ax[0].errorbar( - ell, - ell * self.pseudo_cls[ver]["pseudo_cl"]["EB"], - yerr=ell * np.sqrt(np.diag(cov)), - fmt=self.cc[ver]["marker"], - label=ver + " EB", - color=self.cc[ver]["colour"], - capsize=2, - ) - - ax[0].axhline(0, color="black", linestyle="--") - ax[0].set_ylabel(r"$\ell C_\ell$") - - ax[0].set_xlim(ell.min() - 10, ell.max() + 100) - ax[0].set_xscale("squareroot") - ax[0].set_xticks(np.array([100, 400, 900, 1600])) - ax[0].minorticks_on() - ax[0].tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax[0].xaxis.set_ticks(minor_ticks, minor=True) - - for ver in self.versions: - ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] - cov = self.pseudo_cls[ver]["cov"]["COVAR_EB_EB"].data - ax[1].errorbar( - ell, - self.pseudo_cls[ver]["pseudo_cl"]["EB"], - yerr=np.sqrt(np.diag(cov)), - fmt=self.cc[ver]["marker"], - label=ver + " EB", - color=self.cc[ver]["colour"], + for spectrum in ("EE", "EB", "BB"): + datasets = { + ver: { + "ell": self.pseudo_cls[ver]["pseudo_cl"]["ELL"], + "cl": self.pseudo_cls[ver]["pseudo_cl"][spectrum], + "cov": self.pseudo_cls[ver]["cov"][ + f"COVAR_{spectrum}_{spectrum}" + ].data, + "style": { + "marker": self.cc[ver]["marker"], + "colour": self.cc[ver]["colour"], + }, + } + for ver in self.versions + } + plot_pseudo_cl_spectrum( + datasets, spectrum, self._output_path(f"cell_{spectrum.lower()}.png") ) - ax[1].set_xlabel(r"$\ell$") - ax[1].set_ylabel(r"$C_\ell$") - - ax[1].set_xlim(ell.min() - 10, ell.max() + 100) - ax[1].set_xscale("squareroot") - ax[1].set_yscale("log") - ax[1].set_xticks(np.array([100, 400, 900, 1600])) - ax[1].minorticks_on() - ax[1].tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax[1].xaxis.set_ticks(minor_ticks, minor=True) - - plt.suptitle("Pseudo-Cl EB (Gaussian covariance)") - plt.legend() - plt.savefig(out_path) - - # Plotting BB - out_path = self._output_path("cell_bb.png") - - fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8)) - - for ver in self.versions: - ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] - cov = self.pseudo_cls[ver]["cov"]["COVAR_BB_BB"].data - ax[0].errorbar( - ell, - ell * self.pseudo_cls[ver]["pseudo_cl"]["BB"], - yerr=ell * np.sqrt(np.diag(cov)), - fmt=self.cc[ver]["marker"], - label=ver + " BB", - color=self.cc[ver]["colour"], - capsize=2, - ) - - ax[0].axhline(0, color="black", linestyle="--") - ax[0].set_ylabel(r"$\ell C_\ell$") - - ax[0].set_xlim(ell.min() - 10, ell.max() + 100) - ax[0].set_xscale("squareroot") - ax[0].set_xticks(np.array([100, 400, 900, 1600])) - ax[0].minorticks_on() - ax[0].tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax[0].xaxis.set_ticks(minor_ticks, minor=True) - - for ver in self.versions: - ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] - cov = self.pseudo_cls[ver]["cov"]["COVAR_BB_BB"].data - ax[1].errorbar( - ell, - self.pseudo_cls[ver]["pseudo_cl"]["BB"], - yerr=np.sqrt(np.diag(cov)), - fmt=self.cc[ver]["marker"], - label=ver + " BB", - color=self.cc[ver]["colour"], - ) - - ax[1].set_xlabel(r"$\ell$") - ax[1].set_ylabel(r"$C_\ell$") - - ax[1].set_xlim(ell.min() - 10, ell.max() + 100) - ax[1].set_xscale("squareroot") - ax[1].set_yscale("log") - ax[1].set_xticks(np.array([100, 400, 900, 1600])) - ax[1].minorticks_on() - ax[1].tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax[1].xaxis.set_ticks(minor_ticks, minor=True) - - plt.suptitle("Pseudo-Cl BB (Gaussian covariance)") - plt.legend() - plt.savefig(out_path) - - # Print C_l^BB PTE for each version and save BB data - print("\nC_l^BB PTE summary:") for ver in self.versions: cl_bb = self.pseudo_cls[ver]["pseudo_cl"]["BB"] cov_bb = self.pseudo_cls[ver]["cov"]["COVAR_BB_BB"].data chi2_bb, _, pte_bb = chi2_and_pte(cl_bb, cov_bb) - chi2_bb = float(chi2_bb) print( f" {ver}: C_l^BB PTE = {pte_bb:.4f} " - f"(chi2/dof = {chi2_bb:.1f}/{len(cl_bb)})" - ) - - # Save BB data + covariance to .npz - ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] - bb_out = self._output_path(f"{ver}_cell_bb_data.npz") - np.savez( - bb_out, - ell=ell, - cl_bb=cl_bb, - cov_bb=cov_bb, - chi2_bb=np.array(chi2_bb), - pte_bb=np.array(pte_bb), + f"(chi2/dof = {float(chi2_bb):.1f}/{len(cl_bb)})" ) - print(f" Saved BB data to {bb_out}") diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py index 25da7375..f17d5468 100644 --- a/src/sp_validation/tests/test_assemble_sacc.py +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -66,6 +66,19 @@ def _xi_cov_txt(tmp_path, n=12, seed=21): return str(path), cov +def _pseudo_cl_cov_fits(tmp_path, n=3): + """A NaMaster covariance FITS: one HDU per spectrum. Returns (path, blocks).""" + from astropy.io import fits + + blocks = {"EE": _spd(n, 31), "BB": _spd(n, 32), "EB": _spd(n, 33)} + path = tmp_path / "pseudo_cl_cov.fits" + fits.HDUList( + [fits.PrimaryHDU()] + + [fits.ImageHDU(block, name=f"COVAR_{k}_{k}") for k, block in blocks.items()] + ).writeto(str(path)) + return str(path), blocks + + def _write_parts(tmp_path, *, with_pseudo_cl=True, cov_less=("xi_reporting",)): """Write per-statistic parts to disk; return the ``{name: path}`` mapping. @@ -154,8 +167,11 @@ def test_assemble_sacc_canonical_order(tmp_path): (ξ±, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ).""" paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) cov_path, xi_cov = _xi_cov_txt(tmp_path) + cl_cov_path, _blocks = _pseudo_cl_cov_fits(tmp_path) out = tmp_path / "vSYNTH.sacc" - s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=cov_path) + s = asm.assemble_sacc( + "vSYNTH", paths, str(out), xi_cov=cov_path, pseudo_cl_cov=cl_cov_path + ) assert out.exists() assert type(s.covariance).__name__ == "BlockDiagonalCovariance" assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) @@ -183,13 +199,23 @@ def test_assemble_sacc_canonical_order(tmp_path): assert np.allclose(dense[np.ix_(xi_idx, co_idx)], 0.0) -def test_assemble_sacc_injects_real_xi_covariance(tmp_path): - """A CosmoCov ξ covariance .txt is loaded into the cov-less ξ± block.""" - paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) +def test_injected_xi_covariance_replaces_the_parts_own(tmp_path): + """The analytic ξ± covariance wins over the estimate the part was born with. + + The reporting part carries the jackknife it was measured with — useful as a + diagnostic, but the analysis file takes the CosmoCov block. + """ + paths = _write_parts(tmp_path, cov_less=()) # ξ± born with its own jackknife cov_path, xi_cov = _xi_cov_txt(tmp_path) + cl_cov_path, _blocks = _pseudo_cl_cov_fits(tmp_path) + + born = sio.load(paths["xi_reporting"], allow_unblinded=True).covariance.dense + assert not np.allclose(born, xi_cov) # the two are distinguishable out = tmp_path / "vSYNTH.sacc" - s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=cov_path) + s = asm.assemble_sacc( + "vSYNTH", paths, str(out), xi_cov=cov_path, pseudo_cl_cov=cl_cov_path + ) tr = ("source_0", "source_0") xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) assert np.allclose(s.covariance.dense[np.ix_(xi_idx, xi_idx)], xi_cov) @@ -198,25 +224,15 @@ def test_assemble_sacc_injects_real_xi_covariance(tmp_path): def test_assemble_sacc_injects_pseudo_cl_covariance(tmp_path): """The NaMaster cov FITS (COVAR_EE_EE/BB_BB/EB_EB) → block-diagonal pseudo-Cℓ block, beside the injected CosmoCov ξ± block (the live default).""" - from astropy.io import fits - paths = _write_parts(tmp_path, cov_less=("xi_reporting", "pseudo_cl")) cov_path, xi_cov = _xi_cov_txt(tmp_path) # pseudo-Cℓ part is 3 ell × {EE, BB, EB} = 9 points; per-spectrum 3×3 blocks. - ee, bb, eb = _spd(3, 31), _spd(3, 32), _spd(3, 33) - cov_fits = tmp_path / "pseudo_cl_cov.fits" - fits.HDUList( - [ - fits.PrimaryHDU(), - fits.ImageHDU(ee, name="COVAR_EE_EE"), - fits.ImageHDU(bb, name="COVAR_BB_BB"), - fits.ImageHDU(eb, name="COVAR_EB_EB"), - ] - ).writeto(str(cov_fits)) + cov_fits, blocks = _pseudo_cl_cov_fits(tmp_path) + ee, bb, eb = blocks["EE"], blocks["BB"], blocks["EB"] out = tmp_path / "vSYNTH.sacc" s = asm.assemble_sacc( - "vSYNTH", paths, str(out), xi_cov=cov_path, pseudo_cl_cov=str(cov_fits) + "vSYNTH", paths, str(out), xi_cov=cov_path, pseudo_cl_cov=cov_fits ) tr = ("source_0", "source_0") cl_idx = np.concatenate( @@ -232,11 +248,11 @@ def test_assemble_sacc_injects_pseudo_cl_covariance(tmp_path): assert np.allclose(dense[np.ix_(xi_idx, cl_idx)], 0.0) -def test_assemble_sacc_missing_cov_raises(tmp_path): - """A cov-less part with no injected block fails loudly.""" - paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) +def test_missing_injected_covariance_raises(tmp_path): + """A statistic whose covariance is external cannot fall back to its own.""" + paths = _write_parts(tmp_path, cov_less=()) # every part born with a block out = tmp_path / "vSYNTH.sacc" - with pytest.raises(ValueError, match="carries no covariance"): + with pytest.raises(ValueError, match="takes its analysis covariance from"): asm.assemble_sacc("vSYNTH", paths, str(out)) diff --git a/src/sp_validation/tests/test_xi_grids.py b/src/sp_validation/tests/test_xi_grids.py new file mode 100644 index 00000000..c35a2e3f --- /dev/null +++ b/src/sp_validation/tests/test_xi_grids.py @@ -0,0 +1,109 @@ +"""Tests for the ξ± grid table in ``workflow/common.py``. + +The table names the files the ``xi`` rule writes and the ones every consumer +asks for, so producer and consumer agree only if the tag is built from +canonical values. These tests pin that canonicalisation and the grid lookup. +""" + +import importlib.util +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.fast + + +def _load_common(): + root = next( + p for p in Path(__file__).resolve().parents if (p / "pyproject.toml").exists() + ) + path = root / "workflow" / "common.py" + spec = importlib.util.spec_from_file_location("wf_common_grids", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +common = _load_common() + +# A cosmo_val block as YAML delivers it: integer-valued separations stay ints. +CONFIG = { + "cosmo_val": { + "theta_min": 1.0, + "theta_max": 250.0, + "nbins": 20, + "npatch": 100, + "integration": {"min_sep": 0.08, "max_sep": 300, "nbins": 1000}, + "cosebis": { + "min_sep_int": 0.9, + "max_sep_int": 300, + "nbins_int": 1000, + "npatch": 100, + }, + } +} +FIDUCIAL = { + "min_sep": 1.0, + "max_sep": 250.0, + "nbins": 20, + "npatch": 1, + "min_sep_int": 0.5, + "max_sep_int": 300, + "nbins_int": 1000, +} + + +def test_tag_is_built_from_canonical_values(): + """An integer YAML separation still names the file as a float. + + run_2pcf coerces separations with float() before TreeCorr writes, so a + `max_sep: 300` that reached the tag as "300" would have the consumer ask + for a path the producer never writes. + """ + grids = common.xi_grids(CONFIG, FIDUCIAL) + assert common.grid_binning(grids["integration"]).endswith( + "minsep=0.08_maxsep=300.0_nbins=1000_npatch=1" + ) + assert ( + common.grid_binning(grids["cosebis"]) + == "minsep=0.9_maxsep=300.0_nbins=1000_npatch=100" + ) + # Counts stay integers, so no "nbins=1000.0" creeps into a name. + assert "nbins=1000_" in common.grid_binning(grids["cosebis"]) + + +def test_grid_lookup_round_trips_through_the_tag(): + """Every grid's own binning resolves back to that grid. + + This is the producer/consumer contract: the rule resolves a job's grid from + the wildcards its filename bound. + """ + grids = common.xi_grids(CONFIG, FIDUCIAL) + for name, grid in grids.items(): + binning = {key: grid[key] for key in common.XI_KEYS} + assert common.grid_of(grids, binning) == name + # Wildcards arrive as strings; the comparison is numeric. + assert common.grid_of(grids, {k: str(v) for k, v in binning.items()}) == name + + +def test_covariance_mode_follows_the_patches(): + """Patched grids get a jackknife block, unpatched ones none.""" + grids = common.xi_grids(CONFIG, FIDUCIAL) + assert grids["reporting"]["cov"] == "jackknife" + assert grids["cosebis"]["cov"] == "jackknife" + assert grids["integration"]["cov"] == "none" + + +def test_unnamed_binning_is_a_reporting_measurement(): + """The paper's convergence-check binning belongs to no named grid.""" + grids = common.xi_grids(CONFIG, FIDUCIAL) + stray = {"min_sep": 1.0, "max_sep": 250.0, "nbins": 10000, "npatch": 1} + assert common.grid_of(grids, stray) == "reporting" + + +def test_workflow_without_cosmo_val_falls_back_to_fiducial(): + """papers/bmodes carries no cosmo_val block; its grids come from FIDUCIAL.""" + grids = common.xi_grids({}, FIDUCIAL) + assert grids["reporting"]["npatch"] == 1 + assert grids["integration"]["min_sep"] == 0.5 + assert "cosebis" not in grids diff --git a/workflow/common.py b/workflow/common.py index 70003dcf..ca6901d9 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -184,6 +184,95 @@ def build_redshift_path(version, blind): return f"/n17data/sguerrini/UNIONS/WL/nz/{version_dir}/nz_{base}_{blind}.txt" +# --------------------------------------------------------------------------- +# ξ± angular grids +# --------------------------------------------------------------------------- +# A grid is a binning plus how its covariance is estimated: (min_sep, max_sep, +# nbins, npatch, cov). `reporting` is the analysis grid, `integration` the fine +# one the B-mode integrals run over, `cosebis` the fine patched grid COSEBIs +# propagates its covariance from. cov is "jackknife" (dense, from the patches), +# "diagonal" (TreeCorr varxip/varxim) or "none". +XI_KEYS = ( + "min_sep", + "max_sep", + "nbins", + "npatch", +) # the binning; cov is not in the name + + +def xi_grids(config, fiducial): + """The named ξ± grids of a workflow, canonicalised. + + Workflows carrying no cosmo_val block (e.g. papers/bmodes) fall back to + ``fiducial``. Values are coerced here — separations to float, counts to int + — so the tag the table stamps into a filename is the one the measurement + writes: the separations pass through float() on the way to TreeCorr, so a + YAML ``300`` must become ``300.0`` before it names a file, or producer and + consumer ask for different paths. + """ + cv = config.get("cosmo_val", {}) + grids = { + "reporting": ( + { + "min_sep": cv["theta_min"], + "max_sep": cv["theta_max"], + "nbins": cv["nbins"], + "npatch": cv["npatch"], + } + if cv + else {k: fiducial[k] for k in XI_KEYS} + ), + "integration": dict( + cv.get("integration") + or { + "min_sep": fiducial["min_sep_int"], + "max_sep": fiducial["max_sep_int"], + "nbins": fiducial["nbins_int"], + } + ), + } + grids["integration"].setdefault("npatch", 1) + cb = cv.get("cosebis") + if cb: + grids["cosebis"] = { + "min_sep": cb["min_sep_int"], + "max_sep": cb["max_sep_int"], + "nbins": cb["nbins_int"], + "npatch": cb["npatch"], + } + for grid in grids.values(): + for key in ("min_sep", "max_sep"): + grid[key] = float(grid[key]) + for key in ("nbins", "npatch"): + grid[key] = int(grid[key]) + # A jackknife estimate needs patches; at npatch=1 TreeCorr's var_method + # is "shot" and the diagonal is all it can offer. + grid.setdefault("cov", "jackknife" if grid["npatch"] > 1 else "none") + return grids + + +def grid_binning(grid): + """The `minsep=..._maxsep=..._nbins=..._npatch=...` tag of one grid.""" + return ( + f"minsep={grid['min_sep']}_maxsep={grid['max_sep']}" + f"_nbins={grid['nbins']}_npatch={grid['npatch']}" + ) + + +def grid_of(grids, binning): + """Name of the grid a binning belongs to, compared numerically. + + A "300" wildcard matches a 300.0 grid value. Binnings matching no named + grid (e.g. papers/bmodes' nbins=10000 convergence check) are reporting-style + measurements. + """ + key = tuple(float(binning[k]) for k in XI_KEYS) + for name, grid in grids.items(): + if tuple(float(grid[k]) for k in XI_KEYS) == key: + return name + return "reporting" + + def pseudo_cl_tag(config): """Fiducial harmonic-binning tag stamped into pseudo-Cl filenames.""" fiducial = config["harmonic"]["fiducial"] diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index ebf54464..87138bc0 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -3,29 +3,33 @@ # The original cosmo_val/run_cosmo_val.py was one linear driver that built a # single in-memory `cv` (CosmologyValidation) and called ~13 cv.() # diagnostics in sequence, linked only by lazy properties on that object. Here -# each diagnostic is a rule, and the rules are linked by the *real* data -# products each method writes under COSMO_VAL (= cosmo_val/output): +# each diagnostic is a rule, and the rules are linked by the SACC parts and +# products they write under COSMO_VAL (= cosmo_val/output): # -# rho/tau FITS ──┬─→ rho/tau plots -# ├─→ rho_tau_fits (PSF-error MCMC) -# └─────────────────────────────┐ -# additive bias ──→ xi (2pcf) ──┬─→ 2pcf plot │ -# ├─→ ratio_xi_sys_xi ←┘ (also needs xi_psf_sys) -# ├─→ pure_eb (npz) ─┐ -# └─→ cosebis (npz) ─┤ -# pseudo_cl FITS ──────────────────────────────────┼─→ summarize_bmodes -# ┘ +# catalogue ──→ xi (one job per grid: reporting, integration, cosebis) +# │ +# ├─ reporting part ──┬─→ pure_eb (part, npz, figures) +# ├─ integration part ┘ │ +# ├─ cosebis part ─────→ cosebis (part, npz, figures) +# └─ reporting .txt ──→ 2pcf plot, ratio_xi_sys_xi +# catalogue ──→ pseudo_cl (part) ──┬─→ pseudo-Cl figures +# CosmoCov ──→ covariance ─────────┤ +# rho/tau (part + FITS) ───────────┼─→ summarize_bmodes (reads the products) +# └─→ assemble_sacc ──→ {version}.sacc # -# Granularity decision: methods that write durable data products -# (calculate_rho_tau_stats, calculate_2pcf, calculate_pseudo_cl, plot_pure_eb, -# plot_cosebis) own a compute rule keyed on those files. Methods that only -# emit figures, or whose figure paths derive from internal handler state -# (rho/tau plots, rho_tau_fits, objectwise leakage, 2pcf overlay), declare a -# sentinel under COSMO_VAL/snakemake_sentinels so they stay DAG-trackable. -# Lazy cv state that the original code never persists (c1/c2, xi_psf_sys) is -# either materialized to a small JSON (additive bias) or recomputed in the one -# rule that needs it (xi_psf_sys in ratio_xi_sys_xi) — recompute is cheap next -# to the science it depends on. See workflow/scripts/cv_runner.py. +# The B-mode rules are ingests: pure_eb, cosebis, the pseudo-Cl figures and the +# summary all work from the parts and the covariance inputs, never from a +# catalogue, so a blinded part keeps everything downstream blinded. The +# analytic covariances (CosmoCov ξ±, NaMaster pseudo-Cℓ) are what assembly puts +# in the terminal file, replacing the estimates a part was born with. +# +# Methods that only emit figures, or whose figure paths derive from internal +# handler state (rho/tau plots, rho_tau_fits, objectwise leakage, 2pcf +# overlay), declare a sentinel under COSMO_VAL/snakemake_sentinels so they stay +# DAG-trackable. Lazy cv state the original code never persists (c1/c2, +# xi_psf_sys) is either materialized to a small JSON (additive bias) or +# recomputed in the one rule that needs it — recompute is cheap next to the +# science it depends on. See workflow/scripts/cv_runner.py. CV = config["cosmo_val"] CV_VERSIONS = config["versions"] @@ -150,11 +154,6 @@ def cv_cosebis_figures(version): } -def cv_pseudo_cl_sacc(version): - """Untagged pseudo-Cl SACC part: the B-mode diagnostic, not a data product.""" - return str(COSMO_VAL / f"pseudo_cl_{version}.sacc") - - _PSEUDO_CL_TAG = pseudo_cl_tag(config) @@ -374,18 +373,32 @@ rule cv_ratio_xi_sys_xi: # Harmonic-space pseudo-Cl # --------------------------------------------------------------------------- -rule cv_pseudo_cl: - """Pseudo-Cl E/B spectra for all versions (NaMaster), born as SACC parts.""" +def cv_pseudo_cl_figures(): + """The pseudo-Cl figures, by output key (one per spectrum, all versions).""" + return { + f"figure_{name}": str(COSMO_VAL / f"cell_{name}.png") + for name in ("ee", "eb", "bb") + } + + +rule cv_plot_pseudo_cl: + """The EE/EB/BB pseudo-Cl figures, from the analysis parts.""" + input: + pseudo_cl=[cv_pseudo_cl_analysis_sacc(v) for v in CV_VERSIONS], + pseudo_cl_cov=[cv_pseudo_cl_cov(v) for v in CV_VERSIONS], output: - pseudo_cl=[cv_pseudo_cl_sacc(v) for v in CV_VERSIONS], + **cv_pseudo_cl_figures(), params: - **cv_params(), - threads: 12 + versions=CV_VERSIONS, + # Style is per catalogue, so the derived variants take their parent's. + markers=[CATALOG_CONFIG[base_version(v)]["marker"] for v in CV_VERSIONS], + colours=[CATALOG_CONFIG[base_version(v)]["colour"] for v in CV_VERSIONS], + rundir=CV_RUNDIR, resources: - mem_mb=32000, - runtime=180, + mem_mb=8000, + runtime=20, script: - "../scripts/cv_pseudo_cl.py" + "../scripts/cv_plot_pseudo_cl.py" # --------------------------------------------------------------------------- @@ -458,7 +471,7 @@ rule cv_summarize_bmodes: pure_eb=[cv_pure_eb_npz(v) for v in CV_VERSIONS], cosebis=[cv_cosebis_npz(v) for v in CV_VERSIONS], pseudo_cl=( - [cv_pseudo_cl_sacc(v) for v in CV_VERSIONS] + [cv_pseudo_cl_analysis_sacc(v) for v in CV_VERSIONS] if CV.get("include_pseudo_cl", False) else [] ), pseudo_cl_cov=( @@ -559,5 +572,6 @@ rule cosmo_val_all: str(COSMO_VAL / "ratio_xi_sys_xi.png"), # B-modes str(COSMO_VAL / "bmode_summary.json"), + list(cv_pseudo_cl_figures().values()) if CV.get("include_pseudo_cl", False) else [], # Terminal analysis file: the assembled {version}.sacc per version [cv_analysis_sacc(v) for v in CV_VERSIONS], diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 6d886a05..a3ba7e74 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -3,74 +3,19 @@ # --------------------------------------------------------------------------- # ξ± angular grids # --------------------------------------------------------------------------- -# A grid is a binning plus how its covariance is estimated: (min_sep, max_sep, -# nbins, npatch, cov). `reporting` is the analysis grid, `integration` the fine -# one the B-mode integrals run over, `cosebis` the fine patched grid COSEBIs -# propagates its covariance from. cov is "jackknife" (dense, from the patches), -# "diagonal" (TreeCorr varxip/varxim) or "none". Workflows carrying no cosmo_val -# block (e.g. papers/bmodes) fall back to their fiducial grids. -def _xi_grids(): - cv = config.get("cosmo_val", {}) - reporting = ( - { - "min_sep": cv["theta_min"], - "max_sep": cv["theta_max"], - "nbins": cv["nbins"], - "npatch": cv["npatch"], - } - if cv - else {k: FIDUCIAL[k] for k in ("min_sep", "max_sep", "nbins", "npatch")} - ) - integration = dict( - cv.get("integration") - or { - "min_sep": FIDUCIAL["min_sep_int"], - "max_sep": FIDUCIAL["max_sep_int"], - "nbins": FIDUCIAL["nbins_int"], - } - ) - integration.setdefault("npatch", 1) - grids = {"reporting": reporting, "integration": integration} - cb = cv.get("cosebis") - if cb: - grids["cosebis"] = { - "min_sep": cb["min_sep_int"], - "max_sep": cb["max_sep_int"], - "nbins": cb["nbins_int"], - "npatch": cb["npatch"], - } - for grid in grids.values(): - # A jackknife estimate needs patches; at npatch=1 TreeCorr's var_method - # is "shot" and the diagonal is all it can offer. - grid.setdefault("cov", "jackknife" if int(grid["npatch"]) > 1 else "none") - return grids - - -XI_GRIDS = _xi_grids() -XI_KEYS = ("min_sep", "max_sep", "nbins", "npatch") # the binning; `cov` is not part of the name +# The table itself lives in common.py, where it can be built from a plain config +# dict and tested; these are the workflow's bindings to it. +XI_GRIDS = xi_grids(config, FIDUCIAL) def xi_binning(grid): """The `minsep=..._maxsep=..._nbins=..._npatch=...` tag of a named grid.""" - g = XI_GRIDS[grid] - return ( - f"minsep={g['min_sep']}_maxsep={g['max_sep']}" - f"_nbins={g['nbins']}_npatch={g['npatch']}" - ) + return grid_binning(XI_GRIDS[grid]) def xi_grid_of(wildcards): - """Grid label for the binning a job was requested with. - - Compared numerically, so a "300" wildcard matches a 300.0 config value. - Binnings matching no named grid (e.g. papers/bmodes' nbins=10000 - convergence check) are measured as plain reporting-style measurements. - """ - key = tuple(float(getattr(wildcards, k)) for k in XI_KEYS) - for name, g in XI_GRIDS.items(): - if tuple(float(g[k]) for k in XI_KEYS) == key: - return name - return "reporting" + """Grid label for the binning a job was requested with.""" + return grid_of(XI_GRIDS, {key: getattr(wildcards, key) for key in XI_KEYS}) rule xi: diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index bc9049c4..50d1d62e 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -6,11 +6,11 @@ Each part is a single-statistic SACC; they load in CANONICAL order and are rebuilt into one Sacc with a single ``BlockDiagonalCovariance``. -Every part must carry a covariance block. ξ± reporting and pseudo-Cℓ are born -without one, so theirs are injected here from the CosmoCov ``.txt`` -(``--xi-cov``) and the NaMaster covariance FITS (``--pseudo-cl-cov``); the -pseudo-Cℓ cross-spectrum blocks (EE↔BB, …) are dropped, matching what the -B-mode PTE reads today. +Every part must carry a covariance block. ξ± reporting and pseudo-Cℓ take +theirs from the analytic inputs — the CosmoCov ``.txt`` (``--xi-cov``) and the +NaMaster covariance FITS (``--pseudo-cl-cov``) — which replace any estimate the +part was born with; the pseudo-Cℓ cross-spectrum blocks (EE↔BB, …) are dropped, +matching what the B-mode PTE reads today. """ import argparse @@ -44,23 +44,46 @@ def _pseudo_cl_cov_block(cov_fits): return full +# The statistics whose analysis covariance is external, and the input each one +# takes it from. A part of one of these types may be born with an estimate of +# its own — the ξ± reporting part carries the jackknife it was measured with — +# but the analysis file takes the external one, always. +_INJECTED = {"xi_reporting": "xi_cov", "pseudo_cl": "pseudo_cl_cov"} + + def _attach_cov(part, name, xi_cov, pseudo_cl_cov): - """Ensure ``part`` (mutated in place) carries a covariance block. + """Give ``part`` (mutated in place) the covariance the analysis file uses. - Raises if a required ξ± / pseudo-Cℓ block was not supplied. + For the two statistics with an external covariance the supplied block + replaces whatever the part was born with, loudly; every other part keeps + its own. Raises if the block a part needs was not supplied. """ - if part.covariance is not None: - return part - if name == "xi_reporting" and xi_cov is not None: - part.add_covariance(np.loadtxt(xi_cov)) - return part - if name == "pseudo_cl" and pseudo_cl_cov is not None: - part.add_covariance(_pseudo_cl_cov_block(pseudo_cl_cov)) + if name not in _INJECTED: + if part.covariance is None: + raise ValueError( + f"the {name!r} part carries no covariance and none is injected " + "for it; its writer must attach one" + ) return part - raise ValueError( - f"the {name!r} part carries no covariance and no covariance input was " - "given (--xi-cov / --pseudo-cl-cov); supply the block" + + supplied = xi_cov if name == "xi_reporting" else pseudo_cl_cov + if supplied is None: + raise ValueError( + f"the {name!r} part takes its analysis covariance from " + f"--{_INJECTED[name].replace('_', '-')}, which was not supplied" + ) + block = ( + np.loadtxt(supplied) + if name == "xi_reporting" + else _pseudo_cl_cov_block(supplied) ) + if part.covariance is not None: + print( + f"{name}: replacing the part's own covariance with {supplied} " + "(the analysis covariance)" + ) + part.add_covariance(block, overwrite=True) + return part def assemble_sacc( diff --git a/workflow/scripts/cv_plot_pseudo_cl.py b/workflow/scripts/cv_plot_pseudo_cl.py new file mode 100644 index 00000000..7d54ef79 --- /dev/null +++ b/workflow/scripts/cv_plot_pseudo_cl.py @@ -0,0 +1,40 @@ +"""Rule cv_plot_pseudo_cl: the EE/EB/BB pseudo-Cl figures. + +Plot-only, and an ingest like the other B-mode rules: the spectra come from the +analysis pseudo-Cl parts and their NaMaster covariances, the same pair the +summary and the terminal file are built from, so the figures cannot show +something the data products do not. +""" + +import numpy as np +from astropy.io import fits +from cv_runner import _unbuffer_streams, verify_outputs +from snakemake.script import snakemake + +from sp_validation import sacc_io +from sp_validation.cosmo_val.pseudo_cl import plot_pseudo_cl_spectrum + +_unbuffer_streams() +p = snakemake.params + +spectra = {} +for i, version in enumerate(p["versions"]): + part = sacc_io.load(snakemake.input["pseudo_cl"][i]) + ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(part, (0, 0)) + with fits.open(snakemake.input["pseudo_cl_cov"][i]) as hdul: + covs = { + name: np.asarray(hdul[f"COVAR_{name}_{name}"].data, float) + for name in ("EE", "EB", "BB") + } + for name, cl in (("EE", ee), ("EB", eb), ("BB", bb)): + spectra.setdefault(name, {})[version] = { + "ell": ell, + "cl": cl, + "cov": covs[name], + "style": {"marker": p["markers"][i], "colour": p["colours"][i]}, + } + +for name, datasets in spectra.items(): + plot_pseudo_cl_spectrum(datasets, name, snakemake.output[f"figure_{name.lower()}"]) + +verify_outputs(snakemake) diff --git a/workflow/scripts/cv_pseudo_cl.py b/workflow/scripts/cv_pseudo_cl.py deleted file mode 100644 index 22e01621..00000000 --- a/workflow/scripts/cv_pseudo_cl.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Rule cv_pseudo_cl: harmonic-space pseudo-Cl B-mode spectra. - -plot_pseudo_cl triggers calculate_pseudo_cl, which writes one SACC part per -version (EE/BB/EB with the shared bandpower window) and the cell_ee.png figure. -""" - -from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake - -_unbuffer_streams() -cv = make_cv(snakemake) -cv.plot_pseudo_cl() -verify_outputs(snakemake) From 632b8ee477da53c1f3525cbfada12bf191537ba4 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 04:54:37 +0200 Subject: [PATCH 060/160] Merge feat/sacc-4-cosmo-val-sacc into feat/sacc-6-blinding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pseudo-Cℓ figures and the B-mode summary now read the analysis part, so on a data run they bind its blinded sibling like every other consumer — the plaintext part stays temp() and unread. The blinding dry-run test asserted the un-normalised ξ± name (maxsep=300), which is exactly the producer/consumer mismatch the grid-table canonicalisation fixes; it now asserts the canonical one. The assembly tests supply the analytic ξ± covariance the terminal file takes. --- .../tests/test_blinding_wiring.py | 44 +++++++++++++++---- workflow/rules/cosmo_val.smk | 6 ++- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/sp_validation/tests/test_blinding_wiring.py b/src/sp_validation/tests/test_blinding_wiring.py index 989b5545..c5e01029 100644 --- a/src/sp_validation/tests/test_blinding_wiring.py +++ b/src/sp_validation/tests/test_blinding_wiring.py @@ -48,7 +48,7 @@ def _load_module(rel_path, name): # --------------------------------------------------------------------------- # _STEMS = [ "SP_v1.4.6.3_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=100", - "SP_v1.4.6.3_leak_corr_xi_minsep=0.08_maxsep=300_nbins=1000_npatch=1", + "SP_v1.4.6.3_leak_corr_xi_minsep=0.08_maxsep=300.0_nbins=1000_npatch=1", "pseudo_cl_analysis_SP_v1.4.6.3_powspace_nbins=32", "pseudo_cl_analysis_SP_v1.4.6.3_leak_corr_powspace_nbins=32", ] @@ -99,6 +99,18 @@ def _spd(n, seed): return a @ a.T + n * np.eye(n) +def _xi_cov(tmp_path, n_theta=6): + """The analytic ξ± covariance assembly injects, as a CosmoCov-format .txt. + + Custody is what these tests are about, and the injected covariance carries + none of it — it is a theory product — so any SPD block of the right size + stands in. + """ + path = tmp_path / "xi_cov_processed.txt" + np.savetxt(str(path), _spd(2 * n_theta, 77)) + return str(path) + + def _data_parts(tmp_path, *, conceal, one_plaintext=False, run_type="data"): """Write the five per-statistic parts, stamped ``type=run_type``. @@ -155,7 +167,12 @@ def test_data_assemble_fails_closed_on_unblinded_part(tmp_path): """A data run refuses to assemble an unconcealed real part (fail closed).""" paths = _data_parts(tmp_path, conceal=False) with pytest.raises(ValueError, match="refusing to load an unblinded"): - asm.assemble_sacc("vSYNTH", paths, str(tmp_path / "vSYNTH.sacc")) + asm.assemble_sacc( + "vSYNTH", + paths, + str(tmp_path / "vSYNTH.sacc"), + xi_cov=_xi_cov(tmp_path), + ) def test_data_assemble_passes_on_blinded_parts(tmp_path): @@ -163,7 +180,7 @@ def test_data_assemble_passes_on_blinded_parts(tmp_path): and stamps the shared commitment on the terminal file.""" paths = _data_parts(tmp_path, conceal=True) out = tmp_path / "vSYNTH.sacc" - s = asm.assemble_sacc("vSYNTH", paths, str(out)) + s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=_xi_cov(tmp_path)) assert s.metadata["concealed"] is True assert s.metadata["blind_commitment"] == _COMMIT assert s.metadata["blind_config_digest"] == _DIGEST @@ -177,7 +194,12 @@ def test_data_assemble_refuses_blinded_plaintext_mix(tmp_path): # The plaintext ξ± reporting part fails the load gate first (data + not # concealed), so the mix can never even reach assembly. with pytest.raises(ValueError, match="refusing to load an unblinded"): - asm.assemble_sacc("vSYNTH", paths, str(tmp_path / "vSYNTH.sacc")) + asm.assemble_sacc( + "vSYNTH", + paths, + str(tmp_path / "vSYNTH.sacc"), + xi_cov=_xi_cov(tmp_path), + ) def test_data_assemble_runs_behind_the_custody_guard(tmp_path, monkeypatch): @@ -195,14 +217,19 @@ def test_data_assemble_runs_behind_the_custody_guard(tmp_path, monkeypatch): paths = _data_parts(tmp_path, conceal=True) monkeypatch.setattr(blinding, "draw_scheme", lambda: 99) with pytest.raises(ValueError, match="DRAW_SCHEME"): - asm.assemble_sacc("vSYNTH", paths, str(tmp_path / "vSYNTH.sacc")) + asm.assemble_sacc( + "vSYNTH", + paths, + str(tmp_path / "vSYNTH.sacc"), + xi_cov=_xi_cov(tmp_path), + ) def test_data_assemble_stamps_the_draw_scheme_on_the_terminal_file(tmp_path): """The assembled file carries the blind's draw scheme, like its parts.""" paths = _data_parts(tmp_path, conceal=True) out = tmp_path / "vSYNTH.sacc" - asm.assemble_sacc("vSYNTH", paths, str(out)) + asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=_xi_cov(tmp_path)) assert sio.load(str(out)).metadata["blind_draw_scheme"] == blinding.draw_scheme() @@ -218,7 +245,7 @@ def test_mock_assemble_succeeds_without_a_blind(tmp_path): """ paths = _data_parts(tmp_path, conceal=False, run_type="mock") out = tmp_path / "vSYNTH.sacc" - s = asm.assemble_sacc("vSYNTH", paths, str(out)) + s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=_xi_cov(tmp_path)) assert "concealed" not in s.metadata # No escape hatch: a mock part is not gated by the fail-closed loader. assert sio.load(str(out)).metadata["type"] == "mock" @@ -351,7 +378,8 @@ def test_blinding_subgraph_in_cosmo_val_dry_run(): # blindable_part) to re-derive their concealed E-modes, so blind_part enters # the subgraph for it too. assert "_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=100_blinded.sacc" in out - assert "_xi_minsep=0.08_maxsep=300_nbins=1000_npatch=1_blinded.sacc" in out + # maxsep=300.0, not 300: the grid table canonicalises before it names files. + assert "_xi_minsep=0.08_maxsep=300.0_nbins=1000_npatch=1_blinded.sacc" in out assert "pseudo_cl_analysis_SP_v1.4.6.3_powspace_nbins=32_blinded.sacc" in out # Every blindable plaintext part is temp() on a data run, the analysis # pseudo-Cℓ included (its own rule exists so it can be). diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 3ecefa33..69829bce 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -384,7 +384,9 @@ def cv_pseudo_cl_figures(): rule cv_plot_pseudo_cl: """The EE/EB/BB pseudo-Cl figures, from the analysis parts.""" input: - pseudo_cl=[cv_pseudo_cl_analysis_sacc(v) for v in CV_VERSIONS], + pseudo_cl=[ + blindable_part(cv_pseudo_cl_analysis_sacc(v)) for v in CV_VERSIONS + ], pseudo_cl_cov=[cv_pseudo_cl_cov(v) for v in CV_VERSIONS], output: **cv_pseudo_cl_figures(), @@ -489,7 +491,7 @@ rule cv_summarize_bmodes: pure_eb=[cv_pure_eb_npz(v) for v in CV_VERSIONS], cosebis=[cv_cosebis_npz(v) for v in CV_VERSIONS], pseudo_cl=( - [cv_pseudo_cl_analysis_sacc(v) for v in CV_VERSIONS] + [blindable_part(cv_pseudo_cl_analysis_sacc(v)) for v in CV_VERSIONS] if CV.get("include_pseudo_cl", False) else [] ), pseudo_cl_cov=( From fd9431c0b68c8bc284254bcd5c1d05d3982ef915 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 16:04:07 +0200 Subject: [PATCH 061/160] chore: delete dead cosmocov drivers and repoint dangling references MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `workflow/scripts/run_cosmocov_chain.sh` invoked `cosmo_inference/scripts/cosmocov_process.py`, which was deleted on develop (#236) and relocated as the snakemake-only `workflow/scripts/cosmocov_process.py` (driven by `workflow/rules/covariance.smk`). The chain script's only caller was `papers/bmodes/scripts/run_cov_sweep.sh`, whose own only caller was a human, so both are dead end-to-end and are removed. Tree sweep for the same class of rot, repointing what survives: - README: drop the firecrown/Smokescreen override paragraph. Both files it names (`uv-overrides.txt`, `scripts/patch_firecrown.py`) were removed in f5eb417 when firecrown was dropped, and there is no `blinding` extra left in pyproject. - CLAUDE.md / CONTRIBUTING.md: the single-test example named `tests/test_cosmology.py`, which does not exist; use `tests/test_cosmo_val.py`. - CLAUDE.md: the cosmo_inference section documented a `./pipeline.sh` driver with flags. There is no such script; the pipeline is Snakemake-orchestrated, so the section now carries the invocation from `cosmo_inference/README.md`. - `papers/bmodes/config/ecut_spec.md`: `workflow/config/config.yaml` and `workflow/rules/claims.smk` moved under `papers/bmodes/`; the catalog config is `cosmo_val/cat_config.yaml`, not `code/sp_validation/cosmo_val/…`. - `papers/bmodes/scripts/update_survey_stats.py`: usage docstring still gave its pre-move `workflow/scripts/` path. Co-Authored-By: Claude Fable 5.1 --- CLAUDE.md | 14 +-- CONTRIBUTING.md | 2 +- README.md | 10 --- papers/bmodes/config/ecut_spec.md | 10 +-- papers/bmodes/scripts/run_cov_sweep.sh | 68 -------------- papers/bmodes/scripts/update_survey_stats.py | 2 +- workflow/scripts/run_cosmocov_chain.sh | 93 -------------------- 7 files changed, 15 insertions(+), 184 deletions(-) delete mode 100755 papers/bmodes/scripts/run_cov_sweep.sh delete mode 100644 workflow/scripts/run_cosmocov_chain.sh diff --git a/CLAUDE.md b/CLAUDE.md index c35c20be..b4447758 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,7 @@ Tests live in `src/sp_validation/tests/` and import the full scientific stack, so run them inside the container. - Run all tests: `pytest` (collects from `src/sp_validation/tests`; coverage on by default) - Skip the slow tests: `pytest -m "not slow"` -- Run a single test: `pytest src/sp_validation/tests/test_cosmology.py::test_function_name` +- Run a single test: `pytest src/sp_validation/tests/test_cosmo_val.py::test_function_name` CI runs this same suite inside the freshly-built image before publishing it (see `.github/workflows/deploy-image.yml`). @@ -57,11 +57,13 @@ is the container (full scientific stack pre-built). For a local dev environment: - **Healpy/HealSparse**: Sky map handling ### Cosmology Inference Pipeline (`cosmo_inference/`) -Run via `./pipeline.sh` with flags: -- `--pcf`: Calculate 2-point correlation functions -- `--covmat`: Calculate covariance matrix with CosmoCov -- `--inference`: Run CosmoSIS inference -- `--mcmc_process`: Analyze MCMC chains +Orchestrated through Snakemake, not a standalone driver; see +`cosmo_inference/README.md`. From the repository root: + +```bash +snakemake --profile workflow/profiles/candide -s workflow/Snakefile \ + inference_fiducial --configfile +``` ### Configuration Main configuration in `scripts/calibration/params.py` with parameters: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e385a386..0406741b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,7 +74,7 @@ toolchain (`autoconf`, `automake`, `libtool`, `pkg-config`) available. ```bash pytest # full suite pytest -m "not slow" # skip the slow tests -pytest src/sp_validation/tests/test_cosmology.py::test_name # a single test +pytest src/sp_validation/tests/test_cosmo_val.py::test_name # a single test ``` Tests live in `src/sp_validation/tests/`. The default options (configured in diff --git a/README.md b/README.md index 801bab9d..bbda7a1c 100644 --- a/README.md +++ b/README.md @@ -101,16 +101,6 @@ uv venv --python 3.12 uv pip install -e '.[test]' ``` -To also install the data-vector blinding stack (Smokescreen + firecrown, PRD -[#241](https://github.com/CosmoStat/sp_validation/issues/241)), pass the -dependency-override file — firecrown is not pip-resolvable without it (see -`uv-overrides.txt` for why): - -```bash -uv pip install --overrides uv-overrides.txt -e '.[test,blinding]' -python scripts/patch_firecrown.py # make pip-installed firecrown importable without NumCosmo -``` - ## Flow chart diff --git a/papers/bmodes/config/ecut_spec.md b/papers/bmodes/config/ecut_spec.md index 49da3a92..53041659 100644 --- a/papers/bmodes/config/ecut_spec.md +++ b/papers/bmodes/config/ecut_spec.md @@ -32,7 +32,7 @@ DES-Y3 used e < 0.8 to remove stars. ### How versions flow through the pipeline -Everything is driven by `config["versions"]` in `workflow/config/config.yaml`. Adding a +Everything is driven by `config["versions"]` in `papers/bmodes/config/config.yaml`. Adding a version there (plus its `cat_config.yaml` entry) makes it flow through all existing rules: `xi`, `covariance`, `pure_eb_data_vector`, `cosebis_data_vector`, `cl_data_vector`. The 2PCF and covariance are independent and can run in parallel. @@ -43,7 +43,7 @@ Key resolution functions in `workflow/Snakefile`: - `resolve_covariance_version()` — identity function (each version gets its own covariance) - Wildcard constraint: `version=r"SP_v[\d.]+(_w_iv)?(_leak_corr)?"` — needs `_ecut\d+` -Version comparison rules in `workflow/rules/claims.smk` (lines 131, 271, 386) use +Version comparison rules in `papers/bmodes/rules/claims.smk` (lines 131, 271, 386) use `VERSIONS_LEAK_CORR` for inputs and derive version lists from config in the scripts. These should be parameterized to accept a version list via `snakemake.params`, so the same rules serve both paper and ecut comparisons. @@ -71,10 +71,10 @@ uncorrected columns can't be consistently filtered to guarantee the same rows. | What | Where | |------|-------| -| Workflow config | `workflow/config/config.yaml` (search `ecut`) | -| Catalog config | `code/sp_validation/cosmo_val/cat_config.yaml` (search `ecut07`) | +| Workflow config | `papers/bmodes/config/config.yaml` (search `ecut`) | +| Catalog config | `cosmo_val/cat_config.yaml` (search `ecut07`) | | Pipeline orchestration | `workflow/Snakefile` (wildcard constraints, version resolution functions) | -| Version comparison rules | `workflow/rules/claims.smk` lines 131, 271, 386 | +| Version comparison rules | `papers/bmodes/rules/claims.smk` lines 131, 271, 386 | | Version comparison scripts | `workflow/scripts/{pure_eb,cosebis,cl}_version_comparison.py` | | Covariance params | `workflow/rules/covariance.smk` line 4 (`get_cat_params`) | diff --git a/papers/bmodes/scripts/run_cov_sweep.sh b/papers/bmodes/scripts/run_cov_sweep.sh deleted file mode 100755 index f86f20dc..00000000 --- a/papers/bmodes/scripts/run_cov_sweep.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# CosmoCov integration-grid covariance sweep over non-fiducial versions (Design B). -# -# Loops the non-fiducial version list (resolved via sweep_versions.py) and runs -# the same run_cosmocov_chain.sh chain the fiducial cov_integration_g recipe -# calls, once per version, on the 1000-bin integration grid (Gaussian-only, -# masked, blind A). Each version's processed matrix lands in a per-version subdir -# named canonically so cosebis_version_comparison (--cov-dir) and the pure E/B -# sweep can reconstruct it: -# -# /covariance__A_g_minsep=0.5_maxsep=300.0_nbins=1000_masked/ -# covariance__A_g_minsep=0.5_maxsep=300.0_nbins=1000_masked_processed.txt -# -# run_cosmocov_chain.sh writes covariance_processed.txt into --out; this driver -# renames it to the {base}_processed.txt the plotter/pure_eb sweep expect. -# -# Mask per version: v1.4.8 uses the star-halo footprint power spectrum, every -# other version uses the standard footprint (mirrors covariance.smk -# get_mask_cls_path). Covariance is recomputed per variant — it is NOT -# correction-invariant (masked v1.4.8 differs; uncorrected σ_e differs slightly). -# -# Usage: -# run_cov_sweep.sh --config --cat-config \ -# --planck18-json /planck18.json \ -# --mask-base --out \ -# [--blind A] [--versions "v1 v2 ..."] -set -euo pipefail - -. "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" - -CONFIG=""; CATCONFIG=""; PLANCK18=""; MASKBASE=""; OUT=""; BLIND="A"; VERSIONS="" -MINSEP=0.5; MAXSEP=300.0; NBINS=1000 -while [ $# -gt 0 ]; do - case "$1" in - --config) CONFIG="$2"; shift 2;; - --cat-config) CATCONFIG="$2"; shift 2;; - --planck18-json) PLANCK18="$2"; shift 2;; - --mask-base) MASKBASE="$2"; shift 2;; - --out) OUT="$2"; shift 2;; - --blind) BLIND="$2"; shift 2;; - --versions) VERSIONS="$2"; shift 2;; - *) echo "unknown arg: $1" >&2; exit 2;; - esac -done - -mkdir -p "$OUT" - -VERSIONS=$(sweep_versions "$CONFIG") - -for ver in $VERSIONS; do - base="covariance_${ver}_${BLIND}_g_minsep=${MINSEP}_maxsep=${MAXSEP}_nbins=${NBINS}_masked" - # Version dir key: strip SP_, _leak_corr, _ecutNN (mirrors get_mask_cls_path). - vdir=$(echo "$ver" | sed -e 's/_leak_corr//' -e 's/^SP_//' -e 's/_ecut[0-9]*//') - if [ "$vdir" = "v1.4.8" ]; then - mask="$MASKBASE/mask_cls_footprint_starhalo_nside_4096_norm.txt" - else - mask="$MASKBASE/mask_cls_footprint_nside_4096_norm.txt" - fi - echo "[cov_sweep] $ver (mask: $(basename "$mask"))" - bash "$WSCRIPTS/run_cosmocov_chain.sh" \ - --version "$ver" --blind "$BLIND" \ - --min-sep "$MINSEP" --max-sep "$MAXSEP" --nbins "$NBINS" --gaussian g \ - --planck18-json "$PLANCK18" --cat-config "$CATCONFIG" \ - --mask-cls "$mask" --out "$OUT/$base" - mv "$OUT/$base/covariance_processed.txt" "$OUT/$base/${base}_processed.txt" - echo "[cov_sweep] $ver -> $OUT/$base/${base}_processed.txt" -done -echo "[cov_sweep] done -> $OUT" diff --git a/papers/bmodes/scripts/update_survey_stats.py b/papers/bmodes/scripts/update_survey_stats.py index 6713b3c0..a2dc386c 100644 --- a/papers/bmodes/scripts/update_survey_stats.py +++ b/papers/bmodes/scripts/update_survey_stats.py @@ -5,7 +5,7 @@ memory usage. Usage: - python workflow/scripts/update_survey_stats.py \ + python papers/bmodes/scripts/update_survey_stats.py \ --mask-standard --mask-starhalo """ diff --git a/workflow/scripts/run_cosmocov_chain.sh b/workflow/scripts/run_cosmocov_chain.sh deleted file mode 100644 index 3a7d3a84..00000000 --- a/workflow/scripts/run_cosmocov_chain.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env bash -# CosmoCov covariance chain (lc-native, container:none recipe). -# -# covariance_ini -> covariance_cosmocov (x3 blocks) -> covariance_cat -> -# covariance_process. The CosmoCov C++ binary runs on the bare host (module load -# gcc/intelpython/openmpi); the .ini generation and cosmocov_process steps run -# inside the sp_validation apptainer container. The 3 shear-shear blocks -# (++,--,+-) are independent and run in parallel. -# -# Usage: -# run_cosmocov_chain.sh --version SP_v1.4.6.3_leak_corr --blind A \ -# --min-sep 0.5 --max-sep 300.0 --nbins 1000 --gaussian g \ -# --planck18-json /planck18.json \ -# --cat-config --mask-cls \ -# --out -# -# The checkout is this script's own (workflow/scripts/../..). Deployment paths -# come from the environment, with the current candide values as defaults: -# SPV_CONTAINER apptainer image (default /n17data/cdaley/containers/containers/) -# SPV_BIND apptainer --bind list -# COSMOCOV CosmoCov `cov` binary (also settable with --cosmocov) -set -euo pipefail - -WT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -SRC=$WT/src -CONTAINER=${SPV_CONTAINER:-/n17data/cdaley/containers/containers/} -BIND=${SPV_BIND:-/home,/scratch,/automnt,/n17data,/n23data1,/n09data} -COSMOCOV=${COSMOCOV:-/n23data1/n06data/lgoh/scratch/UNIONS/CosmoCov/covs/cov} - -VERSION=""; BLIND="A"; MINSEP=""; MAXSEP=""; NBINS=""; GAUSSIAN="" -PLANCK18=""; CATCONFIG=""; MASKCLS=""; OUT="" -while [ $# -gt 0 ]; do - case "$1" in - --version) VERSION="$2"; shift 2;; - --blind) BLIND="$2"; shift 2;; - --min-sep) MINSEP="$2"; shift 2;; - --max-sep) MAXSEP="$2"; shift 2;; - --nbins) NBINS="$2"; shift 2;; - --gaussian) GAUSSIAN="$2"; shift 2;; - --planck18-json) PLANCK18="$2"; shift 2;; - --cat-config) CATCONFIG="$2"; shift 2;; - --mask-cls) MASKCLS="$2"; shift 2;; - --cosmocov) COSMOCOV="$2"; shift 2;; - --out) OUT="$2"; shift 2;; - *) echo "unknown arg: $1" >&2; exit 2;; - esac -done - -mkdir -p "$OUT" -# Absolutize OUT before the `cd "$OUT"` below (the CosmoCov binary writes its -# blocks into cwd): every other OUT-relative path would otherwise re-resolve -# against the new cwd and double-nest. lc templates {output} as a -# project-relative path, so both relative and absolute --out must work. -OUT="$(cd "$OUT" && pwd)" -INI="$OUT/covariance.ini" - -echo "[cosmocov] generating .ini" -apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" "$CONTAINER" \ - /usr/local/bin/python "$WT/workflow/scripts/generate_cosmocov_ini.py" \ - --version "$VERSION" --blind "$BLIND" \ - --planck18-json "$PLANCK18" --cat-config "$CATCONFIG" \ - --min-sep "$MINSEP" --max-sep "$MAXSEP" --nbins "$NBINS" --gaussian "$GAUSSIAN" \ - --mask-cls "$MASKCLS" --out-ini "$INI" - -echo "[cosmocov] loading modules + running 3 blocks (parallel)" -source /etc/profile.d/modules.sh -module unload gcc 2>/dev/null || true; module load gcc -module unload intelpython 2>/dev/null || true; module load intelpython/3-2024.1.0 -module load openmpi - -cd "$OUT" -# One CosmoCov invocation per block; see common.py BLOCK_PAIRS. -for idx in 1 2 3; do - ( "$COSMOCOV" "$idx" "$INI" > "$OUT/cosmocov_block_${idx}.log" 2>&1 ) & -done -wait - -# Concatenate blocks in BLOCK_PAIRS order (++, --, +-), as covariance_cat does. -CAT="$OUT/covariance.txt" -: > "$CAT" -for pm_idx in "++:1" "--:2" "+-:3"; do - pm="${pm_idx%%:*}"; idx="${pm_idx##*:}" - blk="$OUT/cov_tmp_ssss_${pm}_cov_Ntheta${NBINS}_Ntomo1_${idx}" - [ -f "$blk" ] || { echo "MISSING block $blk (see cosmocov_block_${idx}.log)" >&2; exit 1; } - cat "$blk" >> "$CAT" -done -echo "[cosmocov] concatenated -> $CAT" - -echo "[cosmocov] processing (positive-definite check, G/G+NG extract, QA plot)" -apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" "$CONTAINER" \ - /usr/local/bin/python "$WT/cosmo_inference/scripts/cosmocov_process.py" \ - "$CAT" "$OUT/covariance_processed" -echo "[cosmocov] done -> $OUT/covariance_processed.txt (+_g.txt, +_plot.pdf)" From 7b37fe548743f42dc6a28278ab0a106e22f4d425 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 16:06:00 +0200 Subject: [PATCH 062/160] papers Snakefiles: read cat_config through code/, not the deprecated pure_eb symlink Same file as common.py's CAT_CONFIG; the symlink is slated for removal. Co-Authored-By: Claude Fable 5.1 --- papers/bmodes/Snakefile | 2 +- papers/cosmo_val/Snakefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/papers/bmodes/Snakefile b/papers/bmodes/Snakefile index c58663d6..c9069c79 100644 --- a/papers/bmodes/Snakefile +++ b/papers/bmodes/Snakefile @@ -2,7 +2,7 @@ # compute workflow at ../../workflow/. configfile: "config/config.yaml" -configfile: "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/cat_config.yaml" +configfile: "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" envvars: "PYTHONUNBUFFERED", diff --git a/papers/cosmo_val/Snakefile b/papers/cosmo_val/Snakefile index 716a3ac6..67bef0a7 100644 --- a/papers/cosmo_val/Snakefile +++ b/papers/cosmo_val/Snakefile @@ -7,7 +7,7 @@ # isolation or as the whole suite via the default `cosmo_val_all` target. configfile: "config/config.yaml" -configfile: "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/cat_config.yaml" +configfile: "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" envvars: "PYTHONUNBUFFERED", From 641a176495ea19afc0af1caafee7b919586eb9ba Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 25 Sep 2026 15:03:51 +0200 Subject: [PATCH 063/160] papers/bmodes: read the pseudo-Cl spectrum from its SACC part The generic pseudo_cl rule writes a SACC part, so the paper's C_ell consumers read that instead of the FITS PSEUDO_CELL table: one shared reader (pseudo_cl_io.load_pseudo_cl_data over sacc_io.get_pseudo_cl) for the six scripts, and the claims.smk path helper points at the .sacc part. The bandpower covariance still comes from pseudo_cl_cov's FITS. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HeMhD6yCyrgBz5oz87bCtR --- papers/bmodes/config/cl.md | 2 +- papers/bmodes/rules/claims.smk | 2 +- .../bb_covariance_blind_independence.py | 13 ++++--- papers/bmodes/scripts/cl_data_vector.py | 23 +++++------ .../bmodes/scripts/cl_version_comparison.py | 22 +++++------ .../harmonic_config_cosebis_comparison.py | 22 ++++++----- .../scripts/harmonic_space_pte_matrices.py | 16 ++++---- .../scripts/plot_cosebis_filter_overlay.py | 8 ++-- papers/bmodes/scripts/pseudo_cl_io.py | 19 ++++++++++ src/sp_validation/tests/test_sacc_writers.py | 38 +++++++++++++++++++ 10 files changed, 115 insertions(+), 50 deletions(-) create mode 100644 papers/bmodes/scripts/pseudo_cl_io.py diff --git a/papers/bmodes/config/cl.md b/papers/bmodes/config/cl.md index d178ddee..ad0b7d70 100644 --- a/papers/bmodes/config/cl.md +++ b/papers/bmodes/config/cl.md @@ -25,7 +25,7 @@ Pseudo-Cl estimation accounts for: ## Data Source Pseudo-Cl files generated by workflow rules using NaMaster: -- `pseudo_cl_{version}_blind={blind}_powspace_nbins={nbins}.fits` — Power spectrum estimates +- `pseudo_cl_{version}_blind={blind}_powspace_nbins={nbins}.sacc` — Power spectrum estimates - `pseudo_cl_cov_{version}_blind={blind}_powspace_nbins={nbins}.fits` — Bandpower covariance matrix Location: `{COSMO_VAL_OUTPUT}/` (defined in Snakefile, typically `/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/output/`) diff --git a/papers/bmodes/rules/claims.smk b/papers/bmodes/rules/claims.smk index 388de007..85891b25 100644 --- a/papers/bmodes/rules/claims.smk +++ b/papers/bmodes/rules/claims.smk @@ -121,7 +121,7 @@ def _pseudo_cl_path(version, blind="A", nbins=32): All leak-corrected versions use consistent local naming with blind and binning. """ - return f"{COSMO_VAL_OUTPUT}/pseudo_cl_{version}_blind={blind}_powspace_nbins={nbins}.fits" + return f"{COSMO_VAL_OUTPUT}/pseudo_cl_{version}_blind={blind}_powspace_nbins={nbins}.sacc" def _pseudo_cl_cov_path(version, blind="A", nbins=32): diff --git a/papers/bmodes/scripts/bb_covariance_blind_independence.py b/papers/bmodes/scripts/bb_covariance_blind_independence.py index 6d38f750..256a79b1 100644 --- a/papers/bmodes/scripts/bb_covariance_blind_independence.py +++ b/papers/bmodes/scripts/bb_covariance_blind_independence.py @@ -22,6 +22,7 @@ import yaml from astropy.io import fits from plotting_utils import PAPER_MPLSTYLE +from pseudo_cl_io import load_pseudo_cl_data from sp_validation.b_modes import calculate_cosebis @@ -436,9 +437,8 @@ def main( nbins_int, ) - # Read ell bin centers from pseudo-Cl data file - with fits.open(pseudo_cl_path) as hdu: - ell_eff = hdu["PSEUDO_CELL"].data["ELL"] + # Read ell bin centers from the pseudo-Cl SACC part + ell_eff = load_pseudo_cl_data(pseudo_cl_path)["ELL"] # Compute ratios relative to blind A pure_eb_results = {} @@ -666,7 +666,10 @@ def _from_cli(argv=None): ap.add_argument( "--cosmo-val-dir", required=True, - help="COSMO_VAL output dir (pseudo_cl / pseudo_cl_cov FITS + xi_integration txt)", + help=( + "COSMO_VAL output dir (pseudo_cl SACC parts, pseudo_cl_cov FITS " + "+ xi_integration txt)" + ), ) ap.add_argument( "--covariance-dir", @@ -711,7 +714,7 @@ def _from_cli(argv=None): f"_nbins={nbins_int}_npatch={npatch}.txt", ) pseudo_cl_path = os.path.join( - a.cosmo_val_dir, f"pseudo_cl_{version}_blind=A_powspace_nbins=32.fits" + a.cosmo_val_dir, f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" ) out_dir = Path(a.out) diff --git a/papers/bmodes/scripts/cl_data_vector.py b/papers/bmodes/scripts/cl_data_vector.py index 6ee5a93f..4eeca960 100644 --- a/papers/bmodes/scripts/cl_data_vector.py +++ b/papers/bmodes/scripts/cl_data_vector.py @@ -26,6 +26,7 @@ get_powspace_bin_edges, iter_version_figures, ) +from pseudo_cl_io import load_pseudo_cl_data plt.style.use(PAPER_MPLSTYLE) @@ -44,10 +45,8 @@ def _compute_pte_with_cuts(data, covariance, ell, ell_min, ell_max): def _load_pseudo_cl_data(pseudo_cl_path, pseudo_cl_cov_path): - """Load pseudo-Cl data and covariance from FITS files.""" - hdu = fits.open(pseudo_cl_path) - data = hdu["PSEUDO_CELL"].data - hdu.close() + """Load pseudo-Cl data from SACC and covariance from FITS.""" + data = load_pseudo_cl_data(pseudo_cl_path) ell = data["ELL"] cl_eb = data["EB"] @@ -163,7 +162,7 @@ def _create_cl_figure( # so the version sweep is self-contained (lc produces only the fiducial version). # --------------------------------------------------------------------------- def _pseudo_cl(results_dir, ver, blind="A", nbins=32): - return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.fits" + return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.sacc" def _pseudo_cl_cov(results_dir, ver, blind="A", nbins=32): @@ -184,10 +183,9 @@ def _resolve_pseudo_cl_paths( """Resolve the pseudo-Cl and covariance paths for one version in the sweep. Every version is read from the reconstructed COSMO_VAL pattern, except the - fiducial version when explicit override paths are supplied: the lc-produced - fiducial FITS files are named `pseudo_cl_{version}.fits` / - `pseudo_cl_cov_{version}.fits` (no blind=/nbins= tokens), so pattern - reconstruction can't find them and an explicit path is required instead. + fiducial version when explicit override paths are supplied: lc-produced + fiducial SACC spectrum and FITS covariance files have untagged names, so + pattern reconstruction can't find them and explicit paths are required. """ if ( ver == fiducial_version @@ -333,7 +331,10 @@ def _from_cli(argv=None): ap.add_argument( "--results-dir", required=True, - help="COSMO_VAL output dir with per-version pseudo_cl_* / pseudo_cl_cov_* FITS", + help=( + "COSMO_VAL output dir with per-version pseudo_cl_*.sacc and " + "pseudo_cl_cov_*.fits" + ), ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") ap.add_argument( @@ -349,7 +350,7 @@ def _from_cli(argv=None): "--fiducial-pseudo-cl-path", default=None, help=( - "Explicit path to the fiducial version's pseudo-Cl FITS (lc output). " + "Explicit path to the fiducial version's pseudo-Cl SACC part (lc output). " "Requires --fiducial-version and --fiducial-pseudo-cl-cov-path." ), ) diff --git a/papers/bmodes/scripts/cl_version_comparison.py b/papers/bmodes/scripts/cl_version_comparison.py index cd283f2a..f70a6977 100644 --- a/papers/bmodes/scripts/cl_version_comparison.py +++ b/papers/bmodes/scripts/cl_version_comparison.py @@ -29,12 +29,13 @@ get_version_alpha, version_label, ) +from pseudo_cl_io import load_pseudo_cl_data plt.style.use(PAPER_MPLSTYLE) def _pseudo_cl(results_dir, ver, blind="A", nbins=32): - return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.fits" + return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.sacc" def _pseudo_cl_cov(results_dir, ver, blind="A", nbins=32): @@ -55,10 +56,9 @@ def _resolve_pseudo_cl_paths( """Resolve the pseudo-Cl and covariance paths for one version in the sweep. Every version is read from the reconstructed COSMO_VAL pattern, except the - fiducial version when explicit override paths are supplied: the lc-produced - fiducial FITS files are named `pseudo_cl_{version}.fits` / - `pseudo_cl_cov_{version}.fits` (no blind=/nbins= tokens), so pattern - reconstruction can't find them and an explicit path is required instead. + fiducial version when explicit override paths are supplied: lc-produced + fiducial SACC spectrum and FITS covariance files have untagged names, so + pattern reconstruction can't find them and explicit paths are required. """ if ( ver == fiducial_version @@ -131,10 +131,7 @@ def main( fiducial_pseudo_cl_cov_path, ) - hdu = fits.open(pseudo_cl_path) - data = hdu["PSEUDO_CELL"].data - hdu.close() - + data = load_pseudo_cl_data(pseudo_cl_path) ell = data["ELL"] cl_bb = data["BB"] cl_eb = data["EB"] @@ -379,7 +376,10 @@ def _from_cli(argv=None): ap.add_argument( "--results-dir", required=True, - help="COSMO_VAL output dir with per-version pseudo_cl_* / pseudo_cl_cov_* FITS", + help=( + "COSMO_VAL output dir with per-version pseudo_cl_*.sacc and " + "pseudo_cl_cov_*.fits" + ), ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") ap.add_argument( @@ -395,7 +395,7 @@ def _from_cli(argv=None): "--fiducial-pseudo-cl-path", default=None, help=( - "Explicit path to the fiducial version's pseudo-Cl FITS (lc output). " + "Explicit path to the fiducial version's pseudo-Cl SACC part (lc output). " "Requires --fiducial-version and --fiducial-pseudo-cl-cov-path." ), ) diff --git a/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py b/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py index f8025a0e..dd34db22 100644 --- a/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py +++ b/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py @@ -32,6 +32,7 @@ iter_version_figures, version_label, ) +from pseudo_cl_io import load_pseudo_cl_data from sp_validation.b_modes import calculate_cosebis @@ -107,11 +108,10 @@ def _compute_harmonic_cosebis( pseudo_cl_path, pseudo_cov_path, nmodes, theta_min, theta_max ): """Compute COSEBIS from pseudo-C_ell and propagate covariance.""" - with fits.open(pseudo_cl_path) as hdul: - data = hdul["PSEUDO_CELL"].data - ell = np.asarray(data["ELL"], dtype=float) - cl_ee = np.asarray(data["EE"], dtype=float) - cl_bb = np.asarray(data["BB"], dtype=float) + data = load_pseudo_cl_data(pseudo_cl_path) + ell = np.asarray(data["ELL"], dtype=float) + cl_ee = np.asarray(data["EE"], dtype=float) + cl_bb = np.asarray(data["BB"], dtype=float) cosebis_obj = COSEBIS(theta_min, theta_max, nmodes) @@ -857,7 +857,10 @@ def _from_cli(argv=None): ap.add_argument( "--cosmo-val-dir", required=True, - help="COSMO_VAL output dir (pseudo_cl / pseudo_cl_cov FITS + xi_integration txt)", + help=( + "COSMO_VAL output dir (pseudo_cl SACC parts, pseudo_cl_cov FITS " + "+ xi_integration txt)" + ), ) ap.add_argument( "--covariance-dir", @@ -883,8 +886,9 @@ def _from_cli(argv=None): "--fiducial-pseudo-cl-path", default=None, help=( - "Explicit path to the fiducial 96-bin pseudo-Cl FITS reproduced by lc " - "(from lc's cl_bandpowers_fine; e.g. pseudo_cl_SP_v1.4.6.3_leak_corr.fits), " + "Explicit path to the fiducial 96-bin pseudo-Cl SACC part reproduced " + "by lc (from lc's cl_bandpowers_fine; e.g. " + "pseudo_cl_SP_v1.4.6.3_leak_corr.sacc), " "overriding the --cosmo-val-dir pattern lookup for --fiducial-version." ), ) @@ -934,7 +938,7 @@ def _from_cli(argv=None): if is_fiducial and a.fiducial_pseudo_cl_path else os.path.join( a.cosmo_val_dir, - f"pseudo_cl_{ver}_blind={a.blind}_powspace_nbins={cosebis_nbins}.fits", + f"pseudo_cl_{ver}_blind={a.blind}_powspace_nbins={cosebis_nbins}.sacc", ) ) # 96-bin pseudo-Cl covariance is intentionally NOT lc-repointed: lc did not diff --git a/papers/bmodes/scripts/harmonic_space_pte_matrices.py b/papers/bmodes/scripts/harmonic_space_pte_matrices.py index f52e1bc8..aeb031c6 100644 --- a/papers/bmodes/scripts/harmonic_space_pte_matrices.py +++ b/papers/bmodes/scripts/harmonic_space_pte_matrices.py @@ -26,12 +26,13 @@ make_pte_colormap, make_pte_norm, ) +from pseudo_cl_io import load_pseudo_cl_data plt.style.use(PAPER_MPLSTYLE) def _pseudo_cl(results_dir, ver, blind="A", nbins=32): - return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.fits" + return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.sacc" def _pseudo_cl_cov(results_dir, ver, blind="A", nbins=32): @@ -48,7 +49,7 @@ def compute_pte_matrix( Parameters ---------- pseudo_cl_path : str - Path to pseudo-Cl FITS file. + Path to pseudo-Cl SACC part. pseudo_cl_cov_path : str Path to pseudo-Cl covariance FITS file. fiducial_ell_min : float, optional @@ -66,10 +67,7 @@ def compute_pte_matrix( Summary statistics. """ # Load pseudo-Cl data - hdu = fits.open(pseudo_cl_path) - data = hdu["PSEUDO_CELL"].data - hdu.close() - + data = load_pseudo_cl_data(pseudo_cl_path) ell = data["ELL"] cl_bb = data["BB"] n_ell = len(ell) @@ -533,7 +531,9 @@ def _from_cli(argv=None): ap.add_argument( "--results-dir", required=True, - help="COSMO_VAL output dir with per-version pseudo_cl_* / pseudo_cl_cov_* FITS", + help=( + "COSMO_VAL output dir with pseudo_cl_*.sacc parts and pseudo_cl_cov_*.fits" + ), ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") ap.add_argument( @@ -544,7 +544,7 @@ def _from_cli(argv=None): ap.add_argument( "--fiducial-pseudo-cl-path", default=None, - help="Explicit path to fiducial pseudo-Cl FITS produced by lc " + help="Explicit path to fiducial pseudo-Cl SACC part produced by lc " "(overrides pattern reconstruction for --fiducial-version)", ) ap.add_argument( diff --git a/papers/bmodes/scripts/plot_cosebis_filter_overlay.py b/papers/bmodes/scripts/plot_cosebis_filter_overlay.py index 8de785b3..68b255ab 100644 --- a/papers/bmodes/scripts/plot_cosebis_filter_overlay.py +++ b/papers/bmodes/scripts/plot_cosebis_filter_overlay.py @@ -21,16 +21,16 @@ from astropy.io import fits from cosmo_numba.B_modes.cosebis import COSEBIS from plotting_utils import PAPER_MPLSTYLE +from pseudo_cl_io import load_pseudo_cl_data plt.style.use(PAPER_MPLSTYLE) def load_bb_data(pseudo_cl_path, pseudo_cov_path): """Load BB bandpower data and errorbars.""" - with fits.open(pseudo_cl_path) as hdul: - data = hdul["PSEUDO_CELL"].data - ell = np.asarray(data["ELL"], dtype=float) - bb = np.asarray(data["BB"], dtype=float) + data = load_pseudo_cl_data(pseudo_cl_path) + ell = np.asarray(data["ELL"], dtype=float) + bb = np.asarray(data["BB"], dtype=float) with fits.open(pseudo_cov_path) as hdul: cov_bb = hdul["COVAR_BB_BB"].data diff --git a/papers/bmodes/scripts/pseudo_cl_io.py b/papers/bmodes/scripts/pseudo_cl_io.py new file mode 100644 index 00000000..b7b1c796 --- /dev/null +++ b/papers/bmodes/scripts/pseudo_cl_io.py @@ -0,0 +1,19 @@ +"""Read pseudo-Cℓ parts for the B-modes paper workflow.""" + +from sp_validation import sacc_io + + +def load_pseudo_cl_data(path): + """Return a pseudo-Cℓ part's spectra as ``{"ELL", "EE", "EB", "BB"}`` arrays. + + The bandpower covariance is a separate FITS product (``pseudo_cl_cov``). + Loading goes through ``sacc_io.load``, so an unblinded real-data part is + refused. + """ + ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(sacc_io.load(str(path)), (0, 0)) + missing = [name for name, values in (("BB", bb), ("EB", eb)) if values is None] + if missing: + raise ValueError( + f"{path} lacks required pseudo-Cℓ spectra: {', '.join(missing)}" + ) + return {"ELL": ell, "EE": ee, "EB": eb, "BB": bb} diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index 59ea333d..c4963b33 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -8,6 +8,9 @@ NaMaster round-trip proves the pseudo-Cℓ window survives the writer path. """ +import importlib.util +from pathlib import Path + import numpy as np import pytest @@ -114,6 +117,41 @@ def get_bandpower_windows(self): assert window.weight.shape == (24, nbp) +def _paper_pseudo_cl_reader(): + # A paper script, not a package module: load it by path. + path = Path(__file__).resolve().parents[3] / "papers/bmodes/scripts/pseudo_cl_io.py" + spec = importlib.util.spec_from_file_location("pseudo_cl_io", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod.load_pseudo_cl_data + + +def test_paper_pseudo_cl_reader(tmp_path): + """The B-mode paper's reader maps NaMaster's EE/EB/BE/BB rows by name.""" + ell = np.array([30.0, 60.0, 90.0]) + cl_all = np.vstack( + [np.arange(3) * 1e-9, np.arange(3) * 2e-9, np.zeros(3), np.arange(3) * 3e-9] + ) + + class _Workspace: + def get_bandpower_windows(self): + window = np.zeros((4, 3, 4, 12)) + for component in range(4): + for band in range(3): + window[component, band, component, band * 4 : (band + 1) * 4] = 1.0 + return window + + part = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell, cl_all, _Workspace()) + path = tmp_path / "pseudo_cl.sacc" + sio.save(part, str(path), type="mock") + + data = _paper_pseudo_cl_reader()(path) + assert np.array_equal(data["ELL"], ell) + assert np.array_equal(data["EE"], cl_all[0]) + assert np.array_equal(data["EB"], cl_all[1]) + assert np.array_equal(data["BB"], cl_all[3]) + + def test_pseudo_cl_to_sacc_real_namaster(tmp_path): """A real small-nside NaMaster workspace's window survives the writer.""" pytest.importorskip("pymaster") From 848ab080a10f7b007f6bb119c2d6915ef2310c37 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 25 Sep 2026 15:36:45 +0200 Subject: [PATCH 064/160] Drop generate_cosmocov_ini.py and the README install section generate_cosmocov_ini.py was deleted on develop (#236) and came back with this branch's stale-base rebuild; nothing calls it. The README "Local Installation" section repeats docs/source/installation.rst and is not part of the SACC migration. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HeMhD6yCyrgBz5oz87bCtR --- README.md | 9 -- workflow/scripts/generate_cosmocov_ini.py | 149 ---------------------- 2 files changed, 158 deletions(-) delete mode 100644 workflow/scripts/generate_cosmocov_ini.py diff --git a/README.md b/README.md index f857fb91..1d269a0c 100644 --- a/README.md +++ b/README.md @@ -92,15 +92,6 @@ the profile puts each job in the container itself. For Docker, development installs, and more depth, see the [installation docs](https://cosmostat.github.io/sp_validation/installation.html). -## Local Installation - -Requires Python ≥ 3.12. With [uv](https://docs.astral.sh/uv/): - -```bash -uv venv --python 3.12 -uv pip install -e '.[test]' -``` - ## Flow chart diff --git a/workflow/scripts/generate_cosmocov_ini.py b/workflow/scripts/generate_cosmocov_ini.py deleted file mode 100644 index 6996c04f..00000000 --- a/workflow/scripts/generate_cosmocov_ini.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Generate a CosmoCov ``.ini`` for one (version, blind, grid, flavour, mask). - -Cosmology comes from the frozen ``planck18.json`` snapshot, survey parameters -(area, n_eff, sigma_e) from the catalog config's per-version ``cov_th``, and -n(z) from ``workflow/common.build_redshift_path``. The footprint mask power -spectrum is passed explicitly (empty string for the unmasked variant). - - python generate_cosmocov_ini.py \ - --version SP_v1.4.6.3_leak_corr --blind A \ - --planck18-json /planck18.json \ - --cat-config \ - --min-sep 0.5 --max-sep 300.0 --nbins 1000 --gaussian g \ - --mask-cls \ - --out-ini -""" - -import argparse -import importlib.util -import json -import os -import sys - -import yaml - - -def _load_workflow_common(): - """Load ``workflow/common.py`` (this script also runs outside Snakemake).""" - path = os.path.join( - os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "common.py" - ) - spec = importlib.util.spec_from_file_location("workflow_common", path) - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -common = _load_workflow_common() - - -INI_TEMPLATE = """\ -# -# Cosmological parameters -# -Omega_m : {Omega_m} -Omega_v : {Omega_v} -sigma_8 : {sigma_8} -n_spec : {n_s} -w0 : -1 -wa : 0 -omb : {Omega_b} -h0 : {h} - - -# Survey and galaxy parameters -# -# area in degrees -# n_gal,lens_n_gal in gals/arcmin^2 - -area : {area} -sourcephotoz : multihisto -lensphotoz : multihisto -source_tomobins : 1 -lens_tomobins : 1 -sigma_e : {sigma_e} -source_n_gal : {n_e} -lens_n_gal : {n_e} - - -shear_REDSHIFT_FILE : {nz} -clustering_REDSHIFT_FILE : {nz} -c_footprint_file : {mask} - - -# IA parameters -IA : 1 -A_ia : 0.0 -eta_ia : 0.0 - - -# Covariance parameters -# -# tmin,tmax in arcminutes -tmin : {min_sep} -tmax : {max_sep} -ntheta : {nbins} -ng : {ng} -cng : {ng} - - -outdir : ./ -filename : cov_tmp -ss : true -ls : false -ll : false -""" - - -def main(argv=None): - ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) - ap.add_argument("--version", required=True) - ap.add_argument("--blind", default="A") - ap.add_argument("--planck18-json", required=True) - ap.add_argument("--cat-config", required=True) - ap.add_argument("--min-sep", required=True, help="tmin arcmin (string, e.g. 0.5)") - ap.add_argument("--max-sep", required=True, help="tmax arcmin (string, e.g. 300.0)") - ap.add_argument("--nbins", required=True, help="ntheta (string, e.g. 1000)") - ap.add_argument("--gaussian", required=True, choices=["g", "ng"]) - ap.add_argument( - "--mask-cls", default="", help="footprint mask Cl path ('' = unmasked)" - ) - ap.add_argument("--out-ini", required=True) - a = ap.parse_args(argv) - - with open(a.planck18_json) as f: - cosmo = json.load(f) - with open(a.cat_config) as f: - cat_config = yaml.safe_load(f) - - cov_th = cat_config[common.base_version(a.version)]["cov_th"] - - ng_value = "1" if a.gaussian == "ng" else "0" - - ini = INI_TEMPLATE.format( - Omega_m=cosmo["Omega_m"], - Omega_v=cosmo["Omega_v"], - sigma_8=cosmo["sigma_8"], - n_s=cosmo["n_s"], - Omega_b=cosmo["Omega_b"], - h=cosmo["h"], - area=cov_th["A"], - sigma_e=cov_th["sigma_e"], - n_e=cov_th["n_e"], - nz=common.build_redshift_path(a.version, a.blind), - mask=a.mask_cls, - min_sep=a.min_sep, - max_sep=a.max_sep, - nbins=a.nbins, - ng=ng_value, - ) - - os.makedirs(os.path.dirname(os.path.abspath(a.out_ini)), exist_ok=True) - with open(a.out_ini, "w") as f: - f.write(ini) - print(f"Wrote {a.out_ini}") - - -if __name__ == "__main__": - main() From 31d8b7705105f3a9ed7db9a5aab6a7e7f500f996 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 25 Sep 2026 20:57:49 +0200 Subject: [PATCH 065/160] =?UTF-8?q?tests:=20pin=20pure-E/B=20on=20exact-bi?= =?UTF-8?q?nning=20and=20committed=20=CE=BE=C2=B1;=20drop=20dead=20catalog?= =?UTF-8?q?ue=20paths?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The synthetic pure-E/B pins passed in CI and failed on candide because TreeCorr's default bin_slop/angle_slop make ξ± follow the tree's top-level split, which varies with the jackknife patches and, through min_top, with the thread count TreeCorr takes from cpu_count(). Fixed patch centres alone leave a 4-vs-48-thread spread (reporting ξ− up to 16%); exact binning removes it (1e-12). The test measures with bin_slop = angle_slop = 0, and test_b_modes pins pure_eb_from_xi on the same ξ±, committed as tests/data/pure_eb_xi_fixture.npz. The configured-path guard skips cat_config's paths.output and directory-less calibration params.input_path values. The two LFmask entries (data gone) and the six unread covmat_file keys leave cat_config.yaml, and the slow duplicate test_catalog_paths_exist goes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- cosmo_val/cat_config.yaml | 88 -------- .../tests/data/pure_eb_xi_fixture.npz | Bin 0 -> 16564 bytes src/sp_validation/tests/test_b_modes.py | 88 +++++++- .../tests/test_config_paths_exist.py | 19 ++ src/sp_validation/tests/test_cosmo_val.py | 212 ++++-------------- 5 files changed, 151 insertions(+), 256 deletions(-) create mode 100644 src/sp_validation/tests/data/pure_eb_xi_fixture.npz diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index a4d34bee..218ba413 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -914,7 +914,6 @@ SP_v1.4.11.3: e2_star_col: HSM_G2_STAR shear: R: 1.0 - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des @@ -960,7 +959,6 @@ SP_v1.4.12.3: e2_star_col: E2_STAR_HSM shear: R: 1.0 - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.12.3/unions_shapepipe_cut_struc_2024_v1.4.12.3.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des @@ -1006,7 +1004,6 @@ SP_v1.4.13.3: e2_star_col: E2_STAR_HSM shear: R: 1.0 - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.13.3/unions_shapepipe_cut_struc_2024_v1.4.13.3.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des @@ -1072,7 +1069,6 @@ SP_v1.4.6.3_uncal: subdir: /n17data/UNIONS/WL/v1.4.x shear: path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec e1_col: e1_uncal @@ -1088,7 +1084,6 @@ SP_v1.4.6.3_uncal_w_iv: subdir: /n17data/UNIONS/WL/v1.4.x shear: path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec e1_col: e1_uncal @@ -1104,7 +1099,6 @@ SP_v1.4.6.3_uncal_w_1: subdir: /n17data/UNIONS/WL/v1.4.x shear: path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec e1_col: e1_uncal @@ -1160,88 +1154,6 @@ SP_v1.4.8_uncal: path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 patch_number: 100 -SP_v1.4_LFmask_8k: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: black - getdist_colour: 0.0, 0.5, 1.0 - ls: solid - marker: d - cov_th: - A: 2137.7977618140676 - n_e: 7.976462506484096 - n_psf: 0.5434016250405327 - sigma_e: 0.31509572849714534 - psf: - PSF_flag: HSM_FLAG_PSF - PSF_size: HSM_T_PSF - star_flag: HSM_FLAG_STAR - star_size: HSM_T_STAR - hdu: 1 - path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: HSM_G1_PSF - e1_star_col: HSM_G1_STAR - e2_PSF_col: HSM_G2_PSF - e2_star_col: HSM_G2_STAR - label: SP_LFmask_psf - shear: - R: 1.0 - path: unions_shapepipe_extended_2022_v1.4.0_mtheli8k.fits - w_col: w - e1_col: e1 - e1_PSF_col: e1_PSF - e2_col: e2 - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - patch_number: 150 -SP_v1.4_LFmask_8k_noalpha: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: brown - getdist_colour: 0.0, 0.5, 1.0 - ls: solid - marker: d - cov_th: - A: 2137.7977618140676 - n_e: 7.976462506484096 - n_psf: 0.5434016250405327 - sigma_e: 0.31509572849714534 - psf: - PSF_flag: HSM_FLAG_PSF - PSF_size: HSM_T_PSF - star_flag: HSM_FLAG_STAR - star_size: HSM_T_STAR - hdu: 1 - path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: HSM_G1_PSF - e1_star_col: HSM_G1_STAR - e2_PSF_col: HSM_G2_PSF - e2_star_col: HSM_G2_STAR - label: SP_LFmask_psf - shear: - R: 1.0 - path: unions_shapepipe_extended_rmalpha_2022_v1.4.0_mtheli8k.fits - w_col: w - e1_col: e1_cor - e1_PSF_col: e1_PSF - e2_col: e2_cor - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - patch_number: 150 nz: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/nz dndz: diff --git a/src/sp_validation/tests/data/pure_eb_xi_fixture.npz b/src/sp_validation/tests/data/pure_eb_xi_fixture.npz new file mode 100644 index 0000000000000000000000000000000000000000..fa67b006d5fae626006bc7670bcafba9e7558242 GIT binary patch literal 16564 zcmd6OXH*r-x-B{9oLS_YbLy|iIU|T58384W7zm016%Y)dhyep)LIgoUlvPSpK@b!a zPy|sxlH?!)kA2VCZ@hiReZTIX+iUdtM)&Hus%F>M)m5{5y4X|Gs8IZCk)ZhYu)}`i z-wOu?4@E+FXhNWWd}wS;e1d9p?0!lL4vMmWAN_Yc=d{Jq#ha2Mks?VhBy@LhyqqCP z&UlBuoH9vn$0ja5Fxo#RJ|y(7{Z@gIyF)kk?+y=)4c*+9*Hu3mh!LW+WVRV56TSe9wPH&NbHo5h-Vk`nzQ0r5j zT0!_HU#$wtrhoE{`d`jhLa)o`?d@fPGg2^p^wMMUg+0FrLwG@6`W#Ck|K&9~#o(7+ z*xg|=mwkD9@YG%6pM0~MM``}%%ldyaIwJc2rZeLHS3dt8|7Yo}uKqtvXeMV{BvNpU4g#WbPkXUMStVYIe4r6 zti<5P9LVXHUf1lJgPR>rW!I?Yp;7l?>3f}dxR}U0{xWDD>`h!o+Op^2OX*%&y_R`! zS2j?sn4Jff*Mac;9((k^$_(UJW2g-?x48{ zx0)FxJ9jO@z%An)@x_ZEwv;qK{b4f}hn3FoEy3fUT}e|`OOVJ`wjp+W2^5U7Z?@Jh z!S83pZl0AT7(KeZwnk$a&iD)$<%ciB6x-;C+}&mHABm>w9a)AD?h686#aBQh(rLmr zcm+6bZN0Yl(h5)%4jvBcS%IPXqe5ryp?YhblAx8}IuI#_hoEhdGmLn7q^_2CEW z;FA5+^VFYpIJAv(i-gSvppc@pmYfaf4j>sk|GWV$E+=()1u2lvkjXTgF$Jmy9N`xa4!aTGn(b3K-mp|T==xWr{ zr@|*Fk>9f=e%eY(R21Xt_~bJs;*BUCOk|`&EWiAz<#njgZN8xLRX-}k_vno8R0V`JyTZKzTAh5fAFd#RCL#8(uBMUwNtP zM{3mTt|wVUPlMiH`o2)0OM^lV+KpV?NrRFbb5sspq(R%ewA=T*p+S;=R`wfi&>+^^ zR;FjQXpvs3V|i#8Ei$dWb-*T{7X5to*irL6Evi10U&%*9hs5hkbvupekeq$B1&*ad zez$l+s_)Vvwnt(8-CyYt-#+5E7au(mkPff1aHdBd+Qwh|kI^Gfm%4@Q7xbvD^^$qw zZ+cYoL&R5EmjNaHZY~~ zm6r+`QE|HZ6#Eb(nx#dj?#ii;ink-1rET9} zJue*{n0ZiML!OzeArCrrzuRMa7Y{0||0vddl?O>sXH96o<3XzzwhabQ^P#IvQJOHQ5J0&c4>QxI1<+JdTUW5MAWGC0p|OY%M6c-<Z`CL6QV8h%D2sx^L%`8ZngX@A0UUP8`iD zbRJE07e^wDmwcZTilexLICjgVIQsa#WWh|7gzje0sK2u$p_={Q)NeN z@-p5*#ZM-oTiu->ltxHM-+D2754!|P>@DV2F_1t3#@@yQArh#UvF=bvwgd{!cr?8A zsRUZ%ZuUjrB~W3MmHJ0^N#v+K_uJT568&m8U2!;E66GGbcWp3F5(R{u*j8RIiQ4UR ztAl1F5sjDe)E+S@Btp-{^U_fYeW=Q>IFTxaz8UbzM3+b*g+lh0>d#WhFSG9416FDD zw!1-V2S_86Ju!WJanfigr#VadhBSI2xN%(SgESf!Y!KjMkU?G?I`le*GKhFl)ano; zgYviiaG$;@gC5F995L^aLF8qaw_=k;ojfB$aptlp${R*V92vbEk9S)zb!b}3C?9alg{ zuTKYjS5id%ug{qG?NUUAKRl@=N)*x8AT|^*sfbXgIEhmoN2>A0~nQb?Djk24S~4# z zU7XoNF{X|z{7-jgb8Dc2cjSkrCK~7ubByN6Fb$;c#VWLLMgtjMiR(hO8fb35#M^6B z16|V?H|gWnMD)}K@fVRM(mtcDJ`|~mR-f7q@4l#s+KuRcb2n=uEm!lr_=U|V{q%B& zloq11r@egAO$$v(bwr*`(?V;OzVqhgT4=BBLRa{p7E&FNZ;<8EM$*Oyc59kzqY{OX z=r;-4=)l_gAopfp++4bK4@-|WvNOlemJqno}hyQay>F#?`-aW zZECXZiw>Fz@#fjXrHd4ltwot^bP@kowvPE!UGy_vmB;0YF3M8e8pb@Si-IB@1!QFO z(534^`=+<+p%~gTKNT+MA&-$@?eq>k^dr6Xlqs`5@?#tgG_lf0*2nL>xtykta-P&m z3e@VOUM}wn#ozkq#C9!}MqLAR%E|MzNP+>Pk!9~Gcw~UmP0-Mp83W|9Euvaj!w@~+ zLlhas8KSp>R);>78zMIS14(pqhG>!IZNznLBgAi~B))I25xU`qg6L|D(1^@{?b4bN z!fm=$!sfJ`lJ|@U+Cr{n28zw07gl$jm zxCsi)Pn2lTCXgNxE#Z+$pf@fFA@~)6j6YvZ*~n|bHW850CwE-b!>-VdflL+l!Nsrg! zH$@jNCg&s~Q)J~n;PHN^DYAUZX?7yd6fFhnJ#~1o8SUoIH;$X4nQt@))A`NNw0qt* zb_+8kSP*wAE!GT$DQJgXykdqPlOn}TTFekM%XJ=GF++OK?sg=|nWOXbY*R{}=19nB zP3Yb+bHwaNpR((*Il5TSyS?g%If{P9IxNj+f!6PG@x@tNpbyFsECKs1km$vyhqm0e zK+Xk;X<1(^5L4vL{CN&bG%sSWB4S~QzJ`9k$G^`K@z7++<(F8ZjZ-I3$)F`VzSqo{ zhTjUMvs>%aIa;9$HBBrhj##0CU1NofHC8B28qq}W-;n39L5e`{GqCl!EiFW8GeHRr+ zl^qIIIm8>WY=`hgZ{Z?Cd(@PAEx{3>Bo;;Efbw6jh_u=`pt7=< zexYm!BzbgE@BMoR^zP2qFH}N~NHF_qVOXFeveU49A$`{oP0}QANX5c+idd3Q$x})XBTEmH-?&$R6?8rM1F4}BCmHLj}_ma1u%sF^fh3CFI#7xqN- z5utA5uAZoe>xN3!2~VV2R=8AyJ<+T1J3aPGo~SD7RcO4L7ji4QrbZp^h5qarjSjx< zg_f?`eI4!fLOPQfDek;mkX>y|}Zz;Ey+|F5T}yGTw$hJZAyrhLLkeH;3nULhESe2{1Ju3p9?KBy_+4$IaiA9PeDTrY*m7s;`g@1A$^MbiyCkICoy zqI6PG(?X9gx^RV-%bv6y9ViZ@d>OVKP295C-SKcc3g=SFHDBJ26r4ncPMi9n;Vuid zf=oYzj@)#M>-Iy3nOIEDN%$jG6QVXZ${#V3oPyhG{Lz;O9`}W40?@c#a;vja06M2} zHKD~V04eKM4%Hk8K+=9Haa=b8&X0*9CnXCZ!Qp>zSwv(O*{xGoLP09un$57yG&Oz z_64DYjGJ4@w}OxcTs-dlJ_v=Y*-Qjd2BZ62{Z1`v!RWd1@2>ZL!N^EDev&^s8112> zr5UXaM&E-!Nm)(>qxp^3?E_*VNTh2&7vU0uI9u*oET@H_WXR59t_(rJRFl_AhC@(? z0wZO;U?}R)`Bk{nF%$(*_eGyg3q=JxTc|~!gd$(IjAiBVP_({a_-2T-10@AH)irtU zKxBFkn(R|M&2+}!VsK`- zdUhZT<=h@+`XCgJN|(-+f7lX^1bn{59Xu0`K0d6si+UT5@S$#7Ev5+cHesdB);aLZZ#;dAQ8sdl2*H$u+nnC(PQ=)x^c4(~+mQ&lF8^*a%tY4z?x znq8>$i078)R=d#koU-YyCwC$7M3?B^)?H|R`S{Knj!1OTD$HDBOC*Y<4eKl_h(sGf z1yzRzA`v~O;E6x7QHXMdHOnqK3h^DDv)=wB3T5|I-`rS_Li#Cmg$g#&NPOL6Z^_wc z^RT^ED)Z>L;gV2DK>cP|^Ssm7vN8Wpv4 z&selkr(1S8H5S=MZkP-e$D$)|`=~nGW6|{P^L#nWv4}?flE+rbI5g2ZaZtq}4xQFD zq>fCCL&>*C7=y0Ip@c9l-^A89L=z*U@?8P*_f`G_>_2* z_0-_A69`v=}4i1d9FB3m_L#xzV z(~?FXh?ToL#bcueM`q0*lr3oBg3=4OKOS)hjpeaNV_vFwDbH6YL(c$&U!HsVs>>2n zXjUaZP&R>wR>cgoBUU(~Z|YNboiDy^KQq`+>jFy06saG6(#2`7z5i?p3W6_bBTFAI zo8a1AvK60PY;aO%>|;M&FE~>-qg;O%VM7bODC0Y}Ac*%nOiuWKbq()Q@wg>+8x%~h z@Nfhk#gQ-HBRpU^a6A1iL3i-i=r0@ybB9Jje?NyfL9D7t*68fqf+>xco{K6=;zzDD zS~oczL9E1!Qpj5tGuLQNnW$T1)2I4P>E=4H^z9qNjW`{APb5u)dY=YV3QaZCIdbDi zJl~k3oi*|7gn35kxC3~{Py0B``hw}s(gYHZ0WgyEr%NRTaU;jx54h6}E@T#19QmvU zC7t>|oGC1@%Po%2Pt#1H-|AUPRh2F5()soI9@4_BD-9DY=Mm)X2 zHg1+>9a&S*s!cpA{YMvThPbiBxR_(h8ylZ=ln{_O@=^|;7R7?*bfak;+E4Y^(smmF}14;t5`R4n0AmY+}F`FF}IP6M>5JL(hK3#iCeJDo( z&c*BTxZSY;)fBI`kpsF|FqiH4;a8UUcs1R^c5PcsePZU8Ww<_c(PYjhKQje&4JL<4 zvN344HxGaEwt?y9v)7L1Xv0|*#oA+^=7_6t^qs<61%cPAP-5w*EEX>BIohv0LsTS3 z#UIGE1;uWCo}42F5I!W&9@ndZFR&#)efdZk&lS@xdEErOC}f<9ESO>C z)T!@&lX_rLd-0ChJx!PqZsg^#v;k_?<42^A8RJ{^kFTF?Gsh>F*=}TTkzhNK&!2rs z8DCWV8m%O$0_CKX-Pz|g@yV|SzU=~%xVGcu`${@3&{I)Q_Ea$k1=C)|16)clI+JlX z##9qiQA}K;%5nuMhC((@cXMz(5vXOBZ-6Ivxt%=jp@ElGg|CQKS%B+vhu7Pd32^IP z;J&z19yjn@e8SDdi;EqP4#yaha1{^N-X>uid}`&|t5i!XI4(L667Ru>B^=yekv_=6 zlxau&$MZax|LLn+nVZ)`c78$lK#DG?<@};fE>Z^XJujG&U+Y8EllU`x0}a7Yf8txt z9D$KAZWSeDOyFgkeXOs{9IFZW=Bh`a%XSCH}@5MWY z%knzFbT{*HA-!M)NgwGA!$AZ}7%vR~ylu$QA2YE8 zwht9teaQk4caSHK|F|5if2>|QE6NC$HM$1hXb8h__1@8|Gjh;-(JgMdkqL}@cA0s+ z`A+Z?aT+rBMd0)=zh6@g)DZvFyXSTdH>7ll=>6t=PcEwcR&!;ueLQW>E$^4pf+?nf z@`fxD+$}4*5ST5Cy-o{xFMte4p9c%J6m>}PtfK^eZMdh$sBzZ^0B%)p%)aM^xe~72 zf};kIYm~iX%3J|HT^Ew^U8RC?dP!D&H64)59DMitu{3lYzJEBnR0j02`1M4ywc*IF z7j?V`I3e&|qhG}tWjM$*vLDN+!uKa_j`6!FvBcR&_63{$(&O4K+=8ouV4I#d|FoY3 zzkIiwoK)b%gNg@fwD)L3;XJZ>m|_S*>nR_!H>hwU`N}@E&kC@zJpOT>TLT!uBxRaf z4;0QUotR0H2J`6J$FF{|V4WWsTXg0%A%5zCpGO}XJUlrmINYfYqLC(BPF!P#!X%Hp zIW=ae{!o_1Evx`{zP=v&C1U`tQt1jjO}yCSgWtNOoeunYp5q&H<{P=g;41Z_DJoo^ z-_0B$uK}IxN1LZa<)G;l``7^q63`A%2JYbDgIAwKZCT+pS$xNb%8ynOa6rm9h$_?& zHkNn!Ot;biRhq=UVvNAEAL)uPoAeuBgK4U8A2K?v;bq(+nIx zH~DzbykWj(s1DS7RT+-dD#C?_u^z_HRN>K<>6hGJ6v1+|?DHtP#&Mybu@8Ww8xSIbIJ-h9|RwvhzP)eoFo0_5P~ zQPv(RHwvJQzTL`wOa@}nH&tFHX$aaTt2y^h15VnpoUbpC0WHsf_~K2z&YAZ|jdrR5 zdwD5~1-A@TN%q(6oKu4caRy!693?naF2~kq0A7FZLLJ{RCB0bGv+Nxr|OfLi}mw!v#@V3QkEjNfb*+tBx)Z(NcC zzPRRfzb=$^!3#NEJJ$g<-_+{BmQf0^FFjy7^gG8eUXq zS=VJSz&9zj=X1%@U{Ldn+aQ@A6iys`JAOzQEZ!Cd7SGYcjN(tVOUEVPaZB~|`(v_T zc_7cof<+0=%6@;SC@2Hzu6w((gf`F5Bgu!nKEklZd^}I1L<-_Fxu2CXOF_c%(_2d< zMPS5QJmZ))55%UIajJghhT9`fomV!`Kgrm~MJ=5K7Gm1UAs-YC;aGvfmtRei5Ll*M+QK9Ovf20JWaLR8uH1VrH(L_&3@{*jpQ5>EEO5y;Bhun4 zFErj}{zYpo2`{(~?f*!{2j`D`{dmoq1J3VN>Sy003@0gu+fNOVfROqWovc8DFm3NP zM=M_7`@LXCq7Va(K~L(PYQm5{xc*j_fg4=oOz3?XmWbTCmj?{gXyLvX1$V?|`ydhG z!t|UFoU7b1%8?=t&2$B?Jy*HGPW_QNLkS!7(w&*vP0I!v=!5jU_|^Pcz=Jch4 z8;0mD%w;W~Ns;m27fcoP)JIeuxL*!iBXfABjsocl2Ms*Iin66F+c?j-2pXAIxR zYjg|{AYAmi){_m^POnIOlB0zi?-vsf*^QCCD@WTeEi4eD?WqFSROukfIeg1h0xg(W z3;)hr9VCWfzPinj3MMos_f*QQ66CC?NiiM{06`H!6Tx4^Vn?qVZ5Aa+{-jwJ6rCn^ z-0O_#F?dZrEnjtTZ$%|B%DdB%HR3ZdTgBtJt7(cj%sVeBdUJ}rqr&3UE89haxjVx7 z?!r9znvJS1y<0!oRek#h|I7iRx@Ij%%%`7VJkKimET)%un$F`DQ}d0mJ)KaTzdS&m znvlx9)UZh2=NZbrG(SUJ*tpQKdzb=;?0s(h@y9s9W|2`>V*HjYT|n)1+LA9_HYkV$APnCJApGUS?z=53EBzDrle8w7OC8ndd>~9r0nj{9m2Gj z#=x~X&tZ(9QfxE!%45V8a&RjL7O4|83Xs?!+wREEsr|`;{Y0p)S1EAdquEP&-3%P~K|BX5%L#7$ zj8SyS{}>DA6tR)C^`^m}i5nJqI!yREJR7vgXk4v$zjfEI2$N>@7nMH{zW>*DLu(8ir~!tsXw2Ds%Tv+{Q#JKk?O@}H3cl{t0%&v)PAZV}{vif>W=|BJ=9-|RZw$&#TEbV5$s8@XXs zeVZgB215`OlPQ~%*a}fpe|n|(w&1O8{h|F%?(mZ>L-X}ZH#qv9M@?>a2RIDsrV!0e zxNNjAVM{9lzY|{hi+Y+^T9|5SGTQ~t8f^JguFr+Lo_lW3qjkah{*Hl~0}c?HJKXf7 z#t8Fse;T+IjPP*j*~fn1j00$bF3$06!OeF^f4m`Y#VI?>9e2+QVUeS`Ia4nXj92f! z^~2W}G;=%}Wex>lqv0K ztY(Gy5xyNQQ{45~6M_iEXA~khkHh6BbywN0biM*6nI&=5C zG;H(feWfk-))I!Z2AIQ zw@AXAM(xQFA9JYxH8FD#8$kW8s}-pnmiR?mU!;Ge0c6r8 zofG1dgsr77LVUWk;LmF5qq<3_&B?MqSt$$Q1NRj~l&z$2uD1HwSKFOHSWoL)b6x7l4+^Rr#(^kR4yILgiSN-5!PQ1L})oY+|CsGDa|41xYl2ymFSNT78VLNEps?eL; zqy;($Z1o>!>R<-1ZkmiffVq>>)t6tH;jOARWlH{bcyD1_#M?a^WFh>m_B*=53JX8A&%A50DQ@xI_%=#-VcnB0CK}IW;D<6}dB+-pG3G>Xj|EeFRMF?qdtL(=Ub#3C zdQ%wqKR7wq6dFS8%``sREDxM`qjjIFsA7t{6_3>MXZ1JqKOsOm|Q3{Icx;s z3TBkkc?c-`#ta_uap31|BFPq2)>x+BXQ=g=2!ywN%sjNQL|%>PTJolvCv#hUCQkiO z#9Xn1bkrW|@O5eVOQE+D`s`0xh@3jSNa4lU~Y-EEG!}_K(vIJ11n7*3i zb-?7p>xXm{jPQ=)?H(uF)uH3pIof(VbNs0F^OFrdX;|dzOiP_Gfd;k9sef9maO4?} zidF>+P}@4Uv30vJJ|cTb=+0+d2uO~SJ11cVS|1IapIb}fl^|M+-*x=hX(d&@&RZSy zG|c2viQ8GZ-Wr-4z#r>*2ODlnTu+lNkR-0h0*~AEvVQ>L#DnW z2A0JyL+bV@f;emAahfU(IKrIMEdCtf(o?Ek)CaZk{m*j9d0rL_xsEZv$yLIP84O8D z{tO^!b)VmR^Jd|`(;z-YOd9T`ZBq{2oWyZyR%T3Bm;rK~HPBjA!HaJDJ6oq{Abv ziS&QA@BlAa;xJzy7dF2+tgSw;59G6b*+L4k&}K6~V_P8%;?NdtPHPGZC)a-24gDtb zKO5kT@e_g({nBfpn-4RnjC&MsC2k(i%G925Lmj+YzTR&gen74rJC#S9O$i@aWgN_x z7YP5H;~iX^Lyn$P-X~7=$>Yv>DqzG47#8)H?JYNm@JF}Y6t7TYwXvDd%PYosV*{V8 z)1d>Gwy-~%UuEGdnJMnh#024`scaR+MvGI2Usd%vielEUqGm~CSs1J|2_M=e1h(d8 zXA@HviH&aRET4UPcI=8S9qFCS9E&I=y5*$G0&u)?G_KF|8|Z=eQA)KY?(_+ zeXI(-rn{3{`xxQUrE=Pn+a++xMYZ7X`*^XWWrmQ~7z3>Q$@h=iY;R^+ze8giESPDR z39;>h0%Rz=6czLcVXDQNFQfWBL{w6l$IfgWsAKQe(?ZU>{$86`wWqwRQ)Xmgz-@uwTg`G5BU*8{4XDf#Tca}<9P$1j!cx! zm$#$FzZ~2?@x53kjBI)bCm2hK$C0$ow8=Dh;yLSCzL80y2@XA{wBvr3NkQZ8QAS^fBt&?9ATlDS~Xg; zPA2WQq*SpI#2b@graKn6@n;6gEZO`iqH{g2*T#SmJ{Ov0y$Em_TdypsYy` zC$@NOewc|~7$e(1lbC`5bMm?zev4akUUIPFbEWk~?|Q!w0rz*& zOWv1;m=M*WRE=e_c*d76!mnnC?g!MK?A-jYe}2}*gGmHC&%5Wx>t z%=#K<#NkXc-M&lDC2^H`&5o&50VsayxFK+%kKlP#Y0k~f0i@G8$|lQi$$}O&`3HkX z$rc7KYQ@2WhbVNO0&$NBeO1BH8tMVD+)K0ph^Lxn?I9S{!_<{!S%5 zH5`~6x+_k+N*^}+Mc3apF@;h2V^B`wBtXbct z&h?H0%Qb(AY<$EIV=w)jWnI{?@{h{k`fV)W#oQ(|@nMZz@Q6Z_|K})K`xlK)9Z{Z~RV{F6HR?d1``iFSnE1 z#zF~mmpuR6d^tj1Q=9iad0zkvg&eQ@EWiTivfsHWhW#YBP_R_p`0$vV%{#K2xrPBh zKC~DIbu5^DqVz|)Og&)_nc3egC}EKElz?a;75wNr_;=$Kf zL{#|wu5PD0g#KegMP@#F9Orm$kHf{=J+jXKvdisyvyLIAV zcVA^r;ym$srx<&CCN1V>*EYC(V6*%qQJiTO6yViI&Bzl*fk`z14~r72$c%&isH}U1 z>}raiu}syHpG-Hpwd4&DE!)gXuLZ8p%O9J*V{NzY&WO zCBBKrUJ$C@ENOY~8Q}N&U7G71z7x9|0d z8HE@P5U!1xZ`Y)j2-iFHI?s0eAm3$3(^dZbjy$~WP9Im#3nIo-`M!z7JRz@WZ%;aVzRZqDzrMMCcW)~^-=>%^yfk_=6} zKghqX{bHf*o+aPfT|(fqG_a8$zwWbEOQ?(*i$>6l6Q%Z_9M|{$B(jQ*jvtO5Ad0?U zdzK+X1K(fjxxG<+PE`8ea|sVWP9rWbm*Ta?|tS$b-M2- zd7)SC_=iIq#K2LS2g`b^L~lekM_PL~A<{NYeC)3zo<~1e=;jYL{c)elUQ1x|P9 zD|pNj`9B_q_xp_z`)a2u=q|q_S5*q8U0fI-1dI+}f3ztoa&5WjacpxGB=KZ7shxeD zP`nprFk$|UT&{#|dApa$&EDHN1Wxu4EMZiOE7UWDcFW#p)V1~G;!BwnE4P-(F?3$) z5lpQ_{1{KuRn|G;KGss_d)Y%?YSk*sGFl+QXUO~4d_ECdJktd2C1=V0^nR7R7wO>& z&JtC5{+{p`QCaRhR6)MPG-g*UH%ESalOAWcERcQfIqdu_wnz}aO8YxbQNh8|<2vIe zzsMrW)rW^BTL_h?yTS^6-w3qUZ4?^*l?dLo<$$aDIGIoWP3K#~yX5AS#BAlnIij+; zsK-@qmJB1g7s{g!5rZ`4(zU!xL{m(C2YbypAu!T#a<^pM-8iRlelB+$h8Q-Fww%LqZ82l|H^#t37=-BelO1#z$BrnV>f9g%wVihr+57jY5w znY%MELhopWjn&!!AwhjuIX1S7sC@C6M>_XAq56ye>8SoN5$|#nUtF0a3;TK`zu3A? zl=1HovkjUc4zqkN*!Q}Q{M`RXV*WTI&{&C77Y4#Hu-`BuoYPe=Gqm#z}LN$u$r6TA>|@71Tv>*GX0 zeCwk0^cRA?HR&+v90h#GM=tA&(843xF87=KGlclqRLR9^M!0?RHd$Ad19C4*Jj-OE z2leT+J?S5NiCLHS2}OB6(0+f2V@!(;D7!LRlW1As*e6-$t3zCy&cLMahlMyH%W=iH zwuu3n6MYRIyx|7Z8Raf*xjy3B%{@tbOe7$4#^_|%4HAIPlM91x!khE_<6fqF1t7V0 zig*5k8q7R;cPnRI5eWEnCHsOBq$Nbw9_Ua3(T*`g{T3}~AA8R>?4||5PB*G9a%lkQ zwHU5#j@wIZ#JkN}H6cvj`p^=u4h*S;)$41j|A*0b{a}Cm-{Kp?U&hm@i0Hp9)F=LJ zq5eDmk36UUQv)@L#6L34;LEygstqlH)z#-R729#7P)c-WJ|7BGF zYvuT_gZ}k(i1M%apZ50uME_+D{~OKr5A^Kj)<3P{|B3y}X#L+~zipPiFXI2TUH>Qc zFVpg0e&oMa)_;`SN%eQF|1ISHC-kouF8_uy{{!mE_pd7de}I39#s599P2pel{r>>} zO7y=H`LC7XA0^tGY*zN)Y5$+~{42TsdlW127wTWB=3-Av_t#%CY<}7|Ph0!uZxj^& E2NTeo6#xJL literal 0 HcmV?d00001 diff --git a/src/sp_validation/tests/test_b_modes.py b/src/sp_validation/tests/test_b_modes.py index 67c64376..42d632fa 100644 --- a/src/sp_validation/tests/test_b_modes.py +++ b/src/sp_validation/tests/test_b_modes.py @@ -1,8 +1,9 @@ """VALUE-DRIFT CHARACTERIZATION TESTS FOR THE B-MODE ESTIMATORS. This module pins the numeric behavior of the pure E/B-mode helpers in -``sp_validation.b_modes`` against fixed, deterministic, in-memory inputs -(seeded RNG and hand-built arrays — no cluster data, no catalogue files). +``sp_validation.b_modes`` against fixed, deterministic inputs (seeded RNG, +hand-built arrays and one committed ξ± fixture — no cluster data, no catalogue +files). Every pinned literal was produced by an actual run of the estimator inside the container; a future refactor that changes the numbers must fail. @@ -15,6 +16,7 @@ """ import types +from pathlib import Path import numpy as np import numpy.testing as npt @@ -293,6 +295,88 @@ def test_calculate_eb_statistics_has_teeth(): assert loud_pte < 0.05 # louder B-modes are clearly rejected +# --------------------------------------------------------------------------- +# 5. pure_eb_from_xi on committed ξ± (the transform pin) +# --------------------------------------------------------------------------- + +_PURE_EB_XI = Path(__file__).parent / "data" / "pure_eb_xi_fixture.npz" + +# pure_eb_from_xi(**fixture); regenerated only when the transform is meant to move. +_PURE_EB_PINS = { + "xip_E": [ + -2.9831529669542025e-06, + -1.5008524620265777e-05, + 3.221623968725757e-07, + 1.1797672310858565e-05, + 5.715510692557323e-06, + 8.825804523824443e-07, + ], + "xim_E": [ + -4.737558091773235e-05, + -0.00010853189443993388, + -9.094825175032069e-05, + -5.826599101284694e-05, + -4.646405415748759e-05, + -1.9978028925333273e-05, + ], + "xip_B": [ + 1.7069121242262332e-05, + 3.059889782373755e-05, + -4.8805399253844115e-06, + -6.999262696335271e-06, + -1.2672006989728095e-05, + -1.214149138979614e-06, + ], + "xim_B": [ + -0.00011478091634539627, + -5.445112002141066e-05, + -3.100806652947907e-05, + -1.0940424256759085e-05, + -5.755185146643215e-06, + -1.628217762504557e-06, + ], + "xip_amb": [ + 0.00014017621792612224, + 0.0001378482153667787, + 0.0001339573019551001, + 0.00012745126271361765, + 0.00011662385105911986, + 9.851844704443032e-05, + ], + "xim_amb": [ + -4.389203999135455e-05, + 5.279277664928643e-05, + 5.800339397836051e-05, + 4.4242350610114584e-05, + 2.9902912946755567e-05, + 1.912262132836568e-05, + ], +} + + +def test_pure_eb_from_xi_reproduces_pins_on_committed_xi(): + """The pure-E/B transform of committed ξ± reproduces its pins. + + The fixture is the exact-binning ξ± that the synthetic pure-E/B test in + ``test_cosmo_val`` measures (reporting [15, 70]′ in 6 bins, integration + [1, 300]′ in 600 bins), keyed by ``pure_eb_from_xi``'s parameters. With ξ± + frozen, these pins move only when the transform does: when that test fails + and this one passes, the drift is upstream of the transform. rtol=1e-6 is + far above the 1e-12 reduction-order noise across thread counts. + """ + pytest.importorskip("cosmo_numba") + with np.load(_PURE_EB_XI) as fixture: + xi = {k: (v.item() if v.ndim == 0 else v) for k, v in fixture.items()} + + modes = b_modes.pure_eb_from_xi(**xi) + for key in b_modes._EB_KEYS: + npt.assert_allclose(modes[key], _PURE_EB_PINS[key], rtol=1e-6, err_msg=key) + + # Teeth: widening the integration interval by 1% leaves the pins. + moved = b_modes.pure_eb_from_xi(**{**xi, "tmax": 1.01 * xi["tmax"]}) + assert not np.allclose(moved["xip_E"], _PURE_EB_PINS["xip_E"], rtol=1e-6, atol=0) + + # --------------------------------------------------------------------------- # 6. Grid edges and the COSEBIs covariance seam # --------------------------------------------------------------------------- diff --git a/src/sp_validation/tests/test_config_paths_exist.py b/src/sp_validation/tests/test_config_paths_exist.py index 2db03bc1..38a269ec 100644 --- a/src/sp_validation/tests/test_config_paths_exist.py +++ b/src/sp_validation/tests/test_config_paths_exist.py @@ -182,12 +182,31 @@ def _config_files() -> list[Path]: ] +def _located_elsewhere(source: Path, key: str, value: str) -> bool: + """Whether a path-shaped value names nothing the config itself locates. + + ``paths.output`` in the catalogue config is where cosmo_val writes, created + by the run. A calibration ``params.input_path`` with no directory is opened + relative to its consumer's run directory (the image-sim run, or the + catalogue directory ``scripts/masking.py`` prefixes). + """ + if source.name == "cat_config.yaml": + return key == "paths.output" + return ( + source.parent.name == "calibration" + and key == "params.input_path" + and "/" not in value + ) + + def _candidate_paths() -> list[tuple[Path, str, Path]]: root = _repo_root() candidates = [] for config_path in _config_files(): iterator = _iter_ini_paths if config_path.suffix == ".ini" else _iter_yaml_paths for source, key, value, base_dir in iterator(config_path): + if _located_elsewhere(source, key, value): + continue expanded = Path(value).expanduser() if expanded.is_absolute(): resolved = expanded diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index f25ed522..33df76e3 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -9,9 +9,7 @@ """ import os -from collections import defaultdict from pathlib import Path -from typing import Dict, Iterator, Tuple import numpy as np import pytest @@ -182,88 +180,6 @@ def test_additive_bias_leak_corrected_columns(self, base_config): assert isinstance(cv.c1[version_leak_corr], float) assert isinstance(cv.c2[version_leak_corr], float) - @staticmethod - def _iter_catalog_entries(config: Dict[str, Dict]) -> Iterator[Tuple[str, Dict]]: - """Yield (name, entry) pairs for catalog-like entries in the config.""" - for name, entry in config.items(): - if not isinstance(entry, dict): - continue - if "subdir" not in entry: - continue - yield name, entry - - @staticmethod - def _resolve(base: Path, candidate: str) -> Path: - """Return an absolute path given a base directory and a candidate string.""" - candidate_path = Path(candidate) - return candidate_path if candidate_path.is_absolute() else base / candidate_path - - @pytest.mark.slow - @requires_catalog_data - def test_catalog_paths_exist(self, base_config): - """Verify that catalog paths for active versions exist on disk. - - This is a lightweight test that checks that all files referenced in the - catalog configuration for UNIONS analysis versions actually exist. It - discovers versions programmatically from cat_config.yaml rather than - using hardcoded lists. - """ - # Get the path to catalog config - repo_root = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(__file__))) - ) - catalog_config_path = os.path.join(repo_root, "cosmo_val", "cat_config.yaml") - - config = yaml.safe_load(Path(catalog_config_path).read_text()) - - # This integrity check needs the real catalogs on disk (cluster only). - # Skip where the data directories aren't mounted — e.g. CI running - # inside the docker image, which has cat_config.yaml but no catalogs. - if not any( - Path(entry["subdir"]).is_dir() - for _, entry in self._iter_catalog_entries(config) - ): - pytest.skip("catalog data directories not present (not on cluster)") - - working = [] - nonfunctional = defaultdict(set) - - for version, entry in self._iter_catalog_entries(config): - # Skip nz entries and versions already tested in heavy tests - if version == "nz": - continue - - base = Path(entry["subdir"]) - version_missing = set() - - # Check shear, star, and psf files - for block_name in ("shear", "star", "psf"): - block = entry.get(block_name) - if not block: - continue - resolved_path = self._resolve(base, block["path"]) - if not resolved_path.is_file(): - version_missing.add(block_name) - - if version_missing: - nonfunctional[version] = version_missing - else: - working.append(version) - - # Print summary - print(f"\n✓ Working versions ({len(working)}):") - for v in sorted(working): - print(f" - {v}") - - if nonfunctional: - print(f"\n✗ Non-functional versions ({len(nonfunctional)}):") - for v in sorted(nonfunctional.keys()): - print(f" - {v}: missing {nonfunctional[v]}") - - assert not nonfunctional, ( - f"Catalog configuration references missing files: {dict(nonfunctional)}" - ) - def test_seed_variant_updates_shear_path(self, tmp_path): """Seeded versions should materialize a seed-specific shear path.""" params, base_version = self._make_seed_config( @@ -347,10 +263,9 @@ def test_v1_4_6_glass_mock_default_seed(self, base_config): # toy catalog written to disk, asserting that sp_validation wires the # catalog/config/estimator together correctly and that the chain produces # output of the right shape with finite values. They do NOT re-test the - # underlying numerical libraries (treecorr, cosmo_numba): no specific - # numerical values are asserted. These are the back-pressure that catches - # config-path / wiring breakage during restructuring. They can be tightened - # to allclose-against-a-committed-reference later for value-drift coverage. + # underlying numerical libraries (treecorr, cosmo_numba); only the pure-E/B + # test pins values. These are the back-pressure that catches config-path / + # wiring breakage during restructuring. # # Environment-independent: the catalog is synthesized in a tmp dir, so no # cluster data is needed. They do require the scientific stack (treecorr, @@ -558,45 +473,23 @@ def test_calculate_scale_dependent_leakage_runs_on_synthetic_catalog( assert hasattr(res, "C_sys_p") and hasattr(res, "C_sys_m") def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path): - """calculate_pure_eb wires xi+/- into cosmo_numba's Schneider E/B split. - - Integration test of the headline B-mode seam: the pure E/B/amb - decomposition runs end-to-end via cosmo_numba and returns vectors of the - configured length, all bins finite, with a jackknife covariance of the - right shape -- AND the deterministic mode vectors match pinned reference - values, so a refactor that silently changes the numerical B-modes fails - rather than staying green on finiteness alone. - - Two layers of teeth: - - 1. Finiteness on EVERY reporting bin (not just the interior). The - Schneider (2022) pure E/B estimator evaluates singular kernel - integrals (Eq. 42-43, 55-56) at each reporting theta, integrating the - fine ``gg_int`` xi+/- over [tmin, tmax]. At the extreme reporting bins - the evaluation point sits at the integration boundary, where the - integrand is near-singular; a *coarse* integration grid fails to - resolve it and the mode goes NaN. The real bmodes workflow - (papers/bmodes/config.yaml) avoids this with a broad-and-fine grid -- - reporting [1, 250] arcmin, integration [0.5, 300] with nbins_int=1000 - -- so the integration range brackets the reporting range AND the grid - is fine enough that the boundary integrals converge. This test mirrors - that: reporting [15, 70] arcmin, integration [1, 300] arcmin (brackets - on both ends) with nbins_int=600. Confirmed directly that nbins_int~80 - over this range NaNs the last xip_E bin and the first xim_E bin, so - coarsening the integration grid back toward ~80 reintroduces edge NaNs - and fails -- this is the finiteness teeth. - - 2. Value-drift pins on the four deterministic mode vectors (xip/xim, - E/B). These come from a seeded synthetic catalog -> full-sample - treecorr xi+/- (no RNG) -> Schneider linear transform, so they are - reproducible. Verified bitwise-stable across two separate container - processes to a worst-case relative drift of ~1.4e-11 (pure float64 - reduction-order noise; absolute drift ~1.5e-17). The pins use - rtol=1e-6 / atol=1e-12 -- ~5 orders of magnitude above that float-noise - floor (no flakiness margin consumed) yet tight enough that a sub- - percent change in any mode bites. The jackknife COVARIANCE depends on - treecorr's kmeans patch assignment and is NOT pinned by value -- only - its shape is asserted. + """calculate_pure_eb carries ξ± through cosmo_numba's pure-E/B split. + + Every reporting bin is finite, and the four mode vectors match pins. + + Finiteness: the Schneider (2022) integrals are near-singular where a + reporting bin meets the integration boundary, so the integration grid + [1, 300]′ brackets the reporting grid [15, 70]′ on both ends and is fine + (600 bins); about 80 integration bins NaN the edge bins. + + Pins: TreeCorr's default bin_slop/angle_slop approximate separations from + its tree, whose top-level split follows the jackknife patches and, through + min_top, the thread count TreeCorr takes from cpu_count(). On this + catalogue that moves the reporting ξ− by up to 16% between 4 and 48 + threads. Exact binning makes ξ± a plain pair sum, so the pins move only + when sp_validation does; rtol=1e-6 is far above its 1e-12 reduction-order + noise and far below a sub-percent change in any mode. The E/B transform + alone is pinned on fixed ξ± in ``test_b_modes``. """ pytest.importorskip("treecorr") pytest.importorskip("cosmo_numba") @@ -615,13 +508,8 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path): nbins=nbins, **params, ) + cv.treecorr_config.update(bin_slop=0, angle_slop=0) - # Integration range strictly brackets the reporting range [15, 70] on - # both ends (1 << 15, 300 >> 70) AND uses a fine grid (nbins_int=600), so - # the near-singular boundary-bin Schneider integrals converge. This - # mirrors the bmodes workflow's broad-and-fine integration grid; every - # reporting bin is well-defined (no edge NaNs). nbins_int~80 here would - # NaN the edge bins -- confirmed -- which is the finiteness teeth. results = cv.calculate_pure_eb( version, npatch=npatch, @@ -630,49 +518,46 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path): nbins_int=600, ) - # Reference mode vectors from the seeded synthetic catalog + Schneider - # transform. Deterministic (full-sample treecorr, no RNG); regenerate by - # running calculate_pure_eb with the setup above and printing repr() of - # results[key]. Tolerances justified in the docstring. + # Regenerate by printing repr(results[key]) from the setup above. expected = { "xip_E": np.array( [ - 1.6688018692521218e-06, - -1.8392317186434428e-05, - 1.4170916007248522e-06, - 8.1454486560987474e-06, - 6.2050467269160570e-06, - 2.6649478149110497e-06, + -2.9831529669572382e-06, + -1.5008524620255579e-05, + 3.221623968699465e-07, + 1.1797672310854472e-05, + 5.715510692580715e-06, + 8.825804523810145e-07, ] ), "xim_E": np.array( [ - -4.4552381788304276e-05, - -1.1082898248960663e-04, - -9.2495668600755951e-05, - -5.8456322151105526e-05, - -4.4270469941501174e-05, - -2.4236697154723798e-05, + -4.7375580917647536e-05, + -0.00010853189443992296, + -9.094825175031718e-05, + -5.826599101284305e-05, + -4.6464054157485714e-05, + -1.997802892533382e-05, ] ), "xip_B": np.array( [ - 1.8508958599700601e-05, - 3.8264056862769537e-05, - -1.0482698132038303e-05, - -7.3081832089716533e-06, - -9.1621105374021936e-06, - -6.4075815485457576e-06, + 1.706912124226553e-05, + 3.059889782372767e-05, + -4.880539925382135e-06, + -6.999262696331131e-06, + -1.267200698975249e-05, + -1.2141491389775202e-06, ] ), "xim_B": np.array( [ - -1.1129938750754923e-04, - -4.7967477760986883e-05, - -3.4334760596175194e-05, - -1.4776328993077835e-05, - -4.0078671892721522e-06, - -8.3202301900417799e-07, + -0.00011478091634543392, + -5.4451120021397075e-05, + -3.100806652947136e-05, + -1.0940424256752644e-05, + -5.755185146639151e-06, + -1.628217762504009e-06, ] ), } @@ -680,10 +565,7 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path): for key in ("xip_E", "xim_E", "xip_B", "xim_B"): vec = np.asarray(results[key]) assert vec.shape == (nbins,) - # All reporting bins are well-defined under the widened integration - # range (no edge NaNs) -- the finiteness teeth. assert np.all(np.isfinite(vec)), f"{key} not finite" - # Value-drift pins -- the deterministic-mode teeth. np.testing.assert_allclose( vec, expected[key], @@ -693,8 +575,6 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path): ) # Jackknife covariance over the 6 stats (xip/xim x E/B/amb) x nbins. - # Patch (kmeans) assignment isn't guaranteed deterministic, so only the - # shape is pinned, not the values. cov = np.asarray(results["cov"]) assert cov.shape == (6 * nbins, 6 * nbins) assert results["n_eff"] == npatch From 4fa7f3920f6092ace820fa0925538f279891e59d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 25 Sep 2026 21:31:53 +0200 Subject: [PATCH 066/160] =?UTF-8?q?cosmo=5Fval:=20pin=20TreeCorr's=20min?= =?UTF-8?q?=5Ftop=20so=20=CE=BE=C2=B1=20does=20not=20depend=20on=20the=20m?= =?UTF-8?q?achine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TreeCorr derives min_top, the depth of its root cells, from its thread count (max(3, ceil(log2 n)), with n from cpu_count() when unset). The root cells set which pairs bin_slop approximates, so at default slop calculate_2pcf's ξ± depended on the node: on the synthetic catalogue, 48 threads move ξ± by 0.038σ against 4. The shared treecorr_config pins min_top = 6, which is what TreeCorr derives on candide's 48- and 64-CPU nodes; the same config reaches the aperture-mass, leakage and ρ/τ correlations. test_calculate_2pcf_does_not_depend_on_thread_count measures at production binning on 4 and 48 threads (fresh Catalog each, shared patch centres) and requires agreement below 1e-6σ; it is red without the pin. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/cosmo_val/core.py | 5 ++++ src/sp_validation/tests/test_cosmo_val.py | 34 +++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 7da61279..fa3a6ecb 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -313,6 +313,11 @@ def __init__( "nbins": nbins, "var_method": var_method, "cross_patch_weight": "match" if var_method == "jackknife" else "simple", + # min_top sets the depth of TreeCorr's root cells, hence which pairs + # bin_slop approximates. Left unset, TreeCorr derives it from its + # thread count (max(3, ceil(log2 n))) and ξ± depends on the machine. + # 6 is what TreeCorr derives on candide's 48- and 64-CPU nodes. + "min_top": 6, } self.catalog_config_path = Path(catalog_config) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 33df76e3..b48f8f9d 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -437,6 +437,40 @@ def test_calculate_2pcf_runs_on_synthetic_catalog(self, tmp_path): # The additive-bias subtraction in the pipeline must have run. assert version in cv.c1 and version in cv.c2 + def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): + """calculate_2pcf's ξ± is the same on 4 and on 48 TreeCorr threads. + + Production binning (default bin_slop/angle_slop), both runs on the + jackknife patches the first one writes, each from a fresh Catalog; they + must agree to far below the jackknife σ. + """ + import treecorr + + params, version = self._write_synthetic_catalogs( + tmp_path, n_gal=4000, coherent_shear=True + ) + cv = CosmologyValidation( + versions=[version], + npatch=8, + theta_min=15.0, + theta_max=70.0, + nbins=6, + **params, + ) + + xi = {} + for n_threads in (4, 48): + # calculate_2pcf reads back an existing text dump instead of measuring. + for dump in Path(params["output_dir"]).glob(f"{version}_xi_*.txt"): + dump.unlink() + gg = cv.calculate_2pcf(version, num_threads=n_threads) + assert treecorr.get_omp_threads() == n_threads # the count took effect + xi[n_threads] = np.concatenate([gg.xip, gg.xim]) + sigma = np.sqrt(np.concatenate([gg.varxip, gg.varxim])) + + shift = np.max(np.abs(xi[48] - xi[4]) / sigma) + assert shift < 1e-6, f"ξ± moves by {shift:.3g}σ between 4 and 48 threads" + def test_calculate_scale_dependent_leakage_runs_on_synthetic_catalog( self, tmp_path ): From b1c9e7407b364dd81e14d8f0cd76f5e8c2b24016 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 25 Sep 2026 21:32:59 +0200 Subject: [PATCH 067/160] b_modes: one pure-E/B kernel call; tests pin each pure-E/B fact once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit calculate_pure_eb_correlation (modes and jackknife) and pure_eb_covariance_mc call cosmo_numba through pure_eb_from_xi, so the transform test_b_modes pins is the one every pure-E/B product runs through. The committed ξ± fixture is now a conftest fixture (pure_eb_xi) with two users. The synthetic pure-E/B test asserts that the ξ± and edges it measures equal the fixture (agreement 2e-13), and that its modes are pure_eb_from_xi of them; this replaces its copy of the mode pins, and np.savez of what it measures regenerates the fixture. test_b_modes pins pure_eb_from_xi on the fixture, which fails loudly if cosmo_numba does not import, with no skip. A failure now names what moved: the measured ξ±, the wiring into the kernel, or the transform. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/b_modes.py | 52 ++++++------- src/sp_validation/tests/conftest.py | 19 +++++ src/sp_validation/tests/test_b_modes.py | 21 ++---- src/sp_validation/tests/test_cosmo_val.py | 91 ++++++++--------------- 4 files changed, 75 insertions(+), 108 deletions(-) create mode 100644 src/sp_validation/tests/conftest.py diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index 7a3cd484..973ecae2 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -156,23 +156,20 @@ def calculate_pure_eb_correlation( dict Dictionary containing pure E/B mode results and covariance """ - from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes - # Calculate min_sep and max_sep from gg object min_sep, max_sep = gg.left_edges[0], gg.right_edges[-1] def pure_EB(corrs): gg, gg_int = corrs - return get_pure_EB_modes( - theta=gg.meanr, - xip=gg.xip, - xim=gg.xim, + return pure_eb_from_xi( + theta_report=gg.meanr, + xip_report=gg.xip, + xim_report=gg.xim, theta_int=gg_int.meanr, xip_int=gg_int.xip, xim_int=gg_int.xim, tmin=min_sep, tmax=max_sep, - parallel=True, ) # The results dict is self-describing: the grids it was measured on travel @@ -190,7 +187,7 @@ def pure_EB(corrs): "xim_int": gg_int.xim, "n_eff": n_samples if cov_path_int is not None else gg.npatch1, } - results.update(dict(zip(_EB_KEYS, pure_EB([gg, gg_int])))) + results.update(pure_EB([gg, gg_int])) if cov_path_int is not None: if z_dist is None or cosmo_cov is None: @@ -214,7 +211,7 @@ def pure_EB(corrs): results["cov"] = treecorr.estimate_multi_cov( [gg, gg_int], var_method, - func=lambda x: np.hstack(pure_EB(x)), + func=lambda x: np.hstack([pure_EB(x)[k] for k in _EB_KEYS]), cross_patch_weight="match" if var_method == "jackknife" else None, ) @@ -236,8 +233,7 @@ def pure_eb_from_xi( ): """Pure-E/B correlation functions from ξ± arrays through the pipeline kernel. - The values-only seam of :func:`calculate_pure_eb_correlation`, for callers - holding ξ± arrays rather than TreeCorr correlations. + The one place this module calls cosmo_numba's Schneider (2022) transform. ``tmin``/``tmax`` are the reporting correlation's TreeCorr *bin edges* (``gg.left_edges[0]`` / ``gg.right_edges[-1]``). The reporting grid must be a @@ -281,15 +277,13 @@ def pure_eb_covariance_mc( ξ± draws come from ``cov_int``, a ξ± covariance on the integration grid, around the theory mean for ``(z, nz)`` under ``cosmo``; each draw is binned - down to the reporting grid and pushed through ``get_pure_EB_modes``. The + down to the reporting grid and pushed through :func:`pure_eb_from_xi`. The covariance of the transformed draws is the result, so it depends on the covariance model and the grids, never on the measured data vector. Returns ``(cov, eb_samples)`` — the covariance in ``_EB_KEYS`` order and the draws behind it. """ - from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes - theta, theta_int = np.asarray(theta), np.asarray(theta_int) nbins_int = len(theta_int) @@ -318,23 +312,21 @@ def pure_eb_covariance_mc( samples_rep_xip = (binning_matrix @ samples_int_xip.T).T samples_rep_xim = (binning_matrix @ samples_int_xim.T).T + def eb_vector(i): + modes = pure_eb_from_xi( + theta_report=theta, + xip_report=samples_rep_xip[i], + xim_report=samples_rep_xim[i], + theta_int=theta_int, + xip_int=samples_int_xip[i], + xim_int=samples_int_xim[i], + tmin=left_edges[0], + tmax=right_edges[-1], + ) + return np.concatenate([modes[k] for k in _EB_KEYS]) + eb_samples = np.array( - [ - np.concatenate( - get_pure_EB_modes( - theta=theta, - theta_int=theta_int, - xip=samples_rep_xip[i], - xim=samples_rep_xim[i], - xip_int=samples_int_xip[i], - xim_int=samples_int_xim[i], - tmin=left_edges[0], - tmax=right_edges[-1], - parallel=True, - ) - ) - for i in tqdm.tqdm(range(n_samples), desc="MC samples") - ] + [eb_vector(i) for i in tqdm.tqdm(range(n_samples), desc="MC samples")] ) return np.cov(eb_samples.T), eb_samples diff --git a/src/sp_validation/tests/conftest.py b/src/sp_validation/tests/conftest.py new file mode 100644 index 00000000..40424120 --- /dev/null +++ b/src/sp_validation/tests/conftest.py @@ -0,0 +1,19 @@ +"""Fixtures shared across test modules.""" + +from pathlib import Path + +import numpy as np +import pytest + +PURE_EB_XI = Path(__file__).parent / "data" / "pure_eb_xi_fixture.npz" + + +@pytest.fixture +def pure_eb_xi(): + """Committed ξ± of the synthetic coherent-shear catalogue. + + Exact-binning reporting [15, 70]′ in 6 bins and integration [1, 300]′ in + 600 bins, keyed by ``b_modes.pure_eb_from_xi``'s parameters. + """ + with np.load(PURE_EB_XI) as npz: + return {k: (v.item() if v.ndim == 0 else v) for k, v in npz.items()} diff --git a/src/sp_validation/tests/test_b_modes.py b/src/sp_validation/tests/test_b_modes.py index 42d632fa..20a9aa67 100644 --- a/src/sp_validation/tests/test_b_modes.py +++ b/src/sp_validation/tests/test_b_modes.py @@ -16,7 +16,6 @@ """ import types -from pathlib import Path import numpy as np import numpy.testing as npt @@ -299,8 +298,6 @@ def test_calculate_eb_statistics_has_teeth(): # 5. pure_eb_from_xi on committed ξ± (the transform pin) # --------------------------------------------------------------------------- -_PURE_EB_XI = Path(__file__).parent / "data" / "pure_eb_xi_fixture.npz" - # pure_eb_from_xi(**fixture); regenerated only when the transform is meant to move. _PURE_EB_PINS = { "xip_E": [ @@ -354,26 +351,18 @@ def test_calculate_eb_statistics_has_teeth(): } -def test_pure_eb_from_xi_reproduces_pins_on_committed_xi(): - """The pure-E/B transform of committed ξ± reproduces its pins. +def test_pure_eb_from_xi_reproduces_pins_on_committed_xi(pure_eb_xi): + """The pure-E/B transform of the committed ξ± reproduces its pins. - The fixture is the exact-binning ξ± that the synthetic pure-E/B test in - ``test_cosmo_val`` measures (reporting [15, 70]′ in 6 bins, integration - [1, 300]′ in 600 bins), keyed by ``pure_eb_from_xi``'s parameters. With ξ± - frozen, these pins move only when the transform does: when that test fails - and this one passes, the drift is upstream of the transform. rtol=1e-6 is + With ξ± frozen, these pins move only when the transform does. rtol=1e-6 is far above the 1e-12 reduction-order noise across thread counts. """ - pytest.importorskip("cosmo_numba") - with np.load(_PURE_EB_XI) as fixture: - xi = {k: (v.item() if v.ndim == 0 else v) for k, v in fixture.items()} - - modes = b_modes.pure_eb_from_xi(**xi) + modes = b_modes.pure_eb_from_xi(**pure_eb_xi) for key in b_modes._EB_KEYS: npt.assert_allclose(modes[key], _PURE_EB_PINS[key], rtol=1e-6, err_msg=key) # Teeth: widening the integration interval by 1% leaves the pins. - moved = b_modes.pure_eb_from_xi(**{**xi, "tmax": 1.01 * xi["tmax"]}) + moved = b_modes.pure_eb_from_xi(**{**pure_eb_xi, "tmax": 1.01 * pure_eb_xi["tmax"]}) assert not np.allclose(moved["xip_E"], _PURE_EB_PINS["xip_E"], rtol=1e-6, atol=0) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index b48f8f9d..c9ff1321 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -264,8 +264,8 @@ def test_v1_4_6_glass_mock_default_seed(self, base_config): # catalog/config/estimator together correctly and that the chain produces # output of the right shape with finite values. They do NOT re-test the # underlying numerical libraries (treecorr, cosmo_numba); only the pure-E/B - # test pins values. These are the back-pressure that catches config-path / - # wiring breakage during restructuring. + # test compares values, against committed ξ±. These are the back-pressure + # that catches config-path / wiring breakage during restructuring. # # Environment-independent: the catalog is synthesized in a tmp dir, so no # cluster data is needed. They do require the scientific stack (treecorr, @@ -506,27 +506,27 @@ def test_calculate_scale_dependent_leakage_runs_on_synthetic_catalog( assert np.all(np.isfinite(res.alpha_leak)) assert hasattr(res, "C_sys_p") and hasattr(res, "C_sys_m") - def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path): + def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path, pure_eb_xi): """calculate_pure_eb carries ξ± through cosmo_numba's pure-E/B split. - Every reporting bin is finite, and the four mode vectors match pins. + The ξ± it measures equal the committed ``pure_eb_xi``, its modes are + ``pure_eb_from_xi`` of those ξ± and edges, and every reporting bin is + finite. ``test_b_modes`` pins the transform itself on the same ξ±, so a + failure names the step that moved: measurement, wiring or transform. Finiteness: the Schneider (2022) integrals are near-singular where a reporting bin meets the integration boundary, so the integration grid [1, 300]′ brackets the reporting grid [15, 70]′ on both ends and is fine (600 bins); about 80 integration bins NaN the edge bins. - Pins: TreeCorr's default bin_slop/angle_slop approximate separations from - its tree, whose top-level split follows the jackknife patches and, through - min_top, the thread count TreeCorr takes from cpu_count(). On this - catalogue that moves the reporting ξ− by up to 16% between 4 and 48 - threads. Exact binning makes ξ± a plain pair sum, so the pins move only - when sp_validation does; rtol=1e-6 is far above its 1e-12 reduction-order - noise and far below a sub-percent change in any mode. The E/B transform - alone is pinned on fixed ξ± in ``test_b_modes``. + ξ±: exact binning (bin_slop = angle_slop = 0) makes ξ± a plain pair sum, + independent of the tree and so of the jackknife patches, whose k-means + centres this test does not fix. """ pytest.importorskip("treecorr") pytest.importorskip("cosmo_numba") + from sp_validation import b_modes + # Coherent shear -> smooth xi+/-, so the pure-E/B integral is well-posed. params, version = self._write_synthetic_catalogs( tmp_path, n_gal=4000, coherent_shear=True @@ -552,61 +552,28 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path): nbins_int=600, ) - # Regenerate by printing repr(results[key]) from the setup above. - expected = { - "xip_E": np.array( - [ - -2.9831529669572382e-06, - -1.5008524620255579e-05, - 3.221623968699465e-07, - 1.1797672310854472e-05, - 5.715510692580715e-06, - 8.825804523810145e-07, - ] - ), - "xim_E": np.array( - [ - -4.7375580917647536e-05, - -0.00010853189443992296, - -9.094825175031718e-05, - -5.826599101284305e-05, - -4.6464054157485714e-05, - -1.997802892533382e-05, - ] - ), - "xip_B": np.array( - [ - 1.706912124226553e-05, - 3.059889782372767e-05, - -4.880539925382135e-06, - -6.999262696331131e-06, - -1.267200698975249e-05, - -1.2141491389775202e-06, - ] - ), - "xim_B": np.array( - [ - -0.00011478091634543392, - -5.4451120021397075e-05, - -3.100806652947136e-05, - -1.0940424256752644e-05, - -5.755185146639151e-06, - -1.628217762504009e-06, - ] - ), + measured = { + "theta_report": results["theta"], + "xip_report": results["xip"], + "xim_report": results["xim"], + "theta_int": results["theta_int"], + "xip_int": results["xip_int"], + "xim_int": results["xim_int"], + "tmin": results["left_edges"][0], + "tmax": results["right_edges"][-1], } + # Regenerate the fixture with np.savez(conftest.PURE_EB_XI, **measured). + for key, value in measured.items(): + np.testing.assert_allclose( + value, pure_eb_xi[key], rtol=1e-10, atol=0, err_msg=key + ) - for key in ("xip_E", "xim_E", "xip_B", "xim_B"): + modes = b_modes.pure_eb_from_xi(**measured) + for key in b_modes._EB_KEYS: vec = np.asarray(results[key]) assert vec.shape == (nbins,) assert np.all(np.isfinite(vec)), f"{key} not finite" - np.testing.assert_allclose( - vec, - expected[key], - rtol=1e-6, - atol=1e-12, - err_msg=f"{key} drifted from pinned reference", - ) + np.testing.assert_allclose(vec, modes[key], rtol=1e-10, err_msg=key) # Jackknife covariance over the 6 stats (xip/xim x E/B/amb) x nbins. cov = np.asarray(results["cov"]) From bb482c090455c351b0a885fc495ca7b3bb20df7f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 25 Sep 2026 22:09:39 +0200 Subject: [PATCH 068/160] b_modes: one pure-E/B transform per jackknife realisation The jackknife covariance function indexed pure_EB(x) once per key, so every realisation ran cosmo_numba's transform six times (55 transforms at npatch=8 where 10 suffice; values unchanged). _eb_vector concatenates the modes in _EB_KEYS order for both the jackknife and the MC covariance. The synthetic pure-E/B test counts transforms around calculate_pure_eb and is red on the per-key closure (55 > npatch + 2). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/b_modes.py | 13 +++++++++---- src/sp_validation/tests/test_cosmo_val.py | 13 ++++++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index 973ecae2..fdc2ddf8 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -19,6 +19,11 @@ _EB_KEYS = ("xip_E", "xim_E", "xip_B", "xim_B", "xip_amb", "xim_amb") +def _eb_vector(modes): + """Pure-E/B modes concatenated in ``_EB_KEYS`` order, the covariance layout.""" + return np.concatenate([modes[k] for k in _EB_KEYS]) + + def find_conservative_scale_cut_key(results, requested_scale_cut): """ Find scale cut key that conservatively fits within requested range. @@ -211,7 +216,7 @@ def pure_EB(corrs): results["cov"] = treecorr.estimate_multi_cov( [gg, gg_int], var_method, - func=lambda x: np.hstack([pure_EB(x)[k] for k in _EB_KEYS]), + func=lambda x: _eb_vector(pure_EB(x)), cross_patch_weight="match" if var_method == "jackknife" else None, ) @@ -312,7 +317,7 @@ def pure_eb_covariance_mc( samples_rep_xip = (binning_matrix @ samples_int_xip.T).T samples_rep_xim = (binning_matrix @ samples_int_xim.T).T - def eb_vector(i): + def eb_draw(i): modes = pure_eb_from_xi( theta_report=theta, xip_report=samples_rep_xip[i], @@ -323,10 +328,10 @@ def eb_vector(i): tmin=left_edges[0], tmax=right_edges[-1], ) - return np.concatenate([modes[k] for k in _EB_KEYS]) + return _eb_vector(modes) eb_samples = np.array( - [eb_vector(i) for i in tqdm.tqdm(range(n_samples), desc="MC samples")] + [eb_draw(i) for i in tqdm.tqdm(range(n_samples), desc="MC samples")] ) return np.cov(eb_samples.T), eb_samples diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index c9ff1321..ca96486a 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -506,13 +506,16 @@ def test_calculate_scale_dependent_leakage_runs_on_synthetic_catalog( assert np.all(np.isfinite(res.alpha_leak)) assert hasattr(res, "C_sys_p") and hasattr(res, "C_sys_m") - def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path, pure_eb_xi): + def test_calculate_pure_eb_runs_on_synthetic_catalog( + self, tmp_path, pure_eb_xi, monkeypatch + ): """calculate_pure_eb carries ξ± through cosmo_numba's pure-E/B split. The ξ± it measures equal the committed ``pure_eb_xi``, its modes are ``pure_eb_from_xi`` of those ξ± and edges, and every reporting bin is finite. ``test_b_modes`` pins the transform itself on the same ξ±, so a failure names the step that moved: measurement, wiring or transform. + The jackknife covariance runs the transform once per realisation. Finiteness: the Schneider (2022) integrals are near-singular where a reporting bin meets the integration boundary, so the integration grid @@ -544,6 +547,12 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path, pure_eb_xi) ) cv.treecorr_config.update(bin_slop=0, angle_slop=0) + kernel, kernel_calls = b_modes.pure_eb_from_xi, [] + monkeypatch.setattr( + b_modes, + "pure_eb_from_xi", + lambda **kw: kernel_calls.append(kw) or kernel(**kw), + ) results = cv.calculate_pure_eb( version, npatch=npatch, @@ -551,6 +560,8 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path, pure_eb_xi) max_sep_int=300.0, nbins_int=600, ) + # The modes, then TreeCorr's jackknife: one sizing call and one per patch. + assert len(kernel_calls) <= npatch + 2, f"{len(kernel_calls)} transforms" measured = { "theta_report": results["theta"], From 66d9cb6d79d3f75d8f74eccea2eb85a6acae3b48 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 25 Sep 2026 22:09:40 +0200 Subject: [PATCH 069/160] tests: glue-section header stops enumerating which tests compare values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two tests in the section compare values (pure-E/B against committed ξ±, ξ± across TreeCorr thread counts); the header defers to each test's docstring instead of listing them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/tests/test_cosmo_val.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index ca96486a..7a7c0780 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -262,10 +262,11 @@ def test_v1_4_6_glass_mock_default_seed(self, base_config): # These run the real compute seams end-to-end on a small, deterministic # toy catalog written to disk, asserting that sp_validation wires the # catalog/config/estimator together correctly and that the chain produces - # output of the right shape with finite values. They do NOT re-test the - # underlying numerical libraries (treecorr, cosmo_numba); only the pure-E/B - # test compares values, against committed ξ±. These are the back-pressure - # that catches config-path / wiring breakage during restructuring. + # output of the right shape with finite values; a test that also compares + # values says against what in its docstring. They do NOT re-test the + # underlying numerical libraries (treecorr, cosmo_numba). These are the + # back-pressure that catches config-path / wiring breakage during + # restructuring. # # Environment-independent: the catalog is synthesized in a tmp dir, so no # cluster data is needed. They do require the scientific stack (treecorr, From 1db81ceccc3ff1ad26fb3b2ab98cd9230a6fce6d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 25 Sep 2026 22:39:44 +0200 Subject: [PATCH 070/160] workflow: host/image parity check, catalogue config from the checkout, one output root, one integration grid - Launch-time parity: container.image_runtime reads the image's Python minor and Snakemake version (SIF via apptainer, sandbox off disk, tags skipped); common.check_host_parity stops a launch whose host Snakemake differs, with the reinstall command. configure() and image_sims.smk call it; the README install line pins --python 3.12. - The catalogue config is the launched checkout's cosmo_val/cat_config.yaml, loaded by configure() into CATALOG_CONFIG; the paper Snakefiles no longer merge it into `config`, and covariance.smk / ecut.smk read CATALOG_CONFIG. - One output root: cv_init_params passes output_dir=COSMO_VAL, cv_runner no longer chdirs into the live checkout, and CosmologyValidation drops its COSMO_VAL environment fallback. - One integration grid (R12): the cosebis grid is gone; cv_cosebis reads the integration part and the CosmoCov g covariance on that grid (the one pure-E/B uses). An npatch=1 grid defaults to the diagonal covariance, and a binning outside the named grids takes its covariance from its own patches. - The candide profile sets jobs: 100. - workflow/tests: host-launcher DAG tests on a toy checkout (P1-P3) and the real papers on candide (P4, P5 = the container smoke test, moved here); CI runs them in a workflow-dag job. test_bmodes_workflow_dry_run.py is replaced by P4. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- .github/workflows/deploy-image.yml | 19 ++ papers/bmodes/Snakefile | 1 - papers/bmodes/rules/ecut.smk | 4 +- papers/cosmo_val/Snakefile | 1 - papers/cosmo_val/config/config.yaml | 8 +- src/sp_validation/container.py | 62 +++++ src/sp_validation/cosmo_val/core.py | 8 +- .../tests/test_bmodes_workflow_dry_run.py | 88 ------- .../tests/test_cv_init_params.py | 1 - src/sp_validation/tests/test_xi_grids.py | 45 ++-- workflow/README.md | 37 ++- workflow/common.py | 132 ++++++---- workflow/profiles/candide/config.yaml | 4 + workflow/rules/cosmo_val.smk | 94 ++++--- workflow/rules/covariance.smk | 4 +- workflow/rules/image_sims.smk | 1 + workflow/rules/twopoint.smk | 12 +- workflow/scripts/cv_cosebis.py | 16 +- workflow/scripts/cv_runner.py | 8 +- workflow/scripts/cv_summarize_bmodes.py | 2 +- workflow/scripts/run_2pcf.py | 4 +- workflow/tests/conftest.py | 231 ++++++++++++++++++ .../tests/data/container_smoke/Snakefile | 2 +- .../data/container_smoke/container_smoke.py | 6 +- workflow/tests/pytest.ini | 3 + .../tests/test_container_smoke.py | 57 ++--- workflow/tests/test_dag.py | 146 +++++++++++ 27 files changed, 705 insertions(+), 291 deletions(-) delete mode 100644 src/sp_validation/tests/test_bmodes_workflow_dry_run.py create mode 100644 workflow/tests/conftest.py rename {src/sp_validation => workflow}/tests/data/container_smoke/Snakefile (89%) rename {src/sp_validation => workflow}/tests/data/container_smoke/container_smoke.py (94%) create mode 100644 workflow/tests/pytest.ini rename {src/sp_validation => workflow}/tests/test_container_smoke.py (62%) create mode 100644 workflow/tests/test_dag.py diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index f65ab5f7..3ad8e9f7 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -9,6 +9,25 @@ env: BRANCH: ${{ github.ref }} jobs: + # The workflow's DAG properties, checked through the host launcher: Snakemake + # on the image's Python with sp_validation absent, as on a user's machine. + workflow-dag: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 + + - name: DAG tests + run: >- + uv run --isolated --no-project --python 3.12 + --with snakemake==9.23.1 --with pytest --with numpy + pytest workflow/tests -m "not candide" + build-and-push-image: runs-on: - ubuntu-latest diff --git a/papers/bmodes/Snakefile b/papers/bmodes/Snakefile index c9069c79..23c9cfec 100644 --- a/papers/bmodes/Snakefile +++ b/papers/bmodes/Snakefile @@ -2,7 +2,6 @@ # compute workflow at ../../workflow/. configfile: "config/config.yaml" -configfile: "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" envvars: "PYTHONUNBUFFERED", diff --git a/papers/bmodes/rules/ecut.smk b/papers/bmodes/rules/ecut.smk index 9f941534..326399aa 100644 --- a/papers/bmodes/rules/ecut.smk +++ b/papers/bmodes/rules/ecut.smk @@ -27,7 +27,7 @@ def _ecut_parent_catalog(wildcards): parent = ECUT_PARENT_VERSIONS.get(wildcards.version) if parent is None: raise ValueError(f"No parent version found for {wildcards.version}") - cat_config = config[parent] + cat_config = CATALOG_CONFIG[parent] shear_path = cat_config["shear"]["path"] if shear_path.startswith("/"): return shear_path @@ -38,7 +38,7 @@ def _ecut_parent_area(version): """Get parent version's area for an ecut version.""" base = version.replace("_leak_corr", "") parent = ECUT_PARENT_VERSIONS.get(base, base) - return config[parent]["cov_th"]["A"] + return CATALOG_CONFIG[parent]["cov_th"]["A"] # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ diff --git a/papers/cosmo_val/Snakefile b/papers/cosmo_val/Snakefile index 67bef0a7..083dd2a6 100644 --- a/papers/cosmo_val/Snakefile +++ b/papers/cosmo_val/Snakefile @@ -7,7 +7,6 @@ # isolation or as the whole suite via the default `cosmo_val_all` target. configfile: "config/config.yaml" -configfile: "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" envvars: "PYTHONUNBUFFERED", diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index ae99aba4..5d8416c6 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -74,18 +74,14 @@ cosmo_val: kmax: 20 kmax_extrapolate: 500 - # The fine ξ± grid the B-mode integrals run over. + # The fine ξ± grid both B-mode statistics (COSEBIs, pure-E/B) integrate over. integration: min_sep: 0.08 max_sep: 300 nbins: 1000 - # COSEBIs decomposition (config space, fine integration binning) + # COSEBIs on the integration grid: mode count and the scale cuts scanned. cosebis: - min_sep_int: 0.9 - max_sep_int: 300 - nbins_int: 1000 - npatch: 100 nmodes: 20 scale_cuts: [ [1, 250], [2, 250], [5, 250], [10, 250], diff --git a/src/sp_validation/container.py b/src/sp_validation/container.py index 7aa5f2ed..ff365823 100755 --- a/src/sp_validation/container.py +++ b/src/sp_validation/container.py @@ -123,6 +123,68 @@ def image_revision(sif): return image_labels(sif).get("org.opencontainers.image.revision") +# The image's virtual environment, as the Dockerfile lays it out. +IMAGE_VENV = "/app/.venv" + + +def image_runtime(image): + """Return ``(python_minor, snakemake_version)`` of an image, or ``None``. + + Read from the venv's ``pyvenv.cfg`` and its ``snakemake-*.dist-info`` + directory: through ``apptainer exec`` for a SIF, straight off disk for a + sandbox. ``None`` when the image cannot be read -- a registry tag, a missing + ``apptainer``, a venv laid out differently. ``snakemake_version`` is + ``None`` when the image carries no snakemake. + """ + image = str(image) + venv = IMAGE_VENV.lstrip("/") + if Path(image).is_dir(): + root = Path(image) / venv + try: + cfg = (root / "pyvenv.cfg").read_text() + except OSError: + return None + listing = [p.name for p in root.glob("lib/python*/site-packages/*")] + elif Path(image).is_file() and shutil.which("apptainer") is not None: + try: + out = subprocess.run( + [ + "apptainer", + "exec", + "--cleanenv", + image, + "sh", + "-c", + f"cat {IMAGE_VENV}/pyvenv.cfg && " + f"ls {IMAGE_VENV}/lib/python*/site-packages", + ], + capture_output=True, + text=True, + timeout=120, + ) + except (OSError, subprocess.SubprocessError): + return None + if out.returncode != 0: + return None + cfg, listing = out.stdout, out.stdout.split() + else: + return None + fields = {} + for line in cfg.splitlines(): + key, sep, value = line.partition("=") + if sep: + fields[key.strip()] = value.strip() + version = fields.get("version_info") + if version is None: + return None + snakemake = [ + name[len("snakemake-") : -len(".dist-info")] + for name in listing + if name.startswith("snakemake-") and name.endswith(".dist-info") + ] + return ".".join(version.split(".")[:2]), (snakemake[0] if snakemake else None) + + def _require_apptainer(): """Exit unless ``apptainer`` is on PATH.""" if shutil.which("apptainer") is None: diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index fa3a6ecb..1ea75134 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -83,8 +83,7 @@ class CosmologyValidation( Path to catalog configuration YAML defining survey metadata, file paths, and analysis settings for each version. output_dir : str, optional - Override for output directory. If None, falls back to the COSMO_VAL - environment variable, then to the catalog config's paths.output. + Output directory. If None, the catalog config's paths.output. rho_tau_method : {'lsq', 'mcmc'}, default 'lsq' Fitting method for PSF leakage systematics parameters. cov_estimate_method : {'th', 'jk'}, default 'th' @@ -396,11 +395,6 @@ def ensure_version_exists(ver): self.versions = final_versions - # Override output directory: explicit arg > COSMO_VAL env var > - # catalog config's paths.output. The env hook lets a reproduction - # run redirect every product to a fresh tree without touching the - # per-script call sites (mirrors workflow/common.py's COSMO_VAL). - output_dir = output_dir or os.environ.get("COSMO_VAL") if output_dir is not None: cc["paths"]["output"] = output_dir diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py deleted file mode 100644 index 3b7eecae..00000000 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Back-pressure guard #2: the paper Snakemake workflows dry-run. - -The reorg is allowed to change the rule graph; these guards only assert that -Snakemake can still parse each composed workflow and construct a dry run. One -guard covers papers/bmodes (config space, no cosmo_val block); a second covers -papers/cosmo_val, whose config DOES carry a cosmo_val block — so it is the only -one that includes cosmo_val.smk and hence the born-as-SACC + assemble rules. -""" - -import os -import subprocess -import sys -from pathlib import Path - -import pytest - -# The workflow composes a catalog configfile and terminal inputs that live at -# candide-absolute paths (Snakefile line 5, workflow/common.py), so the dry -# run can only be constructed on the cluster. Same pattern as test_cosmo_val. -requires_candide_data = pytest.mark.skipif( - not Path("/n17data/cdaley/unions").exists(), - reason="candide-local workflow config/data (/n17data) absent — off-cluster", -) - - -def _repo_root() -> Path: - for parent in Path(__file__).resolve().parents: - if (parent / "pyproject.toml").exists(): - return parent - raise RuntimeError("could not locate repo root (no pyproject.toml above test)") - - -def _dry_run(workflow_dir, targets, *extra_snakemake_args): - """Construct a dry run of the paper workflow at ``workflow_dir``. - - PYTHONUNBUFFERED satisfies the Snakefile's ``envvars:`` declaration. A dry - run never dispatches jobs, so any inherited SNAKEMAKE_PROFILE is dropped - rather than requiring its executor plugin. snakemake is invoked through - sys.executable, since a bare python3.12 may resolve off PATH to an - interpreter without it. - """ - env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} - env.pop("SNAKEMAKE_PROFILE", None) - return subprocess.run( - [ - sys.executable, - "-m", - "snakemake", - *targets, - "--dry-run", - "--cores", - "1", - "--configfile", - "config/config.yaml", - *extra_snakemake_args, - ], - cwd=workflow_dir, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=120, - check=False, - ) - - -@requires_candide_data -def test_bmodes_workflow_dry_runs(): - """The paper B-mode workflow must still parse and dry-run cleanly.""" - result = _dry_run(_repo_root() / "papers/bmodes", ["all_tapestry"]) - assert result.returncode == 0, result.stdout - - -@requires_candide_data -def test_cosmo_val_workflow_assemble_dry_runs(): - """The cosmo_val workflow (the only one including cosmo_val.smk) resolves the - born-as-SACC + assemble DAG, and assemble pulls the tagged pseudo-Cl + cov.""" - version = "SP_v1.4.6.3_leak_corr" - result = _dry_run(_repo_root() / "papers/cosmo_val", ["assemble_sacc_all"]) - assert result.returncode == 0, result.stdout - # assemble_sacc must pull the tagged pseudo-Cl part + its NaMaster - # covariance (not the untagged cv_pseudo_cl diagnostic), plus every part. - out = result.stdout - assert "rule assemble_sacc:" in out, out - assert f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" in out, out - assert f"pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits" in out, out - for part in ("_xi_minsep=", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): - assert part in out, f"missing {part} part in assemble DAG:\n{out}" diff --git a/src/sp_validation/tests/test_cv_init_params.py b/src/sp_validation/tests/test_cv_init_params.py index f769c179..240f9213 100644 --- a/src/sp_validation/tests/test_cv_init_params.py +++ b/src/sp_validation/tests/test_cv_init_params.py @@ -17,7 +17,6 @@ REPO = Path(__file__).resolve().parents[3] EXEMPT = { - "output_dir": "rules set the output tree via the run directory / COSMO_VAL", "blind": "None keeps the n(z) blind declared in the catalogue config", } diff --git a/src/sp_validation/tests/test_xi_grids.py b/src/sp_validation/tests/test_xi_grids.py index c35a2e3f..84ce3472 100644 --- a/src/sp_validation/tests/test_xi_grids.py +++ b/src/sp_validation/tests/test_xi_grids.py @@ -34,12 +34,7 @@ def _load_common(): "nbins": 20, "npatch": 100, "integration": {"min_sep": 0.08, "max_sep": 300, "nbins": 1000}, - "cosebis": { - "min_sep_int": 0.9, - "max_sep_int": 300, - "nbins_int": 1000, - "npatch": 100, - }, + "cosebis": {"nmodes": 20, "scale_cuts": [[12, 83]]}, } } FIDUCIAL = { @@ -61,15 +56,11 @@ def test_tag_is_built_from_canonical_values(): for a path the producer never writes. """ grids = common.xi_grids(CONFIG, FIDUCIAL) - assert common.grid_binning(grids["integration"]).endswith( - "minsep=0.08_maxsep=300.0_nbins=1000_npatch=1" - ) + # Counts stay integers, so no "nbins=1000.0" creeps into a name. assert ( - common.grid_binning(grids["cosebis"]) - == "minsep=0.9_maxsep=300.0_nbins=1000_npatch=100" + common.grid_binning(grids["integration"]) + == "minsep=0.08_maxsep=300.0_nbins=1000_npatch=1" ) - # Counts stay integers, so no "nbins=1000.0" creeps into a name. - assert "nbins=1000_" in common.grid_binning(grids["cosebis"]) def test_grid_lookup_round_trips_through_the_tag(): @@ -86,19 +77,34 @@ def test_grid_lookup_round_trips_through_the_tag(): assert common.grid_of(grids, {k: str(v) for k, v in binning.items()}) == name +def test_one_integration_grid(): + """COSEBIs and pure-E/B share the integration grid; there is no third.""" + assert set(common.xi_grids(CONFIG, FIDUCIAL)) == {"reporting", "integration"} + + def test_covariance_mode_follows_the_patches(): - """Patched grids get a jackknife block, unpatched ones none.""" + """Patched grids get a jackknife block, unpatched ones the diagonal. + + Every part then carries variances: at npatch=1 TreeCorr's var_method is + "shot", and its diagonal is the estimate it has. + """ grids = common.xi_grids(CONFIG, FIDUCIAL) assert grids["reporting"]["cov"] == "jackknife" - assert grids["cosebis"]["cov"] == "jackknife" - assert grids["integration"]["cov"] == "none" + assert grids["integration"]["cov"] == "diagonal" -def test_unnamed_binning_is_a_reporting_measurement(): - """The paper's convergence-check binning belongs to no named grid.""" +@pytest.mark.parametrize("npatch, cov", [(1, "diagonal"), (50, "jackknife")]) +def test_unnamed_binning_is_a_reporting_measurement(npatch, cov): + """The paper's convergence-check binning belongs to no named grid. + + It is tagged as a reporting measurement, and its covariance follows its own + patches rather than the reporting grid's. + """ grids = common.xi_grids(CONFIG, FIDUCIAL) - stray = {"min_sep": 1.0, "max_sep": 250.0, "nbins": 10000, "npatch": 1} + stray = {"min_sep": 1.0, "max_sep": 250.0, "nbins": 10000, "npatch": npatch} assert common.grid_of(grids, stray) == "reporting" + assert common.grid_cov(grids, stray) == cov + assert common.grid_cov(grids, {k: str(v) for k, v in stray.items()}) == cov def test_workflow_without_cosmo_val_falls_back_to_fiducial(): @@ -106,4 +112,3 @@ def test_workflow_without_cosmo_val_falls_back_to_fiducial(): grids = common.xi_grids({}, FIDUCIAL) assert grids["reporting"]["npatch"] == 1 assert grids["integration"]["min_sep"] == 0.5 - assert "cosebis" not in grids diff --git a/workflow/README.md b/workflow/README.md index 4171074e..187cb343 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -93,6 +93,11 @@ This is the default because the alternative is incoherent: Snakemake's rule executes new script code against an old `import sp_validation` — the two halves of one commit, split. +The catalogue config is the launched checkout's too: `cosmo_val/cat_config.yaml`, +read by the host and handed to every job. Every declared output lies under +`COSMO_VAL` or `COSMO_INFERENCE` (environment variables, defaulting to the +shared trees on candide) or the run directory's `results/`. + **Caveat:** `rerun-triggers: code` watches rule bodies and `script:` files, not `src/`. Editing a module under `src/` does not by itself mark outputs stale — force with `-F` or `--forcerun `. @@ -126,11 +131,20 @@ already uses the plain form; keep new paths the same. `snakemake` is a thin host-side tool, pinned once per machine: ```bash -uv tool install snakemake==9.23.1 --with snakemake-executor-plugin-slurm +uv tool install --python 3.12 snakemake==9.23.1 --with snakemake-executor-plugin-slurm ``` -(match the version to `snakemake` in this repo's `uv.lock`). Run every -`snakemake` command directly on the host — do not `apptainer shell` first. +The Python minor and the version must match the image's (the image's Python +and the `snakemake` in this repo's `uv.lock`): a `script:` job appends the +host's `sys.path` — standard library included — to its own so that it can +unpickle the host's `snakemake` object, so a host on another Python loads +modules the image lacks from the host's stdlib, and another Snakemake writes a +pickle the job cannot read. Every launch checks this (`common.check_host_parity`) +and stops with the reinstall command on a mismatch; an image it cannot read, +such as a registry tag, is named in one line and passes. + +Run every `snakemake` command directly on the host — do not `apptainer shell` +first. Snakemake itself never touches the science stack; it only reads rule definitions and submits jobs. Each job carries its own `apptainer exec` wrapping from the profile (see above), so the container is where the science @@ -267,11 +281,18 @@ For the image-sims workflow, set `image_sims: {sif: ...}` in your run config. Either way the image has to sit under one of the profile's bind mounts to be visible. -One trap to know: the `script:` directive bind-mounts the host orchestrator's -`snakemake` into the job and *appends* it to `sys.path`, so a `snakemake` -importable inside the image wins the lookup. If `script:` rules start failing -with `ModuleNotFoundError: No module named 'snakemake.iocontainers'` or similar, -an in-image snakemake older than the host's is the first thing to check. +### Checking the workflow itself + +`workflow/tests/` checks DAG properties through the host launcher, on a toy +checkout and — on candide — on the real papers: + +```bash +uv run --isolated --no-project --python 3.12 --with snakemake==9.23.1 \ + --with snakemake-executor-plugin-slurm --with pytest --with numpy \ + pytest workflow/tests +``` + +CI runs the same suite with `-m "not candide"`. ### `snakemake` in `script:` files diff --git a/workflow/common.py b/workflow/common.py index 4391ddbf..d4a7067e 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -1,5 +1,11 @@ -"""Shared helpers for the B-modes Snakemake workflow.""" +"""Shared helpers for every Snakefile that composes workflow/. +Imported by the host Snakemake, where sp_validation is not installed: this +module imports only the standard library and snakemake, and loads the +stdlib-only project modules it needs by file path. +""" + +import functools import importlib.util import json import os @@ -7,17 +13,13 @@ import sys from pathlib import Path -# This checkout's importable source tree: workflow/common.py -> /src. -REPO_SRC = Path(__file__).resolve().parent.parent / "src" +# The checkout this workflow was launched from: workflow/common.py -> . +REPO_ROOT = Path(__file__).resolve().parent.parent +REPO_SRC = REPO_ROOT / "src" # The container model lives in the package (``sp_validation/container.py``). # Taken from *this checkout's* src/, so the workflow and the ``spv-container`` # CLI can never disagree about which image to run. -# -# Loaded by file path rather than as ``sp_validation.container``: snakemake runs -# on the host, where sp_validation is usually not installed, and importing the -# package would drag in ``__init__`` -> ``version`` -> a metadata warning on -# every launch. The module itself is stdlib-only, so this costs nothing. _container = importlib.util.module_from_spec( importlib.util.spec_from_file_location( "_spv_container", REPO_SRC / "sp_validation" / "container.py" @@ -28,6 +30,7 @@ compare_revision = _container.compare_revision image_revision = _container.image_revision +image_runtime = _container.image_runtime resolve_image = _container.resolve_image @@ -43,7 +46,9 @@ "COSMO_INFERENCE", "/n17data/cdaley/unions/code/sp_validation/cosmo_inference" ) ) -CAT_CONFIG = "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" +# The catalogue config of the launched checkout: the one file both the host +# (CATALOG_CONFIG, loaded in configure) and every job read catalogues from. +CAT_CONFIG = str(REPO_ROOT / "cosmo_val" / "cat_config.yaml") # "blind" is the glass-mock A/B/C realisation convention, NOT Smokescreen # blinding (a separate axis: the concealed=True SACC stamp). The name is baked # into on-disk filenames we do not own (e.g. nz_{version}_{A|B|C}.txt). @@ -158,12 +163,54 @@ def warn_if_image_stale(): ) +@functools.cache +def check_host_parity(image): + """Stop the launch unless this Snakemake matches the image's Python and Snakemake. + + A ``script:`` job appends the host's ``sys.path`` -- standard library + included -- to its own, as the fallback that lets it unpickle the host's + ``snakemake`` object. So the image and the host Snakemake must share a + Python minor (else a module the image lacks loads from the host's stdlib) + and a Snakemake version (else the pickle does not match its reader). + + An image that cannot be read (a registry tag, no apptainer) is named in one + line and passes. + """ + import snakemake + from snakemake.exceptions import WorkflowError + + runtime = image_runtime(image) + if runtime is None: + print( + f"[container] cannot read the Python of {image}; host/image parity " + "unchecked.", + file=sys.stderr, + ) + return + python, image_snakemake = runtime + host_python = ".".join(str(v) for v in sys.version_info[:2]) + host = (host_python, snakemake.__version__) + if (python, image_snakemake or snakemake.__version__) == host: + return + raise WorkflowError( + f"host Snakemake {snakemake.__version__} on Python {host_python} does not " + f"match the image {image} (Snakemake {image_snakemake}, Python {python}).\n" + "Reinstall the host Snakemake to match:\n" + f" uv tool install --force --python {python} " + f"snakemake=={image_snakemake or snakemake.__version__} " + "--with snakemake-executor-plugin-slurm" + ) + + def configure(workflow_config): """Install config-derived values after Snakemake has loaded configfiles.""" global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, FIDUCIAL, PLANCK18 + from snakemake.common.configfile import load_configfile + inject_checkout_pythonpath(workflow_config) warn_if_image_stale() - CATALOG_CONFIG = workflow_config + check_host_parity(resolve_container(workflow_config.get("container"))) + CATALOG_CONFIG = load_configfile(CAT_CONFIG) FIDUCIAL = workflow_config["fiducial"] DEFAULT_MASK_SUFFIX = ( "_masked" if workflow_config["covariance"].get("default_masked", False) else "" @@ -289,9 +336,9 @@ def build_redshift_path(version, blind): # --------------------------------------------------------------------------- # A grid is a binning plus how its covariance is estimated: (min_sep, max_sep, # nbins, npatch, cov). `reporting` is the analysis grid, `integration` the fine -# one the B-mode integrals run over, `cosebis` the fine patched grid COSEBIs -# propagates its covariance from. cov is "jackknife" (dense, from the patches), -# "diagonal" (TreeCorr varxip/varxim) or "none". +# one both B-mode statistics (COSEBIs, pure-E/B) integrate over. cov is +# "jackknife" (dense, from the patches), "diagonal" (TreeCorr varxip/varxim) or +# "none". XI_KEYS = ( "min_sep", "max_sep", @@ -332,25 +379,24 @@ def xi_grids(config, fiducial): ), } grids["integration"].setdefault("npatch", 1) - cb = cv.get("cosebis") - if cb: - grids["cosebis"] = { - "min_sep": cb["min_sep_int"], - "max_sep": cb["max_sep_int"], - "nbins": cb["nbins_int"], - "npatch": cb["npatch"], - } for grid in grids.values(): for key in ("min_sep", "max_sep"): grid[key] = float(grid[key]) for key in ("nbins", "npatch"): grid[key] = int(grid[key]) - # A jackknife estimate needs patches; at npatch=1 TreeCorr's var_method - # is "shot" and the diagonal is all it can offer. - grid.setdefault("cov", "jackknife" if grid["npatch"] > 1 else "none") + grid.setdefault("cov", patch_cov(grid["npatch"])) return grids +def patch_cov(npatch): + """The covariance a measurement with ``npatch`` patches can carry. + + A jackknife needs patches; at npatch=1 TreeCorr's var_method is "shot" and + its diagonal is the estimate there is. + """ + return "jackknife" if int(npatch) > 1 else "diagonal" + + def grid_binning(grid): """The `minsep=..._maxsep=..._nbins=..._npatch=...` tag of one grid.""" return ( @@ -359,6 +405,15 @@ def grid_binning(grid): ) +def _named_grid(grids, binning): + """Name of the grid a binning is, compared numerically, or ``None``.""" + key = tuple(float(binning[k]) for k in XI_KEYS) + for name, grid in grids.items(): + if tuple(float(grid[k]) for k in XI_KEYS) == key: + return name + return None + + def grid_of(grids, binning): """Name of the grid a binning belongs to, compared numerically. @@ -366,11 +421,13 @@ def grid_of(grids, binning): grid (e.g. papers/bmodes' nbins=10000 convergence check) are reporting-style measurements. """ - key = tuple(float(binning[k]) for k in XI_KEYS) - for name, grid in grids.items(): - if tuple(float(grid[k]) for k in XI_KEYS) == key: - return name - return "reporting" + return _named_grid(grids, binning) or "reporting" + + +def grid_cov(grids, binning): + """Covariance mode for a binning: its grid's, else what its patches allow.""" + name = _named_grid(grids, binning) + return grids[name]["cov"] if name else patch_cov(binning["npatch"]) def pseudo_cl_tag(config): @@ -398,19 +455,10 @@ def get_shear_catalog(wildcards): # turns each diagnostic into a rule keyed on the real data products it writes # under COSMO_VAL. Where a method only emits a figure (no data product), the # rule declares a sentinel under CV_SENTINELS so the DAG stays trackable. -# -# COSMO_VAL is the cosmo_val/output directory (already defined above), the same -# location every `cv.*` method writes to via `cc["paths"]["output"]`. # Sentinel directory for pure-plot leaf rules (no natural data-product output). CV_SENTINELS = COSMO_VAL / "snakemake_sentinels" -# Working directory in which `CosmologyValidation` is instantiated: its -# catalogue config comes explicitly from CAT_CONFIG, and it writes to -# `./output` unless COSMO_VAL is set. Resolved to the live (non-worktree) -# checkout so rules share the output tree with interactive runs. -CV_RUNDIR = "/n17data/cdaley/unions/code/sp_validation/cosmo_val" - def cv_basename(version, fiducial=None): """Reproduce CosmologyValidation.basename() for a version. @@ -466,13 +514,15 @@ def cv_init_params(config, version_list=None): """Assemble the CosmologyValidation(...) constructor kwargs from config. Centralizes the run-specific instantiation so every cosmo_val rule script - builds an identical `cv`. The catalogue config is always CAT_CONFIG, never - the constructor's cwd-relative default. `version_list` overrides - config["versions"] (used by per-version rules that pass a single version). + builds an identical `cv`: catalogues from CAT_CONFIG and products under + COSMO_VAL, never the constructor's cwd-relative defaults. `version_list` + overrides config["versions"] (used by per-version rules that pass a single + version). """ cv = config["cosmo_val"] return dict( versions=version_list if version_list is not None else config["versions"], catalog_config=CAT_CONFIG, + output_dir=str(COSMO_VAL), **{key: cv[key] for key in CV_INIT_KEYS}, ) diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 19afd89a..22b549b1 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -62,6 +62,10 @@ default-resources: cpus_per_task: 12 slurm_extra: "'--exclude=n17,n09,n36'" +# How many jobs may be queued or running at once. Snakemake requires a bound +# for any remote executor and refuses a real run without one. +jobs: 100 + # Retry a job once on transient node failure, and keep the SLURM logs of # successful jobs (candide debugging). retries: 1 diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 87138bc0..ce04400d 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -6,14 +6,14 @@ # each diagnostic is a rule, and the rules are linked by the SACC parts and # products they write under COSMO_VAL (= cosmo_val/output): # -# catalogue ──→ xi (one job per grid: reporting, integration, cosebis) -# │ -# ├─ reporting part ──┬─→ pure_eb (part, npz, figures) -# ├─ integration part ┘ │ -# ├─ cosebis part ─────→ cosebis (part, npz, figures) +# catalogue ──→ xi (one job per grid: reporting, integration) +# ├─ reporting part ──────→ pure_eb (part, npz, figures) +# ├─ integration part ─┬──→ pure_eb +# │ └──→ cosebis (part, npz, figures) # └─ reporting .txt ──→ 2pcf plot, ratio_xi_sys_xi +# CosmoCov ξ±, integration grid ──→ pure_eb, cosebis (their covariances) # catalogue ──→ pseudo_cl (part) ──┬─→ pseudo-Cl figures -# CosmoCov ──→ covariance ─────────┤ +# CosmoCov ξ±, reporting grid ─────┤ # rho/tau (part + FITS) ───────────┼─→ summarize_bmodes (reads the products) # └─→ assemble_sacc ──→ {version}.sacc # @@ -73,7 +73,7 @@ def cv_tau_stats(version): def _pure_eb_stub(version): """Shared stem of the pure-E/B diagnostic products (npz + figures).""" - eb = CV["integration"] + eb = XI_GRIDS["integration"] return str( COSMO_VAL / ( @@ -101,39 +101,44 @@ def cv_pure_eb_figures(version): } -def cv_xi_cov_integration(version): - """CosmoCov gaussian ξ± covariance on the integration grid. - - The covariance model the pure-E/B Monte Carlo draws from; gaussian because - the draws only need the scatter a Gaussian field would give. - """ - integ = CV["integration"] +def _grid_cov(version, grid, gaussian): + """CosmoCov-processed ξ± covariance on a named grid's own binning.""" + binning = XI_GRIDS[grid] return covariance_path( version, FIDUCIAL["blind"], - gaussian="g", - min_sep=integ["min_sep"], - max_sep=integ["max_sep"], - nbins=integ["nbins"], + gaussian=gaussian, + min_sep=binning["min_sep"], + max_sep=binning["max_sep"], + nbins=binning["nbins"], mask_suffix=DEFAULT_MASK_SUFFIX, ) +def cv_xi_cov_integration(version): + """CosmoCov gaussian ξ± covariance on the integration grid. + + The one analytic covariance both B-mode statistics take: the pure-E/B Monte + Carlo draws from it and COSEBIs carry it through their kernel. Gaussian + because B-modes only need the scatter a Gaussian field would give. + """ + return _grid_cov(version, "integration", "g") + + def _cosebis_stub(version): """Shared stem of the COSEBIs diagnostic products (npz + figures). - varmethod names where the covariance came from, and these products are the - propagated one — which also keeps them clear of the paths plot_cosebis - builds for its own byproducts, so nothing overwrites a declared output. + Named by the grid, where the covariance came from (varmethod), the mode + count and the fiducial scale cut. """ - cb = CV["cosebis"] + integ = XI_GRIDS["integration"] fsc = CV["fiducial_scale_cut"] return str( COSMO_VAL / ( - f"{version}_cosebis_minsep={cb['min_sep_int']}" - f"_maxsep={cb['max_sep_int']}_nbins={cb['nbins_int']}" - f"_npatch={cb['npatch']}_varmethod=propagated_nmodes={cb['nmodes']}" + f"{version}_cosebis_minsep={integ['min_sep']}" + f"_maxsep={integ['max_sep']}_nbins={integ['nbins']}" + f"_varmethod=analytic_nmodes={CV['cosebis']['nmodes']}" f"_scalecut={fsc[0]}-{fsc[1]}" ) ) @@ -168,16 +173,8 @@ def cv_pseudo_cl_cov(version): def cv_xi_cov(version): - """CosmoCov-processed ξ± covariance, on the reporting grid's own binning.""" - return covariance_path( - version, - FIDUCIAL["blind"], - gaussian="ng", - min_sep=CV["theta_min"], - max_sep=CV["theta_max"], - nbins=CV["nbins"], - mask_suffix=DEFAULT_MASK_SUFFIX, - ) + """CosmoCov ξ± covariance on the reporting grid, the terminal file's.""" + return _grid_cov(version, "reporting", "ng") def cv_cosebis_sacc(version): @@ -207,13 +204,9 @@ def cv_analysis_sacc(version): return str(COSMO_VAL / f"{version}.sacc") -# Common params block shared by every cosmo_val rule: the cv constructor kwargs -# plus the run directory the object must be instantiated in. +# Common params block shared by every cosmo_val rule: the cv constructor kwargs. def cv_params(version_list=None): - return dict( - cv_init=cv_init_params(config, version_list=version_list), - rundir=CV_RUNDIR, - ) + return dict(cv_init=cv_init_params(config, version_list=version_list)) # --------------------------------------------------------------------------- @@ -393,7 +386,6 @@ rule cv_plot_pseudo_cl: # Style is per catalogue, so the derived variants take their parent's. markers=[CATALOG_CONFIG[base_version(v)]["marker"] for v in CV_VERSIONS], colours=[CATALOG_CONFIG[base_version(v)]["colour"] for v in CV_VERSIONS], - rundir=CV_RUNDIR, resources: mem_mb=8000, runtime=20, @@ -427,7 +419,6 @@ rule cv_pure_eb: n_samples=CV.get("n_mc_samples", 1000), cosmo_params=CV["cosmo_params"], fiducial_scale_cut=CV["fiducial_scale_cut"], - rundir=CV_RUNDIR, threads: 24 resources: mem_mb=40000, @@ -437,26 +428,26 @@ rule cv_pure_eb: rule cv_cosebis: - """COSEBIs E/B decomposition for one version, from its ξ± part. + """COSEBIs E/B decomposition for one version, from its integration-grid part. - Values, covariance and PTEs all come from the part: the COSEBIs covariance - is the part's ξ± covariance through the same kernel as the modes. + The modes come from the part; the covariance is the CosmoCov ξ± covariance + on the same grid through the same kernel as the modes. """ input: - xi=lambda w: cv_xi_sacc(w.version, "cosebis"), + xi=lambda w: cv_xi_sacc(w.version, "integration"), + cov=lambda w: cv_xi_cov_integration(w.version), output: npz=cv_cosebis_npz("{version}"), sacc=cv_cosebis_sacc("{version}"), **cv_cosebis_figures("{version}"), params: version="{version}", - min_sep=CV["cosebis"]["min_sep_int"], - max_sep=CV["cosebis"]["max_sep_int"], - nbins=CV["cosebis"]["nbins_int"], + min_sep=XI_GRIDS["integration"]["min_sep"], + max_sep=XI_GRIDS["integration"]["max_sep"], + nbins=XI_GRIDS["integration"]["nbins"], nmodes=CV["cosebis"]["nmodes"], scale_cuts=CV["cosebis"]["scale_cuts"], fiducial_scale_cut=CV["fiducial_scale_cut"], - rundir=CV_RUNDIR, threads: 24 resources: mem_mb=48000, @@ -487,7 +478,6 @@ rule cv_summarize_bmodes: max_sep=CV["theta_max"], nbins=CV["nbins"], include_pseudo_cl=CV.get("include_pseudo_cl", False), - rundir=CV_RUNDIR, resources: mem_mb=8000, runtime=20, diff --git a/workflow/rules/covariance.smk b/workflow/rules/covariance.smk index e9185428..9c26b775 100644 --- a/workflow/rules/covariance.smk +++ b/workflow/rules/covariance.smk @@ -4,9 +4,9 @@ def get_cat_params(version): """Extract covariance parameters (area, n_e, sigma_e) from catalog config.""" base_version = version.replace("_leak_corr", "") - if base_version not in config: + if base_version not in CATALOG_CONFIG: raise KeyError(f"Catalog configuration not found for {base_version}") - cov_th = config[base_version]["cov_th"] + cov_th = CATALOG_CONFIG[base_version]["cov_th"] return cov_th["A"], cov_th["n_e"], cov_th["sigma_e"] diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index cc992391..56097e6a 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -98,6 +98,7 @@ if _missing_structural: # stack). Binds come from the driving profile's ``apptainer-args``. A null # ``sif`` resolves to the workflow's one image (see workflow/image_sims/config.yaml). SIF = common.resolve_container(IMSIM["sif"]) +common.check_host_parity(SIF) # --- repositories (bound into the image; branch code overrides) ----------- SHAPEPIPE_REPO = IMSIM["shapepipe_repo"] diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 1b6663b6..74fe0569 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -13,16 +13,16 @@ def xi_binning(grid): return grid_binning(XI_GRIDS[grid]) -def xi_grid_of(wildcards): - """Grid label for the binning a job was requested with.""" - return grid_of(XI_GRIDS, {key: getattr(wildcards, key) for key in XI_KEYS}) +def xi_binning_of(wildcards): + """The binning a job was requested with, from its wildcards.""" + return {key: getattr(wildcards, key) for key in XI_KEYS} rule xi: """TreeCorr ξ±(θ) for one version on one angular grid. One rule for every grid: outputs are named by their binning, so a request - binds the wildcards and `xi_grid_of` resolves the grid label from them. + binds the wildcards and the grid label and covariance resolve from them. """ input: catalog=get_shear_catalog, @@ -38,8 +38,8 @@ rule xi: npatch="{npatch}", cat_config=CAT_CONFIG, output_dir=str(COSMO_VAL), - grid=lambda w: xi_grid_of(w), - cov=lambda w: XI_GRIDS[xi_grid_of(w)]["cov"], + grid=lambda w: grid_of(XI_GRIDS, xi_binning_of(w)), + cov=lambda w: grid_cov(XI_GRIDS, xi_binning_of(w)), resources: # The fine integration grid needs more memory and wall time than the # ~20-bin reporting one; scale on nbins rather than splitting the rule. diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index c9241eaa..d431b470 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -1,11 +1,12 @@ """Rule cv_cosebis: COSEBIs E/B decomposition for one version. -A consumer of the ξ± part alone — values, covariance, PTEs and figures all -derive from it, so nothing here touches a catalogue. The part's ξ± covariance -goes through the same linear kernel as the modes to give the COSEBIs -covariance; its ``npatch`` metadata sets the Hartlap debiasing. +A consumer of the integration-grid ξ± part and the CosmoCov ξ± covariance on +the same grid — nothing here touches a catalogue. The covariance goes through +the same linear kernel as the modes to give the COSEBIs covariance; it is +analytic, so no Hartlap debiasing applies. """ +import numpy as np from cv_runner import _unbuffer_streams, verify_outputs from sp_validation import sacc_io @@ -26,18 +27,17 @@ fiducial_scale_cut = tuple(p["fiducial_scale_cut"]) part = sacc_io.load(snakemake.input["xi"]) -theta, xip, xim = sacc_io.get_xi(part, (0, 0), grid="cosebis") +theta, xip, xim = sacc_io.get_xi(part, (0, 0), grid="integration") edges = log_bin_edges(p["min_sep"], p["max_sep"], p["nbins"]) results = cosebis_scan_from_xi( theta, xip, xim, - part.covariance.dense, + np.loadtxt(snakemake.input["cov"]), *edges, nmodes=p["nmodes"], scale_cuts=[tuple(sc) for sc in p["scale_cuts"]], - npatch=part.metadata["npatch"], ) fiducial_key = find_conservative_scale_cut_key(results, fiducial_scale_cut) @@ -50,7 +50,7 @@ fiducial_scale_cut=fiducial_scale_cut, ) plot_cosebis_covariance_matrix( - fiducial, version, "jackknife", snakemake.output["figure_covariance"] + fiducial, version, "analytic", snakemake.output["figure_covariance"] ) plot_cosebis_scale_cut_heatmap( results, diff --git a/workflow/scripts/cv_runner.py b/workflow/scripts/cv_runner.py index 040cbe8a..601f2e14 100644 --- a/workflow/scripts/cv_runner.py +++ b/workflow/scripts/cv_runner.py @@ -28,15 +28,11 @@ def _unbuffer_streams(): def make_cv(snakemake): """Build a CosmologyValidation from a rule's ``snakemake.params``. - ``params["cv_init"]`` is the kwargs dict assembled by common.cv_init_params. - The catalogue config path arrives explicitly in ``cv_init``; the object is - created with the run directory as cwd so it writes under ``output/`` - exactly as interactive runs do. + ``params["cv_init"]`` is the kwargs dict assembled by common.cv_init_params, + which names the catalogue config and the output directory explicitly. """ from sp_validation.cosmo_val import CosmologyValidation - rundir = snakemake.params["rundir"] - os.chdir(rundir) return CosmologyValidation(**dict(snakemake.params["cv_init"])) diff --git a/workflow/scripts/cv_summarize_bmodes.py b/workflow/scripts/cv_summarize_bmodes.py index 9200bfaa..ec44142a 100644 --- a/workflow/scripts/cv_summarize_bmodes.py +++ b/workflow/scripts/cv_summarize_bmodes.py @@ -42,7 +42,7 @@ # this table wants. cosebis = np.load(snakemake.input["cosebis"][i]) row["COSEBIS"] = float(cosebis["pte_B"]) - cov_methods.add("COSEBIs: propagated from the ξ± covariance") + cov_methods.add("COSEBIs: analytic (CosmoCov ξ± through the COSEBIs kernel)") if p["include_pseudo_cl"]: from astropy.io import fits diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 1f69ee6d..ed1ee69e 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -20,8 +20,8 @@ grid configures (``--cov``): the dense jackknife estimate from the patches, the TreeCorr ``varxip``/``varxim`` diagonal, or none. -``output_dir`` is passed explicitly (rather than via the ``COSMO_VAL`` env hook) -so lc can point each run at its own ``{output}`` tree. +``output_dir`` is passed explicitly so lc can point each run at its own +``{output}`` tree. """ import argparse diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py new file mode 100644 index 00000000..969b8faf --- /dev/null +++ b/workflow/tests/conftest.py @@ -0,0 +1,231 @@ +"""Host-side harness: the workflow as a person launches it. + +These tests run under the host launcher, never inside the image:: + + uv run --isolated --no-project --python 3.12 --with snakemake==9.23.1 \\ + --with pytest --with numpy pytest workflow/tests + +``sp_validation`` is absent from that environment, so every Snakefile has to +parse with the standard library and Snakemake alone -- the condition a host +Snakemake is in. The ``candide`` tests drive the candide profile and so also +need ``--with snakemake-executor-plugin-slurm``; CI deselects them. + +The ``toy`` fixture is a disposable checkout: copies of ``workflow/`` and +``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, a one-catalogue +``cosmo_val/cat_config.yaml``, a touched catalogue file, the processed CosmoCov +covariances already in place (their inputs live on candide), and both output +roots in tmp. Its runs use a fake image whose Python and Snakemake match the +running ones, so the launch-time parity check passes without apptainer. +""" + +import dataclasses +import importlib.util +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest +import snakemake +import yaml + +REPO = Path(__file__).resolve().parents[2] + +# The toy catalogue and its leakage-corrected variant. +VERSIONS = ("SP_v0.1", "SP_v0.1_leak_corr") + +HOST_PYTHON = ".".join(str(v) for v in sys.version_info[:3]) + + +def fake_image(root, python=HOST_PYTHON, snakemake_version=snakemake.__version__): + """A sandbox-shaped directory carrying only what the parity check reads.""" + venv = Path(root) / "app" / ".venv" + minor = ".".join(python.split(".")[:2]) + site = venv / "lib" / f"python{minor}" / "site-packages" + (site / f"snakemake-{snakemake_version}.dist-info").mkdir(parents=True) + (venv / "pyvenv.cfg").write_text( + f"home = /usr/local/bin\nimplementation = CPython\nversion_info = {python}\n" + ) + return Path(root) + + +@dataclasses.dataclass +class Job: + rule: str + input: list + output: list + wildcards: dict + + +_RULE = re.compile(r"^(?:local)?rule (\w+):$") +_FIELD = re.compile(r"^ (\w+): (.*)$") + + +def parse_jobs(text): + """The jobs a ``snakemake -n`` listing schedules.""" + jobs, fields = [], None + for line in text.splitlines(): + if match := _RULE.match(line): + fields = {"rule": match[1]} + jobs.append(fields) + elif fields is not None and (match := _FIELD.match(line)): + fields[match[1]] = match[2] + else: + fields = None + return [ + Job( + rule=f["rule"], + input=f["input"].split(", ") if "input" in f else [], + output=f["output"].split(", ") if "output" in f else [], + wildcards=dict( + pair.split("=", 1) + for pair in f.get("wildcards", "").split(", ") + if pair + ), + ) + for f in jobs + ] + + +def _load_module(path, name, env): + """Import a workflow module by path under ``env`` (it reads env at import).""" + saved = os.environ.copy() + os.environ.update(env) + try: + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + finally: + os.environ.clear() + os.environ.update(saved) + return module + + +@dataclasses.dataclass +class Toy: + root: Path + rundir: Path + cosmo_val: Path + cosmo_inference: Path + image: Path + env: dict + config: dict + common: object + + def snakemake(self, *args, container=None, timeout=300): + """Run the host Snakemake in the toy's paper directory. + + ``container`` overrides the image (default: the matching fake one). + """ + cmd = [sys.executable, "-m", "snakemake", "--cores", "1", *args] + cmd += ["--config", f"container={container or self.image}"] + return subprocess.run( + cmd, + cwd=self.rundir, + env=self.env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + check=False, + ) + + +def _cat_config(catalogue): + entry = { + "subdir": str(catalogue.parent), + "pipeline": "SP", + "colour": "orange", + "marker": "^", + "cov_th": {"A": 100.0, "n_e": 5.0, "sigma_e": 0.3}, + "shear": { + "path": str(catalogue), + "redshift_path": str(catalogue.parent / "nz_SP_v0.1_A.txt"), + "w_col": "w", + "e1_col": "e1", + "e2_col": "e2", + "e1_col_corrected": "e1_leak_corrected", + "e2_col_corrected": "e2_leak_corrected", + }, + } + return {VERSIONS[0]: entry, "paths": {"output": "./output"}} + + +@pytest.fixture(scope="session") +def toy(tmp_path_factory): + root = tmp_path_factory.mktemp("toy") + skip = shutil.ignore_patterns(".snakemake", "__pycache__", "tests") + shutil.copytree(REPO / "workflow", root / "workflow", ignore=skip) + shutil.copytree( + REPO / "papers" / "cosmo_val", root / "papers" / "cosmo_val", ignore=skip + ) + (root / "src").symlink_to(REPO / "src") + + catalogue = root / "data" / "toy_shear.fits" + catalogue.parent.mkdir() + catalogue.touch() + (root / "cosmo_val").mkdir() + (root / "cosmo_val" / "cat_config.yaml").write_text( + yaml.safe_dump(_cat_config(catalogue)) + ) + + rundir = root / "papers" / "cosmo_val" + config_path = rundir / "config" / "config.yaml" + config = yaml.safe_load(config_path.read_text()) + config["versions"] = list(VERSIONS) + config["fiducial"]["version"] = VERSIONS[1] + config["fiducial"]["mock_version"] = VERSIONS[0] + config_path.write_text(yaml.safe_dump(config, sort_keys=False)) + + env = { + k: v + for k, v in os.environ.items() + if k not in ("SNAKEMAKE_PROFILE", "APPTAINERENV_PYTHONPATH") + } + env.update( + COSMO_VAL=str(root / "out" / "cosmo_val"), + COSMO_INFERENCE=str(root / "out" / "cosmo_inference"), + XDG_CACHE_HOME=str(root / "cache"), + TMPDIR=str(tmp_path_factory.getbasetemp()), + PYTHONUNBUFFERED="1", + PYTHONNOUSERSITE="1", + ) + common = _load_module(root / "workflow" / "common.py", "toy_common", env) + + # The processed CosmoCov covariances the cosmo_val rules read, in place. + grids = common.xi_grids(config, config["fiducial"]) + mask = "_masked" if config["covariance"].get("default_masked") else "" + for version in VERSIONS: + for gaussian, grid in (("ng", grids["reporting"]), ("g", grids["integration"])): + path = Path( + common.covariance_path( + version, + config["fiducial"]["blind"], + gaussian, + grid["min_sep"], + grid["max_sep"], + grid["nbins"], + mask, + ) + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + + return Toy( + root=root, + rundir=rundir, + cosmo_val=Path(env["COSMO_VAL"]), + cosmo_inference=Path(env["COSMO_INFERENCE"]), + image=fake_image(root / "image"), + env=env, + config=config, + common=common, + ) + + +on_candide = pytest.mark.skipif( + not Path("/n17data/cdaley/unions").exists() or shutil.which("apptainer") is None, + reason="needs candide: /n17data and apptainer", +) diff --git a/src/sp_validation/tests/data/container_smoke/Snakefile b/workflow/tests/data/container_smoke/Snakefile similarity index 89% rename from src/sp_validation/tests/data/container_smoke/Snakefile rename to workflow/tests/data/container_smoke/Snakefile index 75527690..0d99f26a 100644 --- a/src/sp_validation/tests/data/container_smoke/Snakefile +++ b/workflow/tests/data/container_smoke/Snakefile @@ -1,4 +1,4 @@ -# Standalone workflow exercised by src/sp_validation/tests/test_container_smoke.py. +# Standalone workflow exercised by workflow/tests/test_container_smoke.py. # # The module-level `container:` below mirrors what every real workflow does # (workflow/Snakefile) -- Snakemake has no way to take a default image from a diff --git a/src/sp_validation/tests/data/container_smoke/container_smoke.py b/workflow/tests/data/container_smoke/container_smoke.py similarity index 94% rename from src/sp_validation/tests/data/container_smoke/container_smoke.py rename to workflow/tests/data/container_smoke/container_smoke.py index 274cd85c..c01eac72 100644 --- a/src/sp_validation/tests/data/container_smoke/container_smoke.py +++ b/workflow/tests/data/container_smoke/container_smoke.py @@ -15,7 +15,7 @@ container -- proves /home is bound and usable, not just readable). Driven by the co-located Snakefile; the assertions on the output YAML live in -src/sp_validation/tests/test_container_smoke.py (marked ``slow``, cluster only). +workflow/tests/test_container_smoke.py (candide only). """ import os @@ -51,11 +51,11 @@ } # --- provenance: what commit is actually running in the container --------- -# src/sp_validation/tests/data/container_smoke/ -> repo root, five levels up. +# workflow/tests/data/container_smoke/ -> repo root, four levels up. # (This is the checkout the Snakefile came from, which is what we want to # report; the editable install may well resolve to a *different* checkout.) repo_dir = os.path.abspath( - os.path.join(os.path.dirname(os.path.abspath(__file__)), *([os.pardir] * 5)) + os.path.join(os.path.dirname(os.path.abspath(__file__)), *([os.pardir] * 4)) ) try: commit = subprocess.run( diff --git a/workflow/tests/pytest.ini b/workflow/tests/pytest.ini new file mode 100644 index 00000000..53d26a02 --- /dev/null +++ b/workflow/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + candide: needs candide (its disks, apptainer and your image); CI deselects these with -m "not candide" diff --git a/src/sp_validation/tests/test_container_smoke.py b/workflow/tests/test_container_smoke.py similarity index 62% rename from src/sp_validation/tests/test_container_smoke.py rename to workflow/tests/test_container_smoke.py index b9b07058..265f5c37 100644 --- a/src/sp_validation/tests/test_container_smoke.py +++ b/workflow/tests/test_container_smoke.py @@ -1,10 +1,9 @@ -"""Smoke test of the profile-driven containerized-SLURM path. +"""P5: one real SLURM job through the committed candide profile. -Submits one real (tiny, 5-minute) SLURM job through the committed candide -profile. The executor, the apptainer deployment method and the bind mounts come -from that profile; the image is the module-level ``container:`` in the test +The executor, the apptainer deployment method and the bind mounts come from +that profile; the image is the module-level ``container:`` in the test Snakefile, exactly as real workflows declare it. That contract is what's under -test, so this can only run on candide -- marked ``slow``, skipped elsewhere. +test, so it runs only on a candide submit host. The job writes a YAML report (see data/container_smoke/container_smoke.py); the assertions below check what it reports. @@ -14,24 +13,16 @@ import re import shutil import subprocess +import sys import tempfile from pathlib import Path import numpy as np import pytest import yaml +from conftest import REPO, on_candide -requires_cluster = pytest.mark.skipif( - not Path("/n17data/cdaley/unions").exists() or shutil.which("sbatch") is None, - reason="needs candide: /n17data and a SLURM submit host", -) - - -def _repo_root() -> Path: - for parent in Path(__file__).resolve().parents: - if (parent / "pyproject.toml").exists(): - return parent - raise RuntimeError("could not locate repo root (no pyproject.toml above test)") +SMOKE = Path(__file__).resolve().parent / "data" / "container_smoke" def _reference_eigenvalues() -> np.ndarray: @@ -43,39 +34,35 @@ def _reference_eigenvalues() -> np.ndarray: def test_smoke_snakefile_names_the_workflow_image(): """The test Snakefile's literal image must track the package's CONTAINER_URI.""" - repo_root = _repo_root() uri = re.search( r'^CONTAINER_URI = "(.+)"$', - (repo_root / "src/sp_validation/container.py").read_text(), + (REPO / "src/sp_validation/container.py").read_text(), re.MULTILINE, ).group(1) - snakefile = ( - repo_root / "src/sp_validation/tests/data/container_smoke/Snakefile" - ).read_text() - assert f'"{uri}"' in snakefile, uri + assert f'"{uri}"' in (SMOKE / "Snakefile").read_text(), uri -@pytest.mark.slow -@requires_cluster +@pytest.mark.candide +@on_candide +@pytest.mark.skipif(shutil.which("sbatch") is None, reason="needs a SLURM submit host") def test_container_smoke(): - repo_root = _repo_root() - workflow_dir = repo_root / "src/sp_validation/tests/data/container_smoke" - - # Not pytest's tmp_path: that lives in the login node's /tmp, which the + # Not pytest's tmp_path: that lives in the submit host's /tmp, which the # compute node cannot see, so the job's output would "go missing". The # workdir must be on a shared filesystem. - tmp_path = Path(tempfile.mkdtemp(prefix="container_smoke_", dir=Path.home())) + workdir = Path(tempfile.mkdtemp(prefix="container_smoke_", dir=Path.home())) env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} result = subprocess.run( [ + sys.executable, + "-m", "snakemake", "--profile", - str(repo_root / "workflow/profiles/candide"), + str(REPO / "workflow/profiles/candide"), "-s", - str(workflow_dir / "Snakefile"), + str(SMOKE / "Snakefile"), "--directory", - str(tmp_path), + str(workdir), "--jobs", "1", "container_smoke", @@ -89,7 +76,7 @@ def test_container_smoke(): ) assert result.returncode == 0, result.stdout - report = yaml.safe_load((tmp_path / "results/container_smoke.yaml").read_text()) + report = yaml.safe_load((workdir / "results/container_smoke.yaml").read_text()) # The job ran inside the image, not on the bare host. Everything below would # pass on the host too, so this is the assertion that makes them mean @@ -97,7 +84,7 @@ def test_container_smoke(): assert report["container"]["apptainer_container"] != "unset", report["container"] # The install must resolve to an editable src/ checkout, not a site-packages - # copy. Note it need not be *this* checkout: the container's editable install + # copy. It need not be *this* checkout: the container's editable install # points at the shared /n17data working tree, while the Snakefile under test # is read from wherever the test runs. module_file = Path(report["sp_validation"]["file"]) @@ -123,4 +110,4 @@ def test_container_smoke(): "provenance" ] - shutil.rmtree(tmp_path) # keep only on failure, for post-mortem + shutil.rmtree(workdir) # keep only on failure, for post-mortem diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py new file mode 100644 index 00000000..993ba9a1 --- /dev/null +++ b/workflow/tests/test_dag.py @@ -0,0 +1,146 @@ +"""DAG properties, checked through the host launcher (see conftest.py).""" + +import importlib.util +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from conftest import HOST_PYTHON, REPO, VERSIONS, fake_image, on_candide, parse_jobs + + +def test_assemble_resolves(toy): + """P1: the terminal files resolve from a catalogue config and a checkout.""" + result = toy.snakemake("-n", "assemble_sacc_all") + assert result.returncode == 0, result.stdout + assembled = {Path(o).name for j in parse_jobs(result.stdout) for o in j.output} + assert {f"{v}.sacc" for v in VERSIONS} <= assembled, result.stdout + + +def test_one_integration_grid(toy): + """P1: ξ± is measured on two grids, and both B-mode statistics share one. + + COSEBIs and pure-E/B read the same integration-grid part and the same + CosmoCov covariance on that grid; no other binning is measured. + """ + result = toy.snakemake("-n", "assemble_sacc_all") + assert result.returncode == 0, result.stdout + jobs = parse_jobs(result.stdout) + grids = toy.common.xi_grids(toy.config, toy.config["fiducial"]) + + measured = { + (j.wildcards["version"], toy.common.grid_of(grids, j.wildcards)) + for j in jobs + if j.rule == "xi" + } + assert len([j for j in jobs if j.rule == "xi"]) == len(measured), result.stdout + assert measured == {(v, g) for v in VERSIONS for g in grids}, measured + assert set(grids) == {"reporting", "integration"} + + tag = toy.common.grid_binning(grids["integration"]) + for version in VERSIONS: + by_rule = { + j.rule: set(j.input) for j in jobs if j.wildcards.get("version") == version + } + part = str(toy.cosmo_val / f"{version}_xi_{tag}.sacc") + (covariance,) = [ + f for f in by_rule["cv_pure_eb"] if Path(f).name.startswith("covariance_") + ] + assert "_g_" in Path(covariance).name + assert by_rule["cv_cosebis"] == {part, covariance}, by_rule["cv_cosebis"] + assert part in by_rule["cv_pure_eb"] + + +def test_outputs_stay_in_the_output_roots(toy): + """P2: nothing the DAG declares lands outside the configured output roots.""" + result = toy.snakemake("-n", "all") + assert result.returncode == 0, result.stdout + roots = [ + r.resolve() + for r in (toy.cosmo_val, toy.cosmo_inference, toy.rundir / "results") + ] + outputs = [Path(o) for j in parse_jobs(result.stdout) for o in j.output] + assert outputs, result.stdout + strays = [ + o + for o in outputs + if not any((toy.rundir / o).resolve().is_relative_to(r) for r in roots) + ] + assert not strays, strays + + +@pytest.mark.parametrize( + "python, snakemake_version", + [("3.13.1", None), (None, "9.0.0")], + ids=["python-minor", "snakemake"], +) +def test_image_parity_is_checked_at_launch(toy, tmp_path, python, snakemake_version): + """P3: an image whose Python minor or Snakemake differs stops the launch.""" + image = fake_image( + tmp_path / "image", + **({"python": python} if python else {}), + **({"snakemake_version": snakemake_version} if snakemake_version else {}), + ) + result = toy.snakemake("-n", "assemble_sacc_all", container=image) + assert result.returncode != 0, result.stdout + minor = ".".join((python or HOST_PYTHON).split(".")[:2]) + assert f"uv tool install --force --python {minor} snakemake==" in result.stdout + assert "rule assemble_sacc" not in result.stdout + + +def test_unreadable_image_is_named_not_fatal(toy): + """P3: a registry tag cannot be inspected; the launch says so and goes on.""" + result = toy.snakemake( + "-n", "assemble_sacc_all", container="docker://example.org/image:tag" + ) + assert result.returncode == 0, result.stdout + assert "parity unchecked" in result.stdout + + +def _real_dry_run(paper, target): + env = {k: v for k, v in os.environ.items() if k != "SNAKEMAKE_PROFILE"} + env.update(PYTHONUNBUFFERED="1", PYTHONNOUSERSITE="1") + return subprocess.run( + [ + sys.executable, + "-m", + "snakemake", + "-n", + "--profile", + str(REPO / "workflow" / "profiles" / "candide"), + target, + ], + cwd=REPO / "papers" / paper, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=600, + check=False, + ) + + +@pytest.mark.candide +@on_candide +@pytest.mark.parametrize( + "paper, target", [("cosmo_val", "assemble_sacc_all"), ("bmodes", "all_tapestry")] +) +def test_papers_resolve_on_candide(paper, target): + """P4: the real paper DAGs resolve against the real catalogues and image. + + A local image (your SIF or sandbox) is read, so parity was checked. + """ + result = _real_dry_run(paper, target) + assert result.returncode == 0, result.stdout + if _resolve_image()[1] != "tag": + assert "parity unchecked" not in result.stdout, result.stdout + + +def _resolve_image(): + spec = importlib.util.spec_from_file_location( + "container", REPO / "src" / "sp_validation" / "container.py" + ) + container = importlib.util.module_from_spec(spec) + spec.loader.exec_module(container) + return container.resolve_image() From 1412a540955203c6ebe32a9eb18f0ac51c27b908 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 25 Sep 2026 23:20:34 +0200 Subject: [PATCH 071/160] workflow: the image lives under ~/.cache; launch guards read the real image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - container.CACHE_DIR is ~/.cache/sp_validation whatever XDG_CACHE_HOME says. A job runs the image from the path the launching host resolved, and on candide XDG_CACHE_HOME is node-local /scratch: from such a shell the launch fell back to the registry tag and skipped the parity check. SPV_CONTAINER / SPV_SANDBOX remain the overrides. - test_launch_reads_the_image_under_home: a mismatched image under a fake ~/.cache stops the launch while XDG_CACHE_HOME points elsewhere. - test_papers_resolve_on_candide asserts unconditionally that the launch read a local image (no "parity unchecked"). - test_image_sims_checks_parity_at_launch: the standalone image-sims Snakefile stops on a mismatched image. - test_assemble_resolves pins each terminal file's inputs: the reporting ξ± part with its CosmoCov ng covariance, the fiducial-binning pseudo-Cl part with its NaMaster covariance, COSEBIs, pure-E/B and ρ/τ. - The container smoke test launches as the README does (no --jobs), so the candide profile's job bound is under test. - Test docstrings drop the design-table row IDs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/container.py | 5 +- workflow/tests/conftest.py | 23 ++++-- workflow/tests/test_container_smoke.py | 13 ++-- workflow/tests/test_dag.py | 104 +++++++++++++++++++------ 4 files changed, 105 insertions(+), 40 deletions(-) diff --git a/src/sp_validation/container.py b/src/sp_validation/container.py index ff365823..ba37939a 100755 --- a/src/sp_validation/container.py +++ b/src/sp_validation/container.py @@ -46,7 +46,10 @@ # per branch, sanitized, so ``:develop`` tracks the integration branch. CONTAINER_URI = "docker://ghcr.io/cosmostat/sp_validation:develop" -CACHE_DIR = Path(os.environ.get("XDG_CACHE_HOME", "~/.cache")) / "sp_validation" +# Under the home directory, which every node mounts: a job runs the image from +# the path the launching host resolved, so the image cannot sit on node-local +# storage -- where a cluster's XDG_CACHE_HOME often points. +CACHE_DIR = Path("~/.cache/sp_validation") # Where this user's image lives. Per-user by construction: one file, one owner, # no coordination. Override with ``SPV_CONTAINER`` (an absolute path). diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py index 969b8faf..cf3733a9 100644 --- a/workflow/tests/conftest.py +++ b/workflow/tests/conftest.py @@ -113,18 +113,22 @@ class Toy: env: dict config: dict common: object + covariances: dict # (version, "g" | "ng") -> the processed CosmoCov file - def snakemake(self, *args, container=None, timeout=300): - """Run the host Snakemake in the toy's paper directory. + def snakemake(self, *args, container=None, cwd=None, env=None, timeout=300): + """Run the host Snakemake in the toy's paper directory, or in ``cwd``. - ``container`` overrides the image (default: the matching fake one). + ``container`` overrides the image (default: the matching fake one; + ``False`` leaves the choice to the launch). ``env`` replaces the toy's + environment. """ cmd = [sys.executable, "-m", "snakemake", "--cores", "1", *args] - cmd += ["--config", f"container={container or self.image}"] + if container is not False: + cmd += ["--config", f"container={container or self.image}"] return subprocess.run( cmd, - cwd=self.rundir, - env=self.env, + cwd=cwd or self.rundir, + env=env or self.env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -188,6 +192,9 @@ def toy(tmp_path_factory): COSMO_VAL=str(root / "out" / "cosmo_val"), COSMO_INFERENCE=str(root / "out" / "cosmo_inference"), XDG_CACHE_HOME=str(root / "cache"), + # No local image: each launch runs the image its test names. + SPV_CONTAINER=str(root / "cache" / "absent.sif"), + SPV_SANDBOX=str(root / "cache" / "absent-sandbox"), TMPDIR=str(tmp_path_factory.getbasetemp()), PYTHONUNBUFFERED="1", PYTHONNOUSERSITE="1", @@ -197,9 +204,10 @@ def toy(tmp_path_factory): # The processed CosmoCov covariances the cosmo_val rules read, in place. grids = common.xi_grids(config, config["fiducial"]) mask = "_masked" if config["covariance"].get("default_masked") else "" + covariances = {} for version in VERSIONS: for gaussian, grid in (("ng", grids["reporting"]), ("g", grids["integration"])): - path = Path( + path = covariances[version, gaussian] = Path( common.covariance_path( version, config["fiducial"]["blind"], @@ -222,6 +230,7 @@ def toy(tmp_path_factory): env=env, config=config, common=common, + covariances=covariances, ) diff --git a/workflow/tests/test_container_smoke.py b/workflow/tests/test_container_smoke.py index 265f5c37..8e4a350b 100644 --- a/workflow/tests/test_container_smoke.py +++ b/workflow/tests/test_container_smoke.py @@ -1,9 +1,10 @@ -"""P5: one real SLURM job through the committed candide profile. +"""One real SLURM job through the committed candide profile. -The executor, the apptainer deployment method and the bind mounts come from -that profile; the image is the module-level ``container:`` in the test -Snakefile, exactly as real workflows declare it. That contract is what's under -test, so it runs only on a candide submit host. +The executor, the apptainer deployment method, the bind mounts and the job +bound come from that profile, launched as the README launches a target; the +image is the module-level ``container:`` in the test Snakefile, exactly as real +workflows declare it. That contract is what's under test, so it runs only on a +candide submit host. The job writes a YAML report (see data/container_smoke/container_smoke.py); the assertions below check what it reports. @@ -63,8 +64,6 @@ def test_container_smoke(): str(SMOKE / "Snakefile"), "--directory", str(workdir), - "--jobs", - "1", "container_smoke", ], env=env, diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 993ba9a1..37eda603 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -1,25 +1,45 @@ """DAG properties, checked through the host launcher (see conftest.py).""" -import importlib.util import os import subprocess import sys from pathlib import Path import pytest +import yaml from conftest import HOST_PYTHON, REPO, VERSIONS, fake_image, on_candide, parse_jobs def test_assemble_resolves(toy): - """P1: the terminal files resolve from a catalogue config and a checkout.""" + """Each terminal file gathers every part and the analytic covariances. + + The ξ± block takes the CosmoCov covariance on the reporting grid, and the + harmonic block is the part on the fiducial harmonic binning with the + NaMaster covariance of that same binning. + """ result = toy.snakemake("-n", "assemble_sacc_all") assert result.returncode == 0, result.stdout - assembled = {Path(o).name for j in parse_jobs(result.stdout) for o in j.output} - assert {f"{v}.sacc" for v in VERSIONS} <= assembled, result.stdout + jobs = [j for j in parse_jobs(result.stdout) if j.rule == "assemble_sacc"] + grids = toy.common.xi_grids(toy.config, toy.config["fiducial"]) + reporting = toy.common.grid_binning(grids["reporting"]) + harmonic = toy.common.pseudo_cl_tag(toy.config) + assert sorted(j.wildcards["version"] for j in jobs) == sorted(VERSIONS) + for job in jobs: + version = job.wildcards["version"] + assert [Path(o).name for o in job.output] == [f"{version}.sacc"] + assert {Path(f).name for f in job.input} == { + f"{version}_xi_{reporting}.sacc", + toy.covariances[version, "ng"].name, + f"pseudo_cl_{version}_{harmonic}.sacc", + f"pseudo_cl_cov_{version}_{harmonic}.fits", + f"{version}_cosebis.sacc", + f"{version}_pure_eb.sacc", + f"rho_tau_{version}_{reporting}.sacc", + }, job.input def test_one_integration_grid(toy): - """P1: ξ± is measured on two grids, and both B-mode statistics share one. + """ξ± is measured on two grids, and both B-mode statistics share one. COSEBIs and pure-E/B read the same integration-grid part and the same CosmoCov covariance on that grid; no other binning is measured. @@ -44,16 +64,13 @@ def test_one_integration_grid(toy): j.rule: set(j.input) for j in jobs if j.wildcards.get("version") == version } part = str(toy.cosmo_val / f"{version}_xi_{tag}.sacc") - (covariance,) = [ - f for f in by_rule["cv_pure_eb"] if Path(f).name.startswith("covariance_") - ] - assert "_g_" in Path(covariance).name + covariance = str(toy.covariances[version, "g"]) assert by_rule["cv_cosebis"] == {part, covariance}, by_rule["cv_cosebis"] - assert part in by_rule["cv_pure_eb"] + assert {part, covariance} <= by_rule["cv_pure_eb"], by_rule["cv_pure_eb"] def test_outputs_stay_in_the_output_roots(toy): - """P2: nothing the DAG declares lands outside the configured output roots.""" + """Nothing the suite declares lands outside the configured output roots.""" result = toy.snakemake("-n", "all") assert result.returncode == 0, result.stdout roots = [ @@ -76,7 +93,7 @@ def test_outputs_stay_in_the_output_roots(toy): ids=["python-minor", "snakemake"], ) def test_image_parity_is_checked_at_launch(toy, tmp_path, python, snakemake_version): - """P3: an image whose Python minor or Snakemake differs stops the launch.""" + """An image whose Python minor or Snakemake differs stops the launch.""" image = fake_image( tmp_path / "image", **({"python": python} if python else {}), @@ -90,7 +107,7 @@ def test_image_parity_is_checked_at_launch(toy, tmp_path, python, snakemake_vers def test_unreadable_image_is_named_not_fatal(toy): - """P3: a registry tag cannot be inspected; the launch says so and goes on.""" + """A registry tag cannot be inspected; the launch says so and goes on.""" result = toy.snakemake( "-n", "assemble_sacc_all", container="docker://example.org/image:tag" ) @@ -98,6 +115,50 @@ def test_unreadable_image_is_named_not_fatal(toy): assert "parity unchecked" in result.stdout +def test_launch_reads_the_image_under_home(toy, tmp_path): + """The launch finds your image under ~/.cache, whatever XDG_CACHE_HOME says. + + Jobs on other nodes run the image from the path the launching host + resolved, and a cluster's XDG_CACHE_HOME is often node-local. The image here + reports another Python, so reading it stops the launch. + """ + home = tmp_path / "home" + fake_image(home / ".cache" / "sp_validation" / "sandbox", python="3.13.1") + env = {k: v for k, v in toy.env.items() if not k.startswith("SPV_")} + env.update(HOME=str(home), XDG_CACHE_HOME=str(tmp_path / "node-local")) + result = toy.snakemake("-n", "assemble_sacc_all", container=False, env=env) + assert result.returncode != 0, result.stdout + assert "uv tool install --force --python 3.13 snakemake==" in result.stdout + + +def test_image_sims_checks_parity_at_launch(toy, tmp_path): + """The standalone image-sims workflow stops on a mismatched image too.""" + run = { + "image_sims": { + "sif": str(fake_image(tmp_path / "image", python="3.13.1")), + "grids_base": str(tmp_path / "grids"), + "mask_config": "mask.yaml", + "match_radius_deg": 0.0002, + "w_cols": ["none"], + "pair_match": True, + "n_bootstrap": 1, + "bootstrap_seed": 0, + } + } + (tmp_path / "run.yaml").write_text(yaml.safe_dump(run)) + result = toy.snakemake( + "-n", + "-s", + "workflow/image_sims/Snakefile", + "--configfile", + str(tmp_path / "run.yaml"), + container=False, + cwd=toy.root, + ) + assert result.returncode != 0, result.stdout + assert "uv tool install --force --python 3.13 snakemake==" in result.stdout + + def _real_dry_run(paper, target): env = {k: v for k, v in os.environ.items() if k != "SNAKEMAKE_PROFILE"} env.update(PYTHONUNBUFFERED="1", PYTHONNOUSERSITE="1") @@ -127,20 +188,13 @@ def _real_dry_run(paper, target): "paper, target", [("cosmo_val", "assemble_sacc_all"), ("bmodes", "all_tapestry")] ) def test_papers_resolve_on_candide(paper, target): - """P4: the real paper DAGs resolve against the real catalogues and image. + """The real paper DAGs resolve against the real catalogues and your image. - A local image (your SIF or sandbox) is read, so parity was checked. + Your image (the SIF or sandbox `spv-container` manages) is read, so parity + was checked. """ result = _real_dry_run(paper, target) assert result.returncode == 0, result.stdout - if _resolve_image()[1] != "tag": - assert "parity unchecked" not in result.stdout, result.stdout - - -def _resolve_image(): - spec = importlib.util.spec_from_file_location( - "container", REPO / "src" / "sp_validation" / "container.py" + assert "parity unchecked" not in result.stdout, ( + "no local image was read; run `spv-container pull`\n" + result.stdout ) - container = importlib.util.module_from_spec(spec) - spec.loader.exec_module(container) - return container.resolve_image() From d6073ce36932950b11a08556c0cc927aaa9e8a9f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 25 Sep 2026 23:20:38 +0200 Subject: [PATCH 072/160] =?UTF-8?q?workflow:=20a=20=CE=BE=C2=B1=20part's?= =?UTF-8?q?=20covariance=20follows=20its=20patches,=20in=20one=20place?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A grid is a binning; rule xi takes cov=patch_cov(npatch) directly, so grid_cov, _named_grid and the grids' cov key go. cv_init_params loses its unused version_list, and the cosmo_val rules pass one CV_INIT. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/tests/test_xi_grids.py | 27 +++++------- workflow/common.py | 52 ++++++++---------------- workflow/rules/cosmo_val.smk | 23 +++++------ workflow/rules/twopoint.smk | 5 ++- workflow/scripts/run_2pcf.py | 6 +-- 5 files changed, 44 insertions(+), 69 deletions(-) diff --git a/src/sp_validation/tests/test_xi_grids.py b/src/sp_validation/tests/test_xi_grids.py index 84ce3472..974fd4eb 100644 --- a/src/sp_validation/tests/test_xi_grids.py +++ b/src/sp_validation/tests/test_xi_grids.py @@ -82,29 +82,24 @@ def test_one_integration_grid(): assert set(common.xi_grids(CONFIG, FIDUCIAL)) == {"reporting", "integration"} -def test_covariance_mode_follows_the_patches(): - """Patched grids get a jackknife block, unpatched ones the diagonal. +@pytest.mark.parametrize( + "npatch, cov", [(1, "diagonal"), ("1", "diagonal"), ("100", "jackknife")] +) +def test_covariance_mode_follows_the_patches(npatch, cov): + """A patched part gets a jackknife block, an unpatched one the diagonal. Every part then carries variances: at npatch=1 TreeCorr's var_method is - "shot", and its diagonal is the estimate it has. + "shot", and its diagonal is the estimate it has. The rule passes the + wildcard, a string. """ - grids = common.xi_grids(CONFIG, FIDUCIAL) - assert grids["reporting"]["cov"] == "jackknife" - assert grids["integration"]["cov"] == "diagonal" - + assert common.patch_cov(npatch) == cov -@pytest.mark.parametrize("npatch, cov", [(1, "diagonal"), (50, "jackknife")]) -def test_unnamed_binning_is_a_reporting_measurement(npatch, cov): - """The paper's convergence-check binning belongs to no named grid. - It is tagged as a reporting measurement, and its covariance follows its own - patches rather than the reporting grid's. - """ +def test_unnamed_binning_is_a_reporting_measurement(): + """The paper's convergence-check binning belongs to no named grid.""" grids = common.xi_grids(CONFIG, FIDUCIAL) - stray = {"min_sep": 1.0, "max_sep": 250.0, "nbins": 10000, "npatch": npatch} + stray = {"min_sep": 1.0, "max_sep": 250.0, "nbins": 10000, "npatch": 1} assert common.grid_of(grids, stray) == "reporting" - assert common.grid_cov(grids, stray) == cov - assert common.grid_cov(grids, {k: str(v) for k, v in stray.items()}) == cov def test_workflow_without_cosmo_val_falls_back_to_fiducial(): diff --git a/workflow/common.py b/workflow/common.py index d4a7067e..8186f72e 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -334,17 +334,10 @@ def build_redshift_path(version, blind): # --------------------------------------------------------------------------- # ξ± angular grids # --------------------------------------------------------------------------- -# A grid is a binning plus how its covariance is estimated: (min_sep, max_sep, -# nbins, npatch, cov). `reporting` is the analysis grid, `integration` the fine -# one both B-mode statistics (COSEBIs, pure-E/B) integrate over. cov is -# "jackknife" (dense, from the patches), "diagonal" (TreeCorr varxip/varxim) or -# "none". -XI_KEYS = ( - "min_sep", - "max_sep", - "nbins", - "npatch", -) # the binning; cov is not in the name +# A grid is a binning: (min_sep, max_sep, nbins, npatch). `reporting` is the +# analysis grid, `integration` the fine one both B-mode statistics (COSEBIs, +# pure-E/B) integrate over. +XI_KEYS = ("min_sep", "max_sep", "nbins", "npatch") def xi_grids(config, fiducial): @@ -384,15 +377,15 @@ def xi_grids(config, fiducial): grid[key] = float(grid[key]) for key in ("nbins", "npatch"): grid[key] = int(grid[key]) - grid.setdefault("cov", patch_cov(grid["npatch"])) return grids def patch_cov(npatch): - """The covariance a measurement with ``npatch`` patches can carry. + """The covariance a ξ± part measured with ``npatch`` patches carries. - A jackknife needs patches; at npatch=1 TreeCorr's var_method is "shot" and - its diagonal is the estimate there is. + "jackknife" (dense, from the patches) needs patches; at npatch=1 TreeCorr's + var_method is "shot" and its "diagonal" (varxip/varxim) is the estimate + there is. """ return "jackknife" if int(npatch) > 1 else "diagonal" @@ -405,15 +398,6 @@ def grid_binning(grid): ) -def _named_grid(grids, binning): - """Name of the grid a binning is, compared numerically, or ``None``.""" - key = tuple(float(binning[k]) for k in XI_KEYS) - for name, grid in grids.items(): - if tuple(float(grid[k]) for k in XI_KEYS) == key: - return name - return None - - def grid_of(grids, binning): """Name of the grid a binning belongs to, compared numerically. @@ -421,13 +405,11 @@ def grid_of(grids, binning): grid (e.g. papers/bmodes' nbins=10000 convergence check) are reporting-style measurements. """ - return _named_grid(grids, binning) or "reporting" - - -def grid_cov(grids, binning): - """Covariance mode for a binning: its grid's, else what its patches allow.""" - name = _named_grid(grids, binning) - return grids[name]["cov"] if name else patch_cov(binning["npatch"]) + key = tuple(float(binning[k]) for k in XI_KEYS) + for name, grid in grids.items(): + if tuple(float(grid[k]) for k in XI_KEYS) == key: + return name + return "reporting" def pseudo_cl_tag(config): @@ -510,18 +492,16 @@ def cv_basename(version, fiducial=None): ) -def cv_init_params(config, version_list=None): +def cv_init_params(config): """Assemble the CosmologyValidation(...) constructor kwargs from config. Centralizes the run-specific instantiation so every cosmo_val rule script builds an identical `cv`: catalogues from CAT_CONFIG and products under - COSMO_VAL, never the constructor's cwd-relative defaults. `version_list` - overrides config["versions"] (used by per-version rules that pass a single - version). + COSMO_VAL, never the constructor's cwd-relative defaults. """ cv = config["cosmo_val"] return dict( - versions=version_list if version_list is not None else config["versions"], + versions=config["versions"], catalog_config=CAT_CONFIG, output_dir=str(COSMO_VAL), **{key: cv[key] for key in CV_INIT_KEYS}, diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index ce04400d..34735a85 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -204,9 +204,8 @@ def cv_analysis_sacc(version): return str(COSMO_VAL / f"{version}.sacc") -# Common params block shared by every cosmo_val rule: the cv constructor kwargs. -def cv_params(version_list=None): - return dict(cv_init=cv_init_params(config, version_list=version_list)) +# The CosmologyValidation constructor kwargs every cv_runner rule passes. +CV_INIT = cv_init_params(config) # --------------------------------------------------------------------------- @@ -227,7 +226,7 @@ rule cv_plot_rho_stats: output: sentinel=str(CV_SENTINELS / "plot_rho_stats.done"), params: - **cv_params(), + cv_init=CV_INIT, resources: runtime=20, script: @@ -241,7 +240,7 @@ rule cv_plot_tau_stats: output: sentinel=str(CV_SENTINELS / "plot_tau_stats.done"), params: - **cv_params(), + cv_init=CV_INIT, resources: runtime=20, script: @@ -256,7 +255,7 @@ rule cv_rho_tau_fits: output: sentinel=str(CV_SENTINELS / "rho_tau_fits.done"), params: - **cv_params(), + cv_init=CV_INIT, resources: mem_mb=16000, runtime=120, @@ -273,7 +272,7 @@ rule cv_footprints: output: sentinel=str(CV_SENTINELS / "footprints.done"), params: - **cv_params(), + cv_init=CV_INIT, resources: mem_mb=16000, runtime=60, @@ -286,7 +285,7 @@ rule cv_objectwise_leakage: output: sentinel=str(CV_SENTINELS / "objectwise_leakage.done"), params: - **cv_params(), + cv_init=CV_INIT, threads: 12 resources: mem_mb=30000, @@ -300,7 +299,7 @@ rule cv_weights: output: weight_hist=str(COSMO_VAL / "weight_hist.png"), params: - **cv_params(), + cv_init=CV_INIT, resources: mem_mb=16000, runtime=30, @@ -322,7 +321,7 @@ rule cv_additive_bias: output: additive_bias=str(COSMO_VAL / "additive_bias.json"), params: - **cv_params(), + cv_init=CV_INIT, resources: mem_mb=16000, runtime=30, @@ -337,7 +336,7 @@ rule cv_plot_2pcf: output: sentinel=str(CV_SENTINELS / "plot_2pcf.done"), params: - **cv_params(), + cv_init=CV_INIT, resources: runtime=20, script: @@ -354,7 +353,7 @@ rule cv_ratio_xi_sys_xi: ratio=str(COSMO_VAL / "ratio_xi_sys_xi.png"), params: offset=0.1, - **cv_params(), + cv_init=CV_INIT, resources: mem_mb=16000, runtime=120, diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 74fe0569..769e2b87 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -22,7 +22,8 @@ rule xi: """TreeCorr ξ±(θ) for one version on one angular grid. One rule for every grid: outputs are named by their binning, so a request - binds the wildcards and the grid label and covariance resolve from them. + binds the wildcards, the grid label resolves from them, and the covariance + follows the patches. """ input: catalog=get_shear_catalog, @@ -39,7 +40,7 @@ rule xi: cat_config=CAT_CONFIG, output_dir=str(COSMO_VAL), grid=lambda w: grid_of(XI_GRIDS, xi_binning_of(w)), - cov=lambda w: grid_cov(XI_GRIDS, xi_binning_of(w)), + cov=lambda w: patch_cov(w.npatch), resources: # The fine integration grid needs more memory and wall time than the # ~20-bin reporting one; scale on nbins rather than splitting the rule. diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index ed1ee69e..f3bd1f1b 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -16,9 +16,9 @@ grids are the same compute with different ``--min-sep/--max-sep/--nbins``. ``CosmologyValidation.calculate_2pcf`` writes the ``.txt`` dump (a raw byproduct); the ξ± data product is born as SACC here, a *part* named by its -binning and tagged with its ``--grid``. The part carries the covariance its -grid configures (``--cov``): the dense jackknife estimate from the patches, the -TreeCorr ``varxip``/``varxim`` diagonal, or none. +binning and tagged with its ``--grid``. The part carries the covariance +``--cov`` names: the dense jackknife estimate from the patches, the TreeCorr +``varxip``/``varxim`` diagonal, or none. ``output_dir`` is passed explicitly so lc can point each run at its own ``{output}`` tree. From ffc4828f55fc7457eb1c2b849b28ea28bbc0be92 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 25 Sep 2026 23:20:38 +0200 Subject: [PATCH 073/160] workflow/README: scope the output-root sentence to papers/cosmo_val Name where the other rules write: masks under the run directory's output/masks/, papers/bmodes' figures and macros under its docs/, image sims under grids_base. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- workflow/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index 187cb343..9d11139a 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -94,9 +94,12 @@ rule executes new script code against an old `import sp_validation` — the two halves of one commit, split. The catalogue config is the launched checkout's too: `cosmo_val/cat_config.yaml`, -read by the host and handed to every job. Every declared output lies under -`COSMO_VAL` or `COSMO_INFERENCE` (environment variables, defaulting to the -shared trees on candide) or the run directory's `results/`. +read by the host and handed to every job. The `papers/cosmo_val` suite writes +only under `COSMO_VAL` or `COSMO_INFERENCE` (environment variables, defaulting +to the shared trees on candide) or the run directory's `results/`. Other rules +write elsewhere: masks under the run directory's `output/masks/`, +`papers/bmodes`' figures and macros under its run directory's `docs/`, the +image sims under their `grids_base`. **Caveat:** `rerun-triggers: code` watches rule bodies and `script:` files, not `src/`. Editing a module under `src/` does not by itself mark outputs stale — From 8ab40990702026704db13a56fb1593a7e3a79f99 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 00:29:14 +0200 Subject: [PATCH 074/160] workflow: SLURM jobs start whatever the launch's XDG_CACHE_HOME; P4/P5 test what a launch does A job's Snakemake inherits the launching shell's environment (--export=ALL), and a login shell may point XDG_CACHE_HOME at node-local storage: every job then fails creating its source cache. Two halves close it, each needed (shown on n33 by replaying the captured sbatch job command with srun stubbed): - the candide profile leaves source-cache out of shared-fs-usage, so a job neither reuses the launch's cache path nor creates its own under XDG; - common.py drops XDG_CACHE_HOME from what jobs inherit, because the slurm-jobstep executor forces a shared source cache on the Snakemake it starts for each job step. test_a_job_needs_no_launch_cache covers both (each mutation turns it red). P4 (test_papers_resolve_on_candide) now passes from a shell with node-local XDG_CACHE_HOME under srun, and its bmodes case also resolves an e-cut catalogue, so both CATALOG_CONFIG readers in ecut.smk are exercised. P5 hands the smoke Snakefile the image a launch resolves (no registry pull). The host launcher and CI carry snakemake-executor-plugin-slurm, as the README install line does, so the candide profile is parsed in CI. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- .github/workflows/deploy-image.yml | 3 +- workflow/README.md | 11 +++-- workflow/common.py | 6 +++ workflow/profiles/candide/config.yaml | 12 ++++++ workflow/tests/conftest.py | 12 ++++-- workflow/tests/test_container_smoke.py | 13 ++++-- workflow/tests/test_dag.py | 58 ++++++++++++++++++++++---- 7 files changed, 97 insertions(+), 18 deletions(-) diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 3ad8e9f7..5867b9c2 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -25,7 +25,8 @@ jobs: - name: DAG tests run: >- uv run --isolated --no-project --python 3.12 - --with snakemake==9.23.1 --with pytest --with numpy + --with snakemake==9.23.1 --with snakemake-executor-plugin-slurm + --with pytest --with numpy pytest workflow/tests -m "not candide" build-and-push-image: diff --git a/workflow/README.md b/workflow/README.md index 9d11139a..e5671848 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -61,9 +61,14 @@ override, e.g. the image-sims `SIF`). A few rules shell out to a host toolchain (CosmoCov, ImageMagick) and keep `container: None`; each says why in its own docstring. -`OMP_NUM_THREADS` is not set by the profile either: the slurm executor's -`--export=ALL` propagates the driver's env, not a profile flag, so a rule that -needs it pinned sets it itself. Per-rule `mem_mb` / `runtime` stay on the rules. +The slurm executor submits with `--export=ALL`, so every job starts with the +launching shell's environment. `OMP_NUM_THREADS` is therefore not a profile +setting: a rule that needs it pinned sets it itself. A path in that +environment reaches nodes where it may not exist. For Snakemake's own cache +this is handled (the launch drops `XDG_CACHE_HOME`, and the profile keeps the +source cache off the shared filesystem), so a login shell that points it at +`/scratch` is fine; keep any other path you export on a shared disk. Per-rule +`mem_mb` / `runtime` stay on the rules. ### Off candide — the default profile diff --git a/workflow/common.py b/workflow/common.py index 8186f72e..b12fe999 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -33,6 +33,12 @@ image_runtime = _container.image_runtime resolve_image = _container.resolve_image +# Every job inherits this launch's environment (the slurm executor submits with +# --export=ALL), and the Snakemake each job step starts keeps its source cache +# under XDG_CACHE_HOME, which a login shell may point at node-local storage. +# Without it, jobs use the home directory's cache, which every node mounts. +os.environ.pop("XDG_CACHE_HOME", None) + # Output roots are env-overridable so a reproduction run can write into a # fresh tree without clobbering (or silently reusing) prior products. diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 22b549b1..01230785 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -66,6 +66,18 @@ default-resources: # for any remote executor and refuses a real run without one. jobs: 100 +# Snakemake's source cache lives under the launching shell's XDG_CACHE_HOME, +# which may be node-local. Leaving source-cache out of the shared-filesystem +# usages keeps each job from reusing the launch's cache and puts the job's own +# in its tmp dir. +shared-fs-usage: + - persistence + - input-output + - software-deployment + - sources + - storage-local-copies + - software-deployment-cache + # Retry a job once on transient node failure, and keep the SLURM logs of # successful jobs (candide debugging). retries: 1 diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py index cf3733a9..29ebf5c4 100644 --- a/workflow/tests/conftest.py +++ b/workflow/tests/conftest.py @@ -3,12 +3,12 @@ These tests run under the host launcher, never inside the image:: uv run --isolated --no-project --python 3.12 --with snakemake==9.23.1 \\ - --with pytest --with numpy pytest workflow/tests + --with snakemake-executor-plugin-slurm --with pytest --with numpy \\ + pytest workflow/tests ``sp_validation`` is absent from that environment, so every Snakefile has to parse with the standard library and Snakemake alone -- the condition a host -Snakemake is in. The ``candide`` tests drive the candide profile and so also -need ``--with snakemake-executor-plugin-slurm``; CI deselects them. +Snakemake is in. The ``candide`` tests need candide itself; CI deselects them. The ``toy`` fixture is a disposable checkout: copies of ``workflow/`` and ``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, a one-catalogue @@ -103,6 +103,12 @@ def _load_module(path, name, env): return module +# The package's image model, loaded by path: sp_validation is absent here. +container = _load_module( + REPO / "src" / "sp_validation" / "container.py", "spv_container", {} +) + + @dataclasses.dataclass class Toy: root: Path diff --git a/workflow/tests/test_container_smoke.py b/workflow/tests/test_container_smoke.py index 8e4a350b..d6312b83 100644 --- a/workflow/tests/test_container_smoke.py +++ b/workflow/tests/test_container_smoke.py @@ -2,9 +2,9 @@ The executor, the apptainer deployment method, the bind mounts and the job bound come from that profile, launched as the README launches a target; the -image is the module-level ``container:`` in the test Snakefile, exactly as real -workflows declare it. That contract is what's under test, so it runs only on a -candide submit host. +image is the one that launch resolves (your SIF or sandbox), handed to the test +Snakefile's module-level ``container:`` as real workflows hand theirs. That +contract is what's under test, so it runs only on a candide submit host. The job writes a YAML report (see data/container_smoke/container_smoke.py); the assertions below check what it reports. @@ -21,7 +21,7 @@ import numpy as np import pytest import yaml -from conftest import REPO, on_candide +from conftest import REPO, container, on_candide SMOKE = Path(__file__).resolve().parent / "data" / "container_smoke" @@ -47,6 +47,9 @@ def test_smoke_snakefile_names_the_workflow_image(): @on_candide @pytest.mark.skipif(shutil.which("sbatch") is None, reason="needs a SLURM submit host") def test_container_smoke(): + image, kind = container.resolve_image() + assert kind != "tag", "no local image; run `spv-container pull`" + # Not pytest's tmp_path: that lives in the submit host's /tmp, which the # compute node cannot see, so the job's output would "go missing". The # workdir must be on a shared filesystem. @@ -65,6 +68,8 @@ def test_container_smoke(): "--directory", str(workdir), "container_smoke", + "--config", + f"container={image}", ], env=env, text=True, diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 37eda603..d84906c4 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -131,6 +131,45 @@ def test_launch_reads_the_image_under_home(toy, tmp_path): assert "uv tool install --force --python 3.13 snakemake==" in result.stdout +def test_a_job_needs_no_launch_cache(toy, tmp_path): + """A job starts on a node where the launch's XDG cache cannot exist. + + Jobs inherit the launching shell's environment, whose XDG_CACHE_HOME may be + node-local. Under the candide profile, a Snakemake that cannot create that + cache still starts, and a job's environment carries no XDG_CACHE_HOME for + the Snakemake its job step starts. + """ + blocker = tmp_path / "a-file" + blocker.touch() + candide = toy.root / "workflow" / "profiles" / "candide" + result = toy.snakemake( + "-n", + "--profile", + str(candide), + "assemble_sacc_all", + env=toy.env | {"XDG_CACHE_HOME": str(blocker / "cache")}, + ) + assert result.returncode == 0, result.stdout + + snakefile = tmp_path / "Snakefile" + snakefile.write_text( + f"import sys\nsys.path.insert(0, {str(toy.root / 'workflow')!r})\n" + "import common\n\n" + 'rule job:\n output: "env.txt"\n' + ' shell: "printenv XDG_CACHE_HOME > {output} || true"\n' + ) + result = toy.snakemake( + "-s", + str(snakefile), + "--directory", + str(tmp_path), + container=False, + env=toy.env | {"XDG_CACHE_HOME": str(tmp_path / "launch-cache")}, + ) + assert result.returncode == 0, result.stdout + assert (tmp_path / "env.txt").read_text() == "" + + def test_image_sims_checks_parity_at_launch(toy, tmp_path): """The standalone image-sims workflow stops on a mismatched image too.""" run = { @@ -159,7 +198,7 @@ def test_image_sims_checks_parity_at_launch(toy, tmp_path): assert "uv tool install --force --python 3.13 snakemake==" in result.stdout -def _real_dry_run(paper, target): +def _real_dry_run(paper, targets): env = {k: v for k, v in os.environ.items() if k != "SNAKEMAKE_PROFILE"} env.update(PYTHONUNBUFFERED="1", PYTHONNOUSERSITE="1") return subprocess.run( @@ -170,7 +209,7 @@ def _real_dry_run(paper, target): "-n", "--profile", str(REPO / "workflow" / "profiles" / "candide"), - target, + *targets, ], cwd=REPO / "papers" / paper, env=env, @@ -185,15 +224,20 @@ def _real_dry_run(paper, target): @pytest.mark.candide @on_candide @pytest.mark.parametrize( - "paper, target", [("cosmo_val", "assemble_sacc_all"), ("bmodes", "all_tapestry")] + "paper, targets", + [ + ("cosmo_val", ["assemble_sacc_all"]), + ("bmodes", ["all_tapestry", "results/ecut/SP_v1.4.6_ecut07.fits"]), + ], + ids=["cosmo_val", "bmodes"], ) -def test_papers_resolve_on_candide(paper, target): +def test_papers_resolve_on_candide(paper, targets): """The real paper DAGs resolve against the real catalogues and your image. - Your image (the SIF or sandbox `spv-container` manages) is read, so parity - was checked. + The e-cut catalogue reads its parent's catalogue entry. Your image (the SIF + or sandbox `spv-container` manages) is read, so parity was checked. """ - result = _real_dry_run(paper, target) + result = _real_dry_run(paper, targets) assert result.returncode == 0, result.stdout assert "parity unchecked" not in result.stdout, ( "no local image was read; run `spv-container pull`\n" + result.stdout From f3af91cb24ce85cf136fef440ace7d59944d45d4 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 00:29:38 +0200 Subject: [PATCH 075/160] papers/bmodes: the sweep drivers read the image from ~/.cache, as container.py does container_env.sh still followed XDG_CACHE_HOME, so from a login shell that points it at node-local storage the sweep drivers ran a missing image while spv-container and the workflow found the real one. test_sweep_drivers_run_the_resolved_image pins the shell copy of the resolution (cache and sandbox precedence) to container.resolve_image. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- papers/bmodes/scripts/container_env.sh | 2 +- workflow/tests/test_image_resolution.py | 36 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 workflow/tests/test_image_resolution.py diff --git a/papers/bmodes/scripts/container_env.sh b/papers/bmodes/scripts/container_env.sh index a63b40f5..cd4c00fc 100644 --- a/papers/bmodes/scripts/container_env.sh +++ b/papers/bmodes/scripts/container_env.sh @@ -11,7 +11,7 @@ PSCRIPTS=$WT/papers/bmodes/scripts # CONTAINER and BIND are resolved exactly as `sp_validation/container.py` does: # the writable sandbox if there is one, else the SIF. -_spv_cache=${XDG_CACHE_HOME:-$HOME/.cache}/sp_validation +_spv_cache=$HOME/.cache/sp_validation _spv_sandbox=${SPV_SANDBOX:-$_spv_cache/sandbox} if [ -d "$_spv_sandbox" ]; then CONTAINER=$_spv_sandbox diff --git a/workflow/tests/test_image_resolution.py b/workflow/tests/test_image_resolution.py new file mode 100644 index 00000000..66687468 --- /dev/null +++ b/workflow/tests/test_image_resolution.py @@ -0,0 +1,36 @@ +"""Every launcher runs the image ``sp_validation/container.py`` resolves.""" + +import os +import subprocess + +import pytest +from conftest import REPO, container + +SWEEP_ENV = REPO / "papers" / "bmodes" / "scripts" / "container_env.sh" + + +@pytest.mark.parametrize("sandbox", [False, True], ids=["sif", "sandbox"]) +def test_sweep_drivers_run_the_resolved_image(tmp_path, monkeypatch, sandbox): + """The Paper II sweep drivers' CONTAINER is the image container.py resolves. + + Their shell copy of the resolution reads the same cache, whatever + XDG_CACHE_HOME says, and prefers the sandbox the same way. + """ + cache = tmp_path / "home" / ".cache" / "sp_validation" + cache.mkdir(parents=True) + (cache / "sp_validation.sif").touch() + if sandbox: + (cache / "sandbox").mkdir() + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "node-local")) + for var in ("SPV_CONTAINER", "SPV_SANDBOX"): + monkeypatch.delenv(var, raising=False) + + shell = subprocess.run( + ["bash", "-c", f'. "{SWEEP_ENV}" && printf %s "$CONTAINER"'], + env=dict(os.environ), + capture_output=True, + text=True, + check=True, + ) + assert shell.stdout == container.resolve_image()[0] From f8c8cc5d1ccd014729dfacdb27fcda517540558f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 00:33:04 +0200 Subject: [PATCH 076/160] =?UTF-8?q?run=5F2pcf:=20a=20=CE=BE=C2=B1=20part?= =?UTF-8?q?=20carries=20the=20covariance=20its=20measurement=20estimated?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Which covariance a part carries had two homes that disagreed: rule xi passed cov=patch_cov(npatch) ("diagonal" at npatch=1), while run_2pcf's own default, reached by the CLI and papers/bmodes' run_xi_sweep, was "none". The same file name then held variances or not depending on who wrote it. run_2pcf now carries what TreeCorr estimated (gg.var_method, which calculate_2pcf sets from npatch): the jackknife covariance with patches, the shot-noise diagonal without. The cov argument, --cov, the jackknife guard, rule xi's cov param and common.patch_cov (with its test) go. Rule xi hands its wildcards to grid_of directly (xi_binning_of goes). test_xi_part_carries_the_covariance_the_measurement_estimated runs run_2pcf on the synthetic catalogue at npatch 1 and 4; the previous run_2pcf fails both cases. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/tests/test_cosmo_val.py | 44 +++++++++++++++++++++++ src/sp_validation/tests/test_xi_grids.py | 13 ------- workflow/common.py | 10 ------ workflow/rules/twopoint.smk | 11 ++---- workflow/scripts/run_2pcf.py | 25 ++++--------- 5 files changed, 52 insertions(+), 51 deletions(-) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 7a7c0780..5e248797 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -354,6 +354,7 @@ def _write_synthetic_catalogs( shear_cfg = { "path": "shear.fits", + "redshift_path": str(nz_dir / "dndz_SP_A.txt"), "w_col": "w", "e1_col": "e1", "e2_col": "e2", @@ -438,6 +439,49 @@ def test_calculate_2pcf_runs_on_synthetic_catalog(self, tmp_path): # The additive-bias subtraction in the pipeline must have run. assert version in cv.c1 and version in cv.c2 + @pytest.mark.parametrize("npatch", [1, 4]) + def test_xi_part_carries_the_covariance_the_measurement_estimated( + self, tmp_path, npatch + ): + """run_2pcf's ξ± part carries TreeCorr's covariance, whoever calls it. + + The jackknife covariance with patches, the shot-noise diagonal without, + so every part has variances whether the rule or the CLI measured it. + """ + import importlib.util + + import sacc + + from sp_validation import sacc_io + + script = Path(__file__).resolve().parents[3] / "workflow/scripts/run_2pcf.py" + spec = importlib.util.spec_from_file_location("run_2pcf_part", script) + run_2pcf = importlib.util.module_from_spec(spec) + spec.loader.exec_module(run_2pcf) + + params, version = self._write_synthetic_catalogs(tmp_path) + part = tmp_path / "part.sacc" + gg = run_2pcf.run_2pcf( + ver=version, + min_sep=5.0, + max_sep=100.0, + nbins=6, + npatch=npatch, + cat_config=params["catalog_config"], + output_dir=params["output_dir"], + sacc_out=str(part), + ) + + cov = sacc_io.load(str(part), allow_unblinded=True).covariance + if npatch > 1: + assert isinstance(cov, sacc.covariance.FullCovariance) + np.testing.assert_array_equal(cov.dense, gg.cov) + else: + assert isinstance(cov, sacc.covariance.DiagonalCovariance) + np.testing.assert_array_equal( + cov.diag, np.concatenate([gg.varxip, gg.varxim]) + ) + def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): """calculate_2pcf's ξ± is the same on 4 and on 48 TreeCorr threads. diff --git a/src/sp_validation/tests/test_xi_grids.py b/src/sp_validation/tests/test_xi_grids.py index 974fd4eb..e196e9e4 100644 --- a/src/sp_validation/tests/test_xi_grids.py +++ b/src/sp_validation/tests/test_xi_grids.py @@ -82,19 +82,6 @@ def test_one_integration_grid(): assert set(common.xi_grids(CONFIG, FIDUCIAL)) == {"reporting", "integration"} -@pytest.mark.parametrize( - "npatch, cov", [(1, "diagonal"), ("1", "diagonal"), ("100", "jackknife")] -) -def test_covariance_mode_follows_the_patches(npatch, cov): - """A patched part gets a jackknife block, an unpatched one the diagonal. - - Every part then carries variances: at npatch=1 TreeCorr's var_method is - "shot", and its diagonal is the estimate it has. The rule passes the - wildcard, a string. - """ - assert common.patch_cov(npatch) == cov - - def test_unnamed_binning_is_a_reporting_measurement(): """The paper's convergence-check binning belongs to no named grid.""" grids = common.xi_grids(CONFIG, FIDUCIAL) diff --git a/workflow/common.py b/workflow/common.py index b12fe999..474cdf39 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -386,16 +386,6 @@ def xi_grids(config, fiducial): return grids -def patch_cov(npatch): - """The covariance a ξ± part measured with ``npatch`` patches carries. - - "jackknife" (dense, from the patches) needs patches; at npatch=1 TreeCorr's - var_method is "shot" and its "diagonal" (varxip/varxim) is the estimate - there is. - """ - return "jackknife" if int(npatch) > 1 else "diagonal" - - def grid_binning(grid): """The `minsep=..._maxsep=..._nbins=..._npatch=...` tag of one grid.""" return ( diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 769e2b87..eeb11038 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -13,17 +13,11 @@ def xi_binning(grid): return grid_binning(XI_GRIDS[grid]) -def xi_binning_of(wildcards): - """The binning a job was requested with, from its wildcards.""" - return {key: getattr(wildcards, key) for key in XI_KEYS} - - rule xi: """TreeCorr ξ±(θ) for one version on one angular grid. One rule for every grid: outputs are named by their binning, so a request - binds the wildcards, the grid label resolves from them, and the covariance - follows the patches. + binds the wildcards and the grid label resolves from them. """ input: catalog=get_shear_catalog, @@ -39,8 +33,7 @@ rule xi: npatch="{npatch}", cat_config=CAT_CONFIG, output_dir=str(COSMO_VAL), - grid=lambda w: grid_of(XI_GRIDS, xi_binning_of(w)), - cov=lambda w: patch_cov(w.npatch), + grid=lambda w: grid_of(XI_GRIDS, w), resources: # The fine integration grid needs more memory and wall time than the # ~20-bin reporting one; scale on nbins rather than splitting the rule. diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index f3bd1f1b..28c8c521 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -16,9 +16,9 @@ grids are the same compute with different ``--min-sep/--max-sep/--nbins``. ``CosmologyValidation.calculate_2pcf`` writes the ``.txt`` dump (a raw byproduct); the ξ± data product is born as SACC here, a *part* named by its -binning and tagged with its ``--grid``. The part carries the covariance -``--cov`` names: the dense jackknife estimate from the patches, the TreeCorr -``varxip``/``varxim`` diagonal, or none. +binning and tagged with its ``--grid``. The part carries the covariance the +measurement estimated: the dense jackknife covariance when it had patches, the +shot-noise ``varxip``/``varxim`` diagonal when it had none. ``output_dir`` is passed explicitly so lc can point each run at its own ``{output}`` tree. @@ -44,7 +44,6 @@ def run_2pcf( output_dir, sacc_out=None, grid="reporting", - cov="none", ): """Measure ξ±(θ) for ``ver`` and write its reporting SACC part. @@ -77,10 +76,8 @@ def run_2pcf( nbins=nbins, ) - if cov == "jackknife" and int(npatch) < 2: - raise ValueError(f"cov='jackknife' needs patches; got npatch={npatch}") - # Born-as-SACC ξ± part. theta = meanr; theta_nom = rnom. + jackknife = gg.var_method == "jackknife" s = xi_to_sacc( cv.sacc_nz(ver), cv.sacc_metadata(ver), @@ -91,10 +88,8 @@ def run_2pcf( theta_nom=gg.rnom, npairs=gg.npairs, weight=gg.weight, - covariance=gg.cov if cov == "jackknife" else None, - variances=( - np.concatenate([gg.varxip, gg.varxim]) if cov == "diagonal" else None - ), + covariance=gg.cov if jackknife else None, + variances=None if jackknife else np.concatenate([gg.varxip, gg.varxim]), ) out_path = sacc_out or os.path.join( output_dir or cv.cc["paths"]["output"], @@ -116,7 +111,6 @@ def _from_snakemake(smk): cat_config=p["cat_config"], output_dir=p["output_dir"], grid=p.get("grid", "reporting"), - cov=p.get("cov", "none"), # The SACC part goes exactly where the rule declares it; the .txt # byproduct still lands under the resolved output dir. sacc_out=smk.output["sacc"], @@ -149,12 +143,6 @@ def _from_cli(argv=None): ap.add_argument( "--grid", default="reporting", help="SACC grid tag for the measured points" ) - ap.add_argument( - "--cov", - default="none", - choices=["jackknife", "diagonal", "none"], - help="Covariance the part carries", - ) a = ap.parse_args(argv) run_2pcf( ver=a.ver, @@ -165,7 +153,6 @@ def _from_cli(argv=None): cat_config=a.cat_config, output_dir=a.out, grid=a.grid, - cov=a.cov, ) From 8052da90dd51fc611282b6309080f653f2ebbcde Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 01:07:23 +0200 Subject: [PATCH 077/160] workflow/tests: the launch-cache test answers the profile's apptainer check with a stub The candide profile deploys with apptainer, and Snakemake asks for the apptainer binary and its version even in a dry-run, so test_a_job_needs_no_launch_cache failed on GitHub's runners, which have no apptainer. A stub apptainer on the test's PATH answers that check; the dry-run reads nothing else from it. In a CI emulation (a clean checkout, a fresh HOME, no apptainer or SLURM on PATH, the workflow-dag command) the suite goes from 1 failed, 11 passed to 12 passed. Both mutations still turn the test red there: source-cache added back to the profile's shared-fs-usage (NotADirectoryError under the blocked XDG_CACHE_HOME) and the XDG_CACHE_HOME pop removed from common.py (the job sees the launch's cache). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- workflow/tests/test_dag.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index d84906c4..ec3f7559 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -141,13 +141,23 @@ def test_a_job_needs_no_launch_cache(toy, tmp_path): """ blocker = tmp_path / "a-file" blocker.touch() + # The profile deploys with apptainer, whose version Snakemake reads even in + # a dry-run; this stub answers where apptainer is not installed. + apptainer = tmp_path / "bin" / "apptainer" + apptainer.parent.mkdir() + apptainer.write_text("#!/bin/sh\necho apptainer version 1.3.4\n") + apptainer.chmod(0o755) candide = toy.root / "workflow" / "profiles" / "candide" result = toy.snakemake( "-n", "--profile", str(candide), "assemble_sacc_all", - env=toy.env | {"XDG_CACHE_HOME": str(blocker / "cache")}, + env=toy.env + | { + "XDG_CACHE_HOME": str(blocker / "cache"), + "PATH": f"{apptainer.parent}{os.pathsep}{toy.env['PATH']}", + }, ) assert result.returncode == 0, result.stdout From 8ace6e82d7818b695b533201f513d0fbe42ed95d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 01:07:23 +0200 Subject: [PATCH 078/160] workflow/tests: the smoke Snakefile composes workflow/ as the entry Snakefiles do P5's Snakefile never imported common, so its launch did not drop XDG_CACHE_HOME: from a login shell pointing it at node-local /scratch, the job's inner Snakemake died creating its source cache (PermissionError), which says nothing about what a real launch does. The Snakefile now imports common, resolves its image with common.resolve_container and checks host/image parity, as the entry Snakefiles do. P5 launches exactly as the README does, with no --config container=, and the literal default image and the test that kept it in step with CONTAINER_URI go. Checked on n33 with sbatch stubbed to record what it would submit, then the recorded job replayed with srun stubbed, from a shell with XDG_CACHE_HOME set to /scratch/cdaley/tmp/xdg: at the parent commit the job environment carries XDG_CACHE_HOME and the replay fails with PermissionError; with this commit it carries none, the job runs in the SIF, and P5's assertions pass on its report. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- workflow/tests/data/container_smoke/Snakefile | 20 ++++++++++------ workflow/tests/test_container_smoke.py | 24 ++++++------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/workflow/tests/data/container_smoke/Snakefile b/workflow/tests/data/container_smoke/Snakefile index 0d99f26a..3dce83fd 100644 --- a/workflow/tests/data/container_smoke/Snakefile +++ b/workflow/tests/data/container_smoke/Snakefile @@ -1,15 +1,21 @@ # Standalone workflow exercised by workflow/tests/test_container_smoke.py. # -# The module-level `container:` below mirrors what every real workflow does -# (workflow/Snakefile) -- Snakemake has no way to take a default image from a -# profile. Everything else under test arrives from the driving profile +# It composes workflow/ as every entry Snakefile does: common's launch-time +# setup, the image a launch resolves, the host/image parity check. The executor, +# the apptainer deployment and the binds come from the driving profile # (workflow/profiles/candide). See container_smoke.py for what the job checks. +import os +import sys -# Literal rather than the package's CONTAINER_URI: this Snakefile is test data, -# deliberately standalone. test_container_smoke.py asserts the two agree, so -# drift fails the test rather than the run. -container: config.get("container", "docker://ghcr.io/cosmostat/sp_validation:develop") +sys.path.insert(0, os.path.realpath(os.path.join(str(workflow.basedir), "../../.."))) +import common + +IMAGE = common.resolve_container(config.get("container")) +common.check_host_parity(IMAGE) + + +container: IMAGE rule container_smoke: diff --git a/workflow/tests/test_container_smoke.py b/workflow/tests/test_container_smoke.py index d6312b83..1482c0dc 100644 --- a/workflow/tests/test_container_smoke.py +++ b/workflow/tests/test_container_smoke.py @@ -2,9 +2,10 @@ The executor, the apptainer deployment method, the bind mounts and the job bound come from that profile, launched as the README launches a target; the -image is the one that launch resolves (your SIF or sandbox), handed to the test -Snakefile's module-level ``container:`` as real workflows hand theirs. That -contract is what's under test, so it runs only on a candide submit host. +test Snakefile composes workflow/ as the entry Snakefiles do, so the job runs +the image that launch resolves (your SIF or sandbox) in the environment that +launch hands its jobs. That contract is what's under test, so it runs only on a +candide submit host. The job writes a YAML report (see data/container_smoke/container_smoke.py); the assertions below check what it reports. @@ -33,22 +34,13 @@ def _reference_eigenvalues() -> np.ndarray: return np.linalg.eigh(a + a.T)[0] -def test_smoke_snakefile_names_the_workflow_image(): - """The test Snakefile's literal image must track the package's CONTAINER_URI.""" - uri = re.search( - r'^CONTAINER_URI = "(.+)"$', - (REPO / "src/sp_validation/container.py").read_text(), - re.MULTILINE, - ).group(1) - assert f'"{uri}"' in (SMOKE / "Snakefile").read_text(), uri - - @pytest.mark.candide @on_candide @pytest.mark.skipif(shutil.which("sbatch") is None, reason="needs a SLURM submit host") def test_container_smoke(): - image, kind = container.resolve_image() - assert kind != "tag", "no local image; run `spv-container pull`" + assert container.resolve_image()[1] != "tag", ( + "no local image; run `spv-container pull`" + ) # Not pytest's tmp_path: that lives in the submit host's /tmp, which the # compute node cannot see, so the job's output would "go missing". The @@ -68,8 +60,6 @@ def test_container_smoke(): "--directory", str(workdir), "container_smoke", - "--config", - f"container={image}", ], env=env, text=True, From 7beec00126192090c28b1b8f9e818a6b28dbccd8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 01:59:46 +0200 Subject: [PATCH 079/160] Blinding: custody declared per catalogue, blind drawn once outside the DAG, parts sealed at birth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Custody (blinded | unblinded | mock) is declared once per base catalogue in cosmo_val/cat_config.yaml; an undeclared entry is blinded, and variants (_leak_corr, _seed, base:) share their base. sp_validation.custody reads the declaration and the blind registry (cosmo_val/blinds//) for the host Snakemake and container jobs alike; the launch prints one [custody] line per catalogue and stops, with the command to run, when a blinded catalogue has no blind. cosmo_val.type is refused. A blind is drawn once by `python -m sp_validation.blinding init`; the seed exists on disk only as Fernet ciphertext beside its key and public record. No rule reads, writes or draws a blind. share, reveal, audit and verify complete the custody cycle; revealing moves concealed files aside and re-measures, never subtracting a shift. sacc_io.save is the only writer: a birth is sealed (ξ± and Cℓ_EE concealed in memory before the stamp is minted, blocks found from the content), a derivation inherits its inputs' one stamp and refuses laundered rows, and an assembly must equal the declaration. load refuses unstamped files. The hidden point is drawn uniformly in (S8, Ωm). blind_part, blind_init, the _blinded naming, maybe_temp, allow_unblinded and the run type are gone; the CAMB oracle moves into the cross-check test. The pure-E/B Monte Carlo covariance is seeded, so a blinded and a true run carry the same covariance. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- cosmo_val/blinds/CONTRACTS | 8 + cosmo_val/cat_config.yaml | 36 + papers/cosmo_val/config/config.yaml | 5 - scripts/blind_data_vector.py | 146 -- src/sp_validation/CONTRACTS | 20 + src/sp_validation/b_modes.py | 8 +- src/sp_validation/blinding.py | 1327 ++++++------ src/sp_validation/blinding_paths.py | 26 - src/sp_validation/blinding_theory.py | 210 +- src/sp_validation/cosmo_val/core.py | 34 +- src/sp_validation/cosmo_val/pseudo_cl.py | 39 +- .../cosmo_val/psf_systematics.py | 10 +- src/sp_validation/cosmo_val/sacc_writers.py | 2 +- src/sp_validation/custody.py | 324 +++ src/sp_validation/sacc_io.py | 223 +- src/sp_validation/tests/_synthetic.py | 143 ++ src/sp_validation/tests/test_architecture.py | 170 ++ src/sp_validation/tests/test_assemble_sacc.py | 34 +- src/sp_validation/tests/test_blinding.py | 1785 ++++++----------- .../tests/test_blinding_wiring.py | 389 ---- .../tests/test_camb_ccl_crosscheck.py | 248 +-- src/sp_validation/tests/test_cosmo_val.py | 148 +- src/sp_validation/tests/test_custody.py | 260 +++ src/sp_validation/tests/test_custody_e2e.py | 445 ++++ src/sp_validation/tests/test_pseudo_cl.py | 3 +- src/sp_validation/tests/test_sacc_io.py | 92 +- src/sp_validation/tests/test_sacc_writers.py | 10 +- workflow/CONTRACTS | 4 + workflow/README.md | 44 + workflow/Snakefile | 3 - workflow/common.py | 208 +- workflow/rules/blinding.smk | 61 - workflow/rules/cosmo_val.smk | 48 +- workflow/rules/twopoint.smk | 60 +- workflow/scripts/assemble_sacc.py | 43 +- workflow/scripts/cv_cosebis.py | 22 +- workflow/scripts/cv_pure_eb.py | 24 +- workflow/scripts/generate_pseudo_cl.py | 20 +- workflow/scripts/run_2pcf.py | 29 +- workflow/scripts/run_rho_tau.py | 49 +- workflow/tests/conftest.py | 55 +- workflow/tests/test_dag.py | 80 +- 42 files changed, 3335 insertions(+), 3560 deletions(-) create mode 100644 cosmo_val/blinds/CONTRACTS delete mode 100644 scripts/blind_data_vector.py create mode 100644 src/sp_validation/CONTRACTS delete mode 100644 src/sp_validation/blinding_paths.py create mode 100644 src/sp_validation/custody.py create mode 100644 src/sp_validation/tests/_synthetic.py create mode 100644 src/sp_validation/tests/test_architecture.py delete mode 100644 src/sp_validation/tests/test_blinding_wiring.py create mode 100644 src/sp_validation/tests/test_custody.py create mode 100644 src/sp_validation/tests/test_custody_e2e.py create mode 100644 workflow/CONTRACTS delete mode 100644 workflow/rules/blinding.smk diff --git a/cosmo_val/blinds/CONTRACTS b/cosmo_val/blinds/CONTRACTS new file mode 100644 index 00000000..f1055209 --- /dev/null +++ b/cosmo_val/blinds/CONTRACTS @@ -0,0 +1,8 @@ +@sc blind-drawn-once +The blind registry: one directory per blind, written only by +`python -m sp_validation.blinding`. `init` creates / with +commitment.json (the public record), seed.fernet (the seed, encrypted) and +key, all read-only, and `bases`; `share` appends a base catalogue to `bases`; +`reveal` writes revealed.json once. The record is committed to git. No +Snakemake rule reads, writes or draws anything here, so no run can re-draw a +blind, and every checkout resolves the same one. diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index 218ba413..a010e072 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -1,4 +1,9 @@ +# One entry per catalogue. `blinding:` declares a base catalogue's custody: +# blinded, unblinded or mock; an entry that declares none is blinded. A variant +# stored as its own entry names its parent with `base:` and shares the parent's +# custody and blind. See src/sp_validation/custody.py. DES: + blinding: unblinded subdir: /n17data/mkilbing/astro/data/DES pipeline: SP colour: orange @@ -40,6 +45,7 @@ DES: path: psf_y3a1-v29.fits patch_number: 120 SP_axel_v0.0: + blinding: unblinded subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 pipeline: SP colour: cyan @@ -80,6 +86,7 @@ SP_axel_v0.0: path: star_cat.fits patch_number: 120 SP_v0.1.1: + blinding: unblinded subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 pipeline: SP colour: cyan @@ -114,6 +121,7 @@ SP_v0.1.1: path: star_cat.fits patch_number: 150 SP_v1.3: + blinding: unblinded subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe pipeline: SP colour: green @@ -150,6 +158,7 @@ SP_v1.3: path: unions_shapepipe_star_2022_v1.0.3.fits patch_number: 150 SP_v1.3.6: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.3.x pipeline: SP colour: coral @@ -194,6 +203,7 @@ SP_v1.3.6: path: unions_shapepipe_star_2022_v1.0.3.fits patch_number: 150 SP_v1.4.5: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: dodgerblue @@ -238,6 +248,7 @@ SP_v1.4.5: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.5.A: + blinding: unblinded subdir: /n17data/guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5 pipeline: SP colour: red @@ -279,6 +290,7 @@ SP_v1.4.5.A: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.5_bright: + blinding: unblinded subdir: /n17data/murray/unions_cats pipeline: SP colour: mediumblue @@ -320,6 +332,7 @@ SP_v1.4.5_bright: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.5_faint: + blinding: unblinded subdir: /n17data/murray/unions_cats pipeline: SP colour: forestgreen @@ -361,6 +374,7 @@ SP_v1.4.5_faint: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6_glass_mock: + blinding: mock subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: darkgreen @@ -404,6 +418,7 @@ SP_v1.4.6_glass_mock: path: unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.5_intermediate: + blinding: unblinded subdir: /n17data/murray/unions_cats pipeline: SP colour: orange @@ -445,6 +460,7 @@ SP_v1.4.5_intermediate: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: green @@ -489,6 +505,7 @@ SP_v1.4.6: path: unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.3: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: orange @@ -533,6 +550,7 @@ SP_v1.4.6.3: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.3_B: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: green @@ -577,6 +595,7 @@ SP_v1.4.6.3_B: path: unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.3_C: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: green @@ -621,6 +640,7 @@ SP_v1.4.6.3_C: path: unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.3_A: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: green @@ -665,6 +685,7 @@ SP_v1.4.6.3_A: path: unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6_ecut07: + base: SP_v1.4.6 subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: darkgreen @@ -709,6 +730,7 @@ SP_v1.4.6_ecut07: path: unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.1: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: red @@ -753,6 +775,7 @@ SP_v1.4.6.1: path: unions_shapepipe_star_2024_v1.4.a.fits patch_number: 100 SP_v1.4.7: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: darkorchid @@ -799,6 +822,7 @@ SP_v1.4.7: path: unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.8: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: darkorchid @@ -843,6 +867,7 @@ SP_v1.4.8: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.11.2: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: orange @@ -887,6 +912,7 @@ SP_v1.4.11.2: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.11.3: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: red @@ -931,6 +957,7 @@ SP_v1.4.11.3: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.12.3: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: lightblue @@ -976,6 +1003,7 @@ SP_v1.4.12.3: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.13.3: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: cyan @@ -1021,6 +1049,7 @@ SP_v1.4.13.3: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.11.3_ecut07: + base: SP_v1.4.11.3 subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: darkblue @@ -1065,6 +1094,7 @@ SP_v1.4.11.3_ecut07: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.3_uncal: + blinding: unblinded pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: @@ -1080,6 +1110,7 @@ SP_v1.4.6.3_uncal: hdu: 1 patch_number: 100 SP_v1.4.6.3_uncal_w_iv: + blinding: unblinded pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: @@ -1095,6 +1126,7 @@ SP_v1.4.6.3_uncal_w_iv: hdu: 1 patch_number: 100 SP_v1.4.6.3_uncal_w_1: + blinding: unblinded pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: @@ -1110,6 +1142,7 @@ SP_v1.4.6.3_uncal_w_1: hdu: 1 patch_number: 100 SP_v1.4.5_uncal: + blinding: unblinded pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: @@ -1125,6 +1158,7 @@ SP_v1.4.5_uncal: hdu: 1 patch_number: 100 SP_v1.4.7_uncal: + blinding: unblinded pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: @@ -1140,6 +1174,7 @@ SP_v1.4.7_uncal: hdu: 1 patch_number: 100 SP_v1.4.8_uncal: + blinding: unblinded pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: @@ -1162,6 +1197,7 @@ nz: paths: output: ./output SP_v1.6.6: + blinding: unblinded subdir: /n17data/UNIONS/WL/v1.6.x pipeline: SP colour: violet diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 9908c759..5d8416c6 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -19,11 +19,6 @@ versions: [ # CosmologyValidation suite parameters (cosmo_val.py) # --------------------------------------------------------------------------- cosmo_val: - # Campaign run type: "data" or "mock". One switch: it gates Smokescreen - # blind-at-birth and is stamped as the SACC `type` of every part written - # (custody state at assembly — see blinding.assert_consistent_blind). - type: data - # CosmologyValidation constructor npatch: 100 theta_min: 1.0 diff --git a/scripts/blind_data_vector.py b/scripts/blind_data_vector.py deleted file mode 100644 index e31cb5b3..00000000 --- a/scripts/blind_data_vector.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python3 - -"""Script blind_data_vector.py - -CLI for :mod:`sp_validation.blinding`. ``blind-init`` fixes the blind for one -catalogue version; ``blind-part`` conceals one intermediate part SACC under it, -escrowing the true vector and deleting the plaintext; ``unblind`` verifies the -custody triple and restores a true part (or the assembled file); ``verify`` is -a cheap seedless check of a blinded file against a commitment. - -:Authors: Cail Daley - -Examples --------- -Once per catalogue version:: - - blind_data_vector.py blind-init blinded/ - -Per intermediate part, at birth:: - - blind_data_vector.py blind-part parts/xi_integration.fits --blind-dir blinded/ - -Unblind one part:: - - blind_data_vector.py unblind parts/xi_integration_blinded.fits \\ - --blind-dir blinded/ -o parts/xi_integration.fits - -Verify:: - - blind_data_vector.py verify parts/xi_integration_blinded.fits \\ - blinded/commitment.json -""" - -import argparse -import json -import pathlib -import sys - -from sp_validation import blinding, sacc_io - - -def _config_from_args(args): - """A :class:`blinding.BlindingConfig` from optional CLI overrides.""" - overrides = {} - if args.s8_half_width is not None: - overrides["s8_half_width"] = args.s8_half_width - if args.omega_m_half_width is not None: - overrides["omega_m_half_width"] = args.omega_m_half_width - return blinding.BlindingConfig.from_overrides(overrides) - - -def _blind_init(args): - config = _config_from_args(args) - blind_dir = pathlib.Path(args.blind_dir) - blind_dir.mkdir(parents=True, exist_ok=True) - try: - blinding.blind_init(str(blind_dir), config=config, label=args.label) - except FileExistsError as exc: - raise SystemExit(f"{exc}\nPick a fresh blind dir (never overwrite a blind).") - print( - "Commit the commitment JSON to the repo; keep the bundle + key safe " - "and separated (colocation in the blind dir is not at-rest protection)." - ) - - -def _blind_part(args): - blinding.blind_part( - args.part, - args.blind_dir, - config=_config_from_args(args), - keep_input=args.keep_input, - ) - - -def _unblind(args): - blinding.unblind_part( - args.blinded, - args.blind_dir, - args.output, - config=_config_from_args(args), - ) - - -def _verify(args): - # allow_unblinded=True: reporting that a file is *not* concealed is one of - # the outcomes here, so the fail-closed loader must not pre-empt it. - s = sacc_io.load(args.blinded, allow_unblinded=True) - with open(args.commitment, encoding="utf-8") as f: - commitment = json.load(f) - problems = blinding.verify(s, commitment) - if problems: - raise SystemExit("verification FAILED:\n " + "\n ".join(problems)) - print( - f"OK: {args.blinded} matches {args.commitment} " - f"(blind {s.metadata.get('blind')!r})" - ) - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[1]) - sub = parser.add_subparsers(dest="mode", required=True) - - for name in ("blind-init", "blind-part", "unblind"): - p = sub.add_parser(name) - p.add_argument("--s8-half-width", type=float, default=None) - p.add_argument("--omega-m-half-width", type=float, default=None) - if name == "blind-init": - p.add_argument( - "blind_dir", - help="directory for the blind's fixed state (commitment + " - "encrypted seed bundle)", - ) - p.add_argument("--label", default="A", help="blind label (default A)") - p.set_defaults(func=_blind_init) - elif name == "blind-part": - p.add_argument("part", help="intermediate part SACC file to blind") - p.add_argument( - "--blind-dir", required=True, help="blind-init state directory" - ) - p.add_argument( - "--keep-input", - action="store_true", - help="retain the plaintext input part (default: delete it " - "after blinding — the true vector is escrowed beside the " - "blinded output)", - ) - p.set_defaults(func=_blind_part) - else: - p.add_argument("blinded", help="blinded part (or assembled) SACC file") - p.add_argument( - "--blind-dir", required=True, help="blind-init state directory" - ) - p.add_argument("-o", "--output", required=True, help="output SACC path") - p.set_defaults(func=_unblind) - - p = sub.add_parser("verify") - p.add_argument("blinded", help="blinded SACC file") - p.add_argument("commitment", help="commitment JSON") - p.set_defaults(func=_verify) - - args = parser.parse_args(argv) - args.func(args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/sp_validation/CONTRACTS b/src/sp_validation/CONTRACTS new file mode 100644 index 00000000..3ce58a3f --- /dev/null +++ b/src/sp_validation/CONTRACTS @@ -0,0 +1,20 @@ +@sc custody-is-stdlib +Custody is resolved by the host Snakemake, which loads custody.py by path with +no sp_validation installed, and by container jobs, from the same declaration. +It imports nothing but the standard library. +only: sp_validation.custody may import stdlib + +@sc container-is-stdlib +The image model is loaded by path by the host Snakemake and runs as the +spv-container CLI before any image exists. +only: sp_validation.container may import stdlib + +@sc blinding-owns-smokescreen +The Smokescreen fork draws the hidden cosmology and computes the concealing +factor; only the blind module calls it, so the draw has one home. +only: smokescreen* may be imported by sp_validation.blinding + +@sc blinding-owns-the-seed-cipher +A blind's seed exists on disk only as Fernet ciphertext, written and read by +the blind module. +only: cryptography* may be imported by sp_validation.blinding diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index fdc2ddf8..89dd5e0a 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -277,6 +277,7 @@ def pure_eb_covariance_mc( nz, cosmo, n_samples=1000, + rng=None, ): """Pure-E/B covariance by Monte Carlo through the same kernel as the modes. @@ -284,7 +285,9 @@ def pure_eb_covariance_mc( around the theory mean for ``(z, nz)`` under ``cosmo``; each draw is binned down to the reporting grid and pushed through :func:`pure_eb_from_xi`. The covariance of the transformed draws is the result, so it depends on the - covariance model and the grids, never on the measured data vector. + covariance model and the grids, never on the measured data vector. ``rng`` + (a ``numpy.random.Generator``) draws the realisations; fresh entropy when + ``None``. Returns ``(cov, eb_samples)`` — the covariance in ``_EB_KEYS`` order and the draws behind it. @@ -309,7 +312,8 @@ def pure_eb_covariance_mc( theta=theta_int, z=z, nz=nz, backend="ccl", cosmo=cosmo ).values() mean_int = np.concatenate(xi_pm) - samples_int = np.random.multivariate_normal(mean_int, cov_int, size=n_samples) + rng = np.random.default_rng() if rng is None else rng + samples_int = rng.multivariate_normal(mean_int, cov_int, size=n_samples) samples_int_xip, samples_int_xim = ( samples_int[:, :nbins_int], samples_int[:, nbins_int:], diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 90f93023..9cbfafb4 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -1,841 +1,722 @@ -"""Blinding — conceal each intermediate data product behind a hidden cosmology. +"""The blind: one secret seed, hence one hidden cosmology, and its custody. :Name: blinding.py -:Description: Smokescreen blinding, per part, at birth. Each blindable - intermediate SACC product — reporting ξ±, integration ξ±, pseudo-Cℓ — is - shifted the moment the pipeline computes it by a difference of theory - vectors between the fiducial and a *hidden* cosmology drawn inside a fixed - amplitude envelope (Muir et al. 2019: ``d → d + t(hidden) − t(fiducial)``), - so S8 cannot be read off the data before the collaboration unblinds. Only - blinded parts persist on disk. - - The ``UNIONS-WL/Smokescreen`` fork draws the hidden cosmology and computes - the shift; blinding is a vector operation, so this module calls its vector - core (``smokescreen.concealing_factor``), which never sees a SACC. What is - sp_validation-specific is supplied here: the ``theory_fn`` backends - matching each part's row layout, and the SACC handling around the factors. - - Derived statistics are born blinded — COSEBIs and pure-E/B run downstream - on the already-blinded integration ξ±. Covariance and ρ/τ are never blinded: - blinding hides the vector, not the uncertainty, and the shift is pure - E-mode so B-mode null tests stay honest under the blind. - - **Custody: hash commitment, no keyholder.** A blind is reproduced by a - *triple* — seed, config digest, and Smokescreen ``DRAW_SCHEME`` (the seed - alone is not enough; see :func:`draw_scheme`). :func:`blind_init` fixes all - three per catalogue version, publishing the commitment triple as a - repo-committable ``commitment.json`` and encrypting the seed into a Fernet - bundle — the plaintext seed is never written. Every gate that draws or - subtracts a shift fails closed unless the triple matches - (:func:`_assert_draw_scheme`, :func:`assert_consistent_blind`, - :func:`unblind_sacc`), and there is no override. +:Description: A blind conceals a catalogue's cosmological signal (Muir et al. + 2019): every ξ± and pseudo-Cℓ_EE row is shifted by t(hidden) − t(fiducial), + the theory difference between a hidden cosmology and the fiducial, before + the row is first written (:func:`sp_validation.sacc_io.seal`). The hidden + point is drawn uniformly in S8 and Ωm about the fiducial from a secret seed, + with the Smokescreen fork's per-key RNG. COSEBIs and pure-E/B computed from + concealed ξ± are concealed with it; the shift is pure E-mode, so B-mode null + tests stay valid. + + A blind is drawn once, by a person, with ``python -m sp_validation.blinding + init``, into the registry beside the catalogue config + (``cosmo_val/blinds//``): the public ``commitment.json`` (the seed's + commitment and the full :class:`BlindingConfig`), the seed as Fernet + ciphertext (``seed.fernet``), its ``key``, and the ``bases`` it covers. The + seed exists on disk only inside the ciphertext; the encryption keeps anyone + from reading it by accident. ``share`` adds a catalogue to a blind; + ``reveal`` publishes the seed and moves the concealed files aside; + ``audit`` proves blinded − true = shift(seed) once the true files are + re-measured; ``verify`` checks a file's stamp against the record, seedless. """ +import argparse import dataclasses import functools import hashlib import json import os import secrets -import warnings +import shutil +import sys +from datetime import datetime, timezone +from pathlib import Path import numpy as np +import yaml +from . import custody as _custody from . import sacc_io -from .blinding_paths import init_paths, part_paths # noqa: F401 (re-exported) -from .blinding_theory import TheoryConfig, cl_ee, coerce_fields, xi_ccl, xi_ell_grid +from .blinding_theory import TheoryConfig, cl_ee, xi_ccl + +_BLIND_NAME = "abcdefghijklmnopqrstuvwxyz0123456789-_." # --------------------------------------------------------------------------- # -# Configuration surface — the blinding envelope +# The config a blind is drawn under, and its digest # --------------------------------------------------------------------------- # @dataclasses.dataclass(frozen=True) class BlindingConfig: - """Blinding envelope and fiducial. - - The hidden cosmology is drawn (by the fork) uniformly and independently - per key inside ``shifts_dict()``'s half-widths about the fiducial. The - half-widths are the deliberate, configurable size of the blind — config, - not code; the group may resize the envelope. ``theory`` carries the - fiducial :class:`TheoryConfig` whose defaults *are* the blinding - fiducial. + """The envelope of the hidden draw and the theory the shift is computed in. + + ``envelope`` maps a :class:`TheoryConfig` field to the half-width of its + uniform draw about the fiducial ``theory``. """ - s8_half_width: float = 0.075 - omega_m_half_width: float = 0.1 + envelope: dict = dataclasses.field( + default_factory=lambda: {"S8": 0.075, "Omega_m": 0.1} + ) theory: TheoryConfig = dataclasses.field(default_factory=TheoryConfig) - def shifts_dict(self): - """The (S8, Ωm) envelope as CCL-native ``{sigma8, Omega_c}`` half-widths. + def record(self): + """The config as plain data: every field, numbers as floats.""" + return _canonical(self) - Evaluated at the fiducial: a ΔS8 half-width maps to - ``ΔS8/√(Ωm_fid/0.3)`` in σ8 (at fixed Ωm), and a ΔΩm half-width maps - one-to-one to Ω_c (Ω_b and Ω_ν fixed). Exact enough for a smear whose - target is a characteristic amplitude, not a precise posterior. - """ + @classmethod + def from_record(cls, record): + """The config a record describes; fields it omits take their defaults.""" + fields = {} + if "envelope" in record: + fields["envelope"] = {k: float(v) for k, v in record["envelope"].items()} + if "theory" in record: + fields["theory"] = TheoryConfig(**record["theory"]) + return cls(**fields) + + def digest(self): + """sha256 of the canonical record; int and float literals agree.""" + text = json.dumps(self.record(), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _canonical(value): + if dataclasses.is_dataclass(value): return { - "sigma8": self.s8_half_width / np.sqrt(self.theory.Omega_m / 0.3), - "Omega_c": self.omega_m_half_width, + f.name: _canonical(getattr(value, f.name)) + for f in dataclasses.fields(value) } - - def config_digest(self): - """sha256 of a canonical serialization of the full blinding config. - - Binds the envelope half-widths and the complete fiducial - :class:`TheoryConfig` into one digest, as JSON with sorted keys. - Fields go through :func:`~sp_validation.blinding_theory.coerce_fields`, - so the digest depends on the numeric value rather than on an - int-vs-float literal, and ``json`` emits floats by shortest round-trip - ``repr`` — two runs of one config give byte-identical digests. Checked - with the seed commitment at unblind. - """ - payload = coerce_fields( - type(self), - { - "s8_half_width": self.s8_half_width, - "omega_m_half_width": self.omega_m_half_width, - }, - ) - payload["theory"] = coerce_fields( - TheoryConfig, - { - f.name: getattr(self.theory, f.name) - for f in dataclasses.fields(self.theory) - }, - ) - return hashlib.sha256( - json.dumps(payload, sort_keys=True).encode("utf-8") - ).hexdigest() - - @classmethod - def from_overrides(cls, overrides): - """Build from a mapping of field overrides (fail loud on unknown keys). - - ``theory`` may be given as a :class:`TheoryConfig` or a mapping of - TheoryConfig overrides, mirroring :meth:`TheoryConfig.from_overrides`. - """ - overrides = coerce_fields(cls, overrides) - theory = overrides.get("theory") - if theory is not None and not isinstance(theory, TheoryConfig): - overrides["theory"] = TheoryConfig.from_overrides(dict(theory)) - return cls(**overrides) - - -# --------------------------------------------------------------------------- # -# Custody primitives -# --------------------------------------------------------------------------- # -def seed_commitment(seed): - """Public commitment for a seed: the fork's domain-separated sha256 digest. - - Domain-separated because the fork derives its RNG base seed from the bare - sha256 of the same string — an undomained commitment would publish it. - """ - from smokescreen import seed_commitment as _fork_seed_commitment - - return _fork_seed_commitment(seed) + if isinstance(value, dict): + return {str(k): _canonical(v) for k, v in value.items()} + if isinstance(value, bool) or isinstance(value, str): + return value + if isinstance(value, (int, float, np.integer, np.floating)): + return float(value) + raise TypeError(f"cannot serialise {value!r} into a blind's record") def draw_scheme(): - """The installed Smokescreen fork's shift-draw semantics version. - - ``smokescreen.DRAW_SCHEME`` versions *how* a seed becomes parameter deltas - (scheme 1: upstream DESC's single global RNG over the sorted keys; scheme - 2: this fork's per-key RNG from ``(seed, key)``). Two installs agreeing on - ``(seed, config)`` but not on this number draw different hidden cosmologies. - - This is the one blinding failure with no loud symptom — a scheme mismatch - leaves a smooth residual cosmological shift in the "unblinded" vector, and - every hash, digest and escrow check still passes. Hence the scheme is - custody state, checked wherever a shift is drawn or subtracted. - """ + """The installed fork's shift-draw semantics (``smokescreen.DRAW_SCHEME``).""" from smokescreen import DRAW_SCHEME return int(DRAW_SCHEME) -def _assert_draw_scheme(recorded, what): - """Fail closed unless ``recorded`` is the installed fork's draw scheme. - - ``what`` names the surface the scheme was read from, for the message. - A missing record (``None``) is a failure, not a pass: a blind whose scheme - is unknown cannot be shown to be reproducible by this install (see - :func:`draw_scheme`). - """ - installed = draw_scheme() - if recorded is None: - raise ValueError( - f"{what} carries no draw-scheme record — refusing to proceed. " - f"It predates draw-scheme binding, so there is no way to tell " - f"whether the installed Smokescreen (DRAW_SCHEME={installed}) " - f"reproduces the shift it was blinded with." - ) - if int(recorded) != installed: - raise ValueError( - f"{what} was drawn under Smokescreen DRAW_SCHEME={int(recorded)} " - f"but the installed fork implements DRAW_SCHEME={installed} — " - f"refusing to proceed. The same seed draws a different hidden " - f"cosmology under a different scheme, so this install would " - f"subtract the wrong shift and pass every other check. Install " - f"the Smokescreen the blind was made with." - ) - - -def hidden_params(seed, config): - """The hidden CCL parameter point the fork realizes for ``(seed, config)``. +def hidden_theory(seed, config): + """The hidden point: the fiducial theory moved by the seed's draw. - Re-runs the fork's draw and overlays the deltas on the fiducial, as - ``concealing_factor`` does internally. Introspection only (what *was* the - hidden cosmology, once revealed) — the blinding path never calls it, so it - does not gate on the draw scheme; every gate that acts on a shift does. + @sc hidden-draw-uniform-s8-om + The draw is uniform within the envelope in the physical coordinates it + names (S8 and Ωm), by the fork's per-key RNG; CCL's σ8 and Ω_c follow from + the drawn point through ``TheoryConfig.ccl_params``, never the reverse. """ from smokescreen.param_shifts import draw_param_shifts - deltas = draw_param_shifts(config.shifts_dict(), seed) - params = dict(config.theory.ccl_params()) - for key, delta in deltas.items(): - params[key] += delta - return params + shift = draw_param_shifts(dict(config.envelope), seed) + return dataclasses.replace( + config.theory, + **{key: getattr(config.theory, key) + delta for key, delta in shift.items()}, + ) # --------------------------------------------------------------------------- # -# Block discovery on a SACC (a standalone part, or the assembled file) +# Opening a blind # --------------------------------------------------------------------------- # -def source_bins(s): - """Sorted source-bin indices present in ``s`` (from ``source_i`` tracers).""" - return sorted( - int(name.split("_", 1)[1]) for name in s.tracers if name.startswith("source_") - ) - - -def _pairs(s, data_type, **tags): - """Unordered source-bin pairs ``(i ≤ j)`` carrying ``data_type`` in ``s``.""" - bins = source_bins(s) - return [ - (i, j) - for a, i in enumerate(bins) - for j in bins[a:] - if len(s.indices(data_type, sacc_io._pair((i, j)), **tags)) - ] +@dataclasses.dataclass(frozen=True) +class Blind: + """An opened blind. Its ``repr`` hides the seed.""" + name: str + seed: str = dataclasses.field(repr=False) + config: BlindingConfig + hidden: TheoryConfig -def xi_pairs(s, grid): - """Unordered source-bin pairs ``(i ≤ j)`` carrying ξ+ on ``grid``.""" - return _pairs(s, sacc_io.XI_PLUS, grid=grid) +def _read_record(registry, name): + record = Path(registry) / name + try: + commitment = json.loads((record / "commitment.json").read_text()) + except (OSError, ValueError) as err: + raise _custody.CustodyError( + f"blind {name}: no readable record: {err}" + ) from None + return record, commitment -def cl_pairs(s): - """Source-bin pairs ``(i ≤ j)`` carrying pseudo-Cℓ_EE.""" - return _pairs(s, sacc_io.CL_EE) +@functools.cache +def _open(registry, name): + from cryptography.fernet import Fernet, InvalidToken -def _xi_indices(s, grid): - """Row indices of the ξ± block on ``grid`` (ascending).""" - return np.sort( - np.concatenate( - [ - s.indices(sacc_io.XI_PLUS, grid=grid), - s.indices(sacc_io.XI_MINUS, grid=grid), - ] + record, commitment = _read_record(registry, name) + try: + key = (record / "key").read_bytes() + payload = json.loads(Fernet(key).decrypt((record / "seed.fernet").read_bytes())) + except (OSError, ValueError, InvalidToken) as err: + raise _custody.CustodyError( + f"blind {name}: the seed does not decrypt with its key ({type(err).__name__})" + ) from None + config = BlindingConfig.from_record(commitment["config"]) + checks = { + "blind name": (payload["blind"], commitment["blind"], name), + "config digest": ( + payload["config_digest"], + commitment["config_digest"], + config.digest(), + ), + "draw scheme": ( + payload["draw_scheme"], + commitment["draw_scheme"], + draw_scheme(), + ), + "seed commitment": ( + _custody.seed_commitment(payload["seed"]), + commitment["seed_commitment"], + ), + } + for what, values in checks.items(): + if len(set(values)) != 1: + raise _custody.CustodyError( + f"blind {name}: the {what} disagrees between the sealed seed, the " + "record and this install; the record was edited or the install " + "draws differently" + ) + seed = payload["seed"] + return Blind(name, seed, config, hidden_theory(seed, config)) + + +def open_blind(custody): + """The verified blind a blinded custody names (cached per process).""" + if custody.status != "blinded": + raise _custody.CustodyError(f"{custody.catalogue} is {custody.status}") + blind = _open(Path(custody.registry), custody.blind) + if _custody.seed_commitment(blind.seed) != custody.commitment: + raise _custody.CustodyError( + f"blind {custody.blind} is not the one {custody.catalogue} resolved" ) - ).astype(int) - - -def _cl_ee_indices(s): - """Row indices of the pseudo-Cℓ_EE block (ascending). - - Only EE: a pure E-mode cosmology shift leaves BB and EB identically - zero, so those blocks are never extracted, never concealed. - """ - return np.sort(s.indices(sacc_io.CL_EE)).astype(int) - - -def _pair_nz(s, i, j): - """The two per-bin n(z) for pair ``(i, j)`` as ``((z_i, n_i), (z_j, n_j))``.""" - z_i, n_i = sacc_io.get_nz(s, i) - z_j, n_j = sacc_io.get_nz(s, j) - return (np.asarray(z_i), np.asarray(n_i)), (np.asarray(z_j), np.asarray(n_j)) + return blind # --------------------------------------------------------------------------- # -# The three theory backends — callables aligned to a sub-SACC block's rows +# Blocks, from the content, and the concealment # --------------------------------------------------------------------------- # -def xi_theory_fn(block, theory, grid): - """``theory_fn`` for a ξ± sub-SACC block (reporting or integration grid). - - Reads each bin's n(z) from the block's own tracers and lays the output out - to match the block's SACC rows element-for-element: per pair, ξ± at that - pair's stored θ (the ``theta`` tag, arcmin), scattered to the rows - ``block.indices`` reports — never an assumed pairing. - """ - pairs = xi_pairs(block, grid) - layout = [] - for i, j in pairs: - tr = sacc_io._pair((i, j)) - idx_p = block.indices(sacc_io.XI_PLUS, tr, grid=grid) - idx_m = block.indices(sacc_io.XI_MINUS, tr, grid=grid) - theta = sacc_io._tag(block, sacc_io.XI_PLUS, tr, "theta", grid=grid) - layout.append(((i, j), idx_p, idx_m, np.asarray(theta, dtype=float))) - ell = xi_ell_grid() - - def theory_fn(params): - out = np.full(len(block.mean), np.nan) - for (i, j), idx_p, idx_m, theta in layout: - nz_i, nz_j = _pair_nz(block, i, j) - xip, xim = xi_ccl(params, theory, nz_i, nz_j, theta, ell) - out[idx_p] = xip - out[idx_m] = xim - return out - - return theory_fn - - -def cl_theory_fn(block, theory): - """``theory_fn`` for the pseudo-Cℓ_EE sub-SACC block. - - Per pair: theory Cℓ_EE on the stored ``BandpowerWindows`` support, binned - by the same window matrix the measurement used (``W @ Cℓ_EE``), scattered - to the block's own rows — so the shift lands in the measured bandpowers. - ΔBB = ΔEB ≡ 0 for a pure E-mode shift, hence EE-only rows. - """ - layout = [] - for i, j in cl_pairs(block): - tr = sacc_io._pair((i, j)) - idx = block.indices(sacc_io.CL_EE, tr) - window = block.get_bandpower_windows(idx) - layout.append( - ( - (i, j), - np.asarray(idx), - np.asarray(window.values, dtype=float), # (n_ell,) - np.asarray(window.weight, dtype=float), # (n_ell, n_bp) - ) - ) - - def theory_fn(params): - out = np.full(len(block.mean), np.nan) - for (i, j), idx, w_ell, w_mat in layout: - nz_i, nz_j = _pair_nz(block, i, j) - out[idx] = w_mat.T @ cl_ee(params, theory, nz_i, nz_j, w_ell) - return out +@dataclasses.dataclass(frozen=True) +class Block: + """Rows of one statistic and tracer pair, and their theory at a point.""" - return theory_fn + name: str + rows: np.ndarray + theory: object # (ccl params, TheoryConfig) -> values aligned to rows -# --------------------------------------------------------------------------- # -# The concealing factor, per block -# --------------------------------------------------------------------------- # -def _blindable_blocks(s): - """The blindable blocks of a SACC as ``(name, indices, factory)``. +def _pair_nz(s, tracers): + nzs = [] + for name in tracers: + tracer = s.tracers[name] + if not hasattr(tracer, "nz"): + raise ValueError(f"shiftable rows name tracer {name}, which has no n(z)") + nzs.append((np.asarray(tracer.z, float), np.asarray(tracer.nz, float))) + return nzs - Works identically on a standalone part (exactly one block) and on the - assembled file (integration rows selected by the ``grid`` tag). ``indices`` - are each block's recorded row indices (ascending); ``factory`` builds the - matching ``theory_fn``. Blocks absent from the file are not listed. - """ - blocks = [] - for grid in ("reporting", "integration"): - idx = _xi_indices(s, grid) - if len(idx): - blocks.append( - (f"{grid} ξ±", idx, functools.partial(xi_theory_fn, grid=grid)) - ) - idx = _cl_ee_indices(s) - if len(idx): - blocks.append(("pseudo-Cℓ_EE", idx, cl_theory_fn)) - return blocks - - -def _concealing_factor(s, indices, factory, config, seed): - """The fork-computed additive concealing factor for one block of ``s``. - - ``smokescreen.concealing_factor`` draws the hidden deltas from ``seed``, - evaluates the block's ``theory_fn`` at both cosmologies and differences - them. No data vector and no SACC reach the fork, and ``s`` is not modified. - Both :func:`blind_sacc` and :func:`unblind_sacc` come through here, so the - added and subtracted shifts cannot drift apart. - - A ``theory_fn`` fills only its own rows and leaves the rest NaN, so - slicing to ``indices`` drops the NaNs by construction and the finite check - proves the converse: that *all* of this block's rows were filled. A row - the block claims but the factory cannot cover (a pair with ξ− but no ξ+, - say) would otherwise be shifted by NaN, silently. - - Returns - ------- - np.ndarray - ``t(hidden) − t(fiducial)``, aligned to ``indices``. - """ - from smokescreen import concealing_factor - - full = np.asarray( - concealing_factor( - config.theory.ccl_params(), - config.shifts_dict(), - seed=seed, - theory_fn=factory(s, config.theory), - factor_type="add", - ), - dtype=float, - ) - factor = full[indices] - if not np.all(np.isfinite(factor)): - raise ValueError( - f"the theory backend left {int(np.sum(~np.isfinite(factor)))} of " - f"{len(indices)} blindable rows unfilled — refusing to apply the " - "concealing factor (these rows would be shifted by NaN). The " - "block's row layout is not fully covered by its theory_fn." - ) - return factor +def _xi_block(s, tracers, rows): + nz_i, nz_j = _pair_nz(s, tracers) + theta = np.array([s.data[i].tags["theta"] for i in rows], float) + plus = np.array([s.data[i].data_type == sacc_io.XI_PLUS for i in rows]) + grid = np.unique(theta) + at = np.searchsorted(grid, theta) -def _set_values(s, indices, values): - """Overwrite ``s.data[i].value`` for ``indices`` with ``values`` (aligned).""" - for i, v in zip(indices, values): - s.data[int(i)].value = float(v) + def theory(params, config): + xip, xim = xi_ccl(params, config, nz_i, nz_j, grid) + return np.where(plus, np.asarray(xip)[at], np.asarray(xim)[at]) + return Block(f"ξ± {tracers[0]}×{tracers[1]}", np.asarray(rows), theory) -def _concealed(s): - """Whether ``s`` is already a blinded file (its ``concealed`` mark is set).""" - return bool(s.metadata.get("concealed")) +def _cl_block(s, tracers, rows): + nz_i, nz_j = _pair_nz(s, tracers) + window = s.get_bandpower_windows(rows) + ell = np.asarray(window.values, float) + weight = np.asarray(window.weight, float) -def _apply_blocks(src, seed, config, sign, verb, log): - """Return a copy of ``src`` with each block's concealing factor applied. + def theory(params, config): + return weight.T @ cl_ee(params, config, nz_i, nz_j, ell) - ``sign`` is ``+1`` to conceal and ``-1`` to reveal; the factor itself comes - from the one :func:`_concealing_factor` call both directions share. - """ - dst = src.copy() - for name, indices, factory in _blindable_blocks(src): - factor = _concealing_factor(src, indices, factory, config, seed) - _set_values(dst, indices, np.asarray(dst.mean)[indices] + sign * factor) - log(f"[{verb}] {name}: {len(indices)} points") - return dst + return Block(f"Cℓ_EE {tracers[0]}×{tracers[1]}", np.asarray(rows), theory) -def blind_sacc(part, seed, config=None, label="smokescreen", log=print): - """Return a blinded copy of a part SACC (covariance and tags untouched). +def shiftable_blocks(s): + """Every ξ± and Cℓ_EE block of ``s``: by tracer pair (and window, for Cℓ). - Adds each blindable block's concealing factor at the block's recorded - indices; only ``value`` changes, and only on blindable rows. Provenance is - stamped and any leaked seed key stripped. A file with no blindable block - (a ρ/τ diagnostic part, say) is refused loudly. + @sc shift-from-content + Blocks are discovered from the data types and tracers alone, whatever the + ``grid`` tags or file names; each ξ± row is evaluated at its own ``theta`` + from its pair's two n(z), each Cℓ_EE row through its bandpower window. A + shiftable row without a ``theta`` tag or a window is refused. """ - config = config or BlindingConfig() - if _concealed(part): - raise ValueError("already concealed — unblind first") - if not _blindable_blocks(part): - raise ValueError( - "no blindable block (reporting/integration ξ± or pseudo-Cℓ_EE) in this SACC " - "— ρ/τ diagnostic parts are never blinded" - ) + xi, cl = {}, {} + for i, dp in enumerate(s.data): + if dp.data_type in (sacc_io.XI_PLUS, sacc_io.XI_MINUS): + if "theta" not in dp.tags: + raise ValueError(f"ξ± row {i} has no theta tag to evaluate theory at") + xi.setdefault(tuple(dp.tracers), []).append(i) + elif dp.data_type == sacc_io.CL_EE: + if dp.tags.get("window") is None: + raise ValueError(f"Cℓ_EE row {i} has no bandpower window") + key = (tuple(dp.tracers), id(dp.tags["window"])) + cl.setdefault(key, []).append(i) + return [_xi_block(s, t, rows) for t, rows in xi.items()] + [ + _cl_block(s, t, rows) for (t, _), rows in cl.items() + ] - blinded = _apply_blocks(part, seed, config, +1, "blind", log) - _stamp_provenance(blinded, seed_commitment(seed), label, config.config_digest()) - return blinded +def _factor(block, fiducial, hidden): + """t(hidden) − t(fiducial) on ``block``'s rows, by the fork.""" + from smokescreen import factor_from_params -def unblind_sacc(blinded, seed, config=None, log=print): - """Recover the true part SACC from a blinded one + the revealed ``seed``. + return np.asarray( + factor_from_params( + fiducial.ccl_params(), + hidden.ccl_params(), + theory_fn=lambda params: block.theory(params, fiducial), + ), + float, + ) - Verifies the custody triple — draw scheme, seed commitment, config digest — - against the file's stamps before subtracting anything, then recomputes each - block's shift the same way :func:`blind_sacc` added it. Works on a part or - on the assembled file. Derived statistics, if present, are *not* recomputed - here — the pipeline re-derives them from the unblinded integration ξ±. - """ - config = config or BlindingConfig() - if not _concealed(blinded): - raise ValueError("file is not concealed — nothing to unblind") - _assert_draw_scheme(blinded.metadata.get("blind_draw_scheme"), "this blinded file") - if seed_commitment(seed) != blinded.metadata["blind_commitment"]: - raise ValueError( - "seed does not match blind_commitment — refusing to unblind " - "(a wrong seed would silently produce a wrong data vector)" - ) - if config.config_digest() != blinded.metadata["blind_config_digest"]: - raise ValueError( - "blinding config does not match blind_config_digest — refusing to " - "unblind (this config would subtract a different shift than was " - "added)" - ) - part = _apply_blocks(blinded, seed, config, -1, "unblind", log) - for key in ( - "concealed", - "blind", - "blind_commitment", - "blind_config_digest", - "blind_draw_scheme", - ): - part.metadata.pop(key, None) - return part +def factors(s, fiducial, hidden): + """Each shiftable block of ``s`` and its factor; together they cover every row.""" + blocks = shiftable_blocks(s) + covered = np.sort(np.concatenate([b.rows for b in blocks])) if blocks else [] + shiftable = [i for i, dp in enumerate(s.data) if dp.data_type in sacc_io.SHIFTABLE] + if not np.array_equal(covered, shiftable): + raise ValueError("shiftable blocks do not cover every ξ± and Cℓ_EE row once") + out = [] + for block in blocks: + factor = _factor(block, fiducial, hidden) + if factor.shape != block.rows.shape or not np.all(np.isfinite(factor)): + raise ValueError( + f"{block.name}: the theory left rows unfilled or non-finite" + ) + out.append((block, factor)) + return out -def _stamp_provenance(s, commitment, label, config_digest): - """Stamp the custody triple and the ``concealed``/``blind`` marks. +def conceal(s, blind): + """A copy of ``s`` with every ξ± and Cℓ_EE row shifted by the blind. - Pops ``seed_smokescreen`` (which upstream Smokescreen's writer would - stamp): the seed never rides a kept file. The scheme stamped is the - installed fork's, which every caller has already checked the blind against. - """ - s.metadata.pop("seed_smokescreen", None) - s.metadata["concealed"] = True - s.metadata["blind"] = label - s.metadata["blind_commitment"] = commitment - s.metadata["blind_config_digest"] = config_digest - s.metadata["blind_draw_scheme"] = draw_scheme() - - -def stamp_concealed_passthrough(s, commitment_path): - """Stamp a part concealed under an existing blind, values untouched. - - The seam for parts already blind (COSEBIs / pure-E/B, re-derived from the - blinded integration ξ±) or blind-irrelevant (ρ/τ, no cosmological vector): - it shifts nothing and needs no blindable block, only the custody stamp that - lets the load gate and :func:`assert_consistent_blind` admit the part. The - stamp is read from the version's ``commitment.json``, so a pass-through - part carries the exact custody state of the blinded parts. The committed - scheme is checked first — stamping from an install that draws differently - would mint a custody claim it cannot honour. + Every block's factor is computed before any value changes, and only the + copy changes. """ - with open(commitment_path, encoding="utf-8") as f: - commitment = json.load(f) - _assert_draw_scheme( - commitment.get("draw_scheme"), f"the blind at {commitment_path}" - ) - _stamp_provenance( - s, - commitment["seed_commitment"], - commitment["label"], - commitment["config_digest"], - ) - return s + shifts = factors(s, blind.config.theory, blind.hidden) + out = s.copy() + for block, factor in shifts: + for row, delta in zip(block.rows, factor): + out.data[int(row)].value = out.data[int(row)].value + float(delta) + return out # --------------------------------------------------------------------------- # -# Assembly-time custody: one blind across all parts +# Custody commands: init, share, reveal, audit, verify # --------------------------------------------------------------------------- # -def assert_consistent_blind(parts): - """Assert every blindable part shares one blind; return the shared stamp. - - The assembly gate of :func:`sp_validation.sacc_io.gather`. A part is - *blindable* if it carries a blindable block (ξ± or pseudo-Cℓ_EE); ρ/τ and - covariance-only parts are exempt. Fails closed on a blinded/plaintext mix, - on parts whose custody triples disagree, or on a shared scheme this install - does not implement (it could not unblind what it is assembling). The - consistency key is the triple; the ``blind`` label is provenance, not - custody, so differing labels warn rather than fail. - - Unconcealed blindable parts must all be declared ``type == "mock"``: an - unconcealed ``type == "data"`` part, or one missing the tag, fails closed, - so skipping the blind can never silently expose real data. - - Returns - ------- - dict or None - The shared blind metadata (``concealed``, ``blind``, - ``blind_commitment``, ``blind_config_digest``, ``blind_draw_scheme``) - for the gather to stamp on the assembled file, or ``None`` when - nothing is blinded. - """ - blindable = [p for p in parts if _blindable_blocks(p)] - concealed = [p for p in blindable if _concealed(p)] - if not concealed: - # `.get` is deliberate here: a missing `type` tag must count as - # not-a-mock and fail closed, not KeyError with less context. - exposed = sorted( - {str(p.metadata.get("type", "")) for p in blindable} - {"mock"} - ) - if exposed: - raise ValueError( - f"unconcealed blindable parts with type {exposed} in assembly " - "— only parts declared `type: mock` may assemble without a " - "blind (an unconcealed data part exposes the real vector)" - ) - return None - if len(concealed) != len(blindable): - raise ValueError( - f"blinded and plaintext blindable parts mixed in one assembly " - f"({len(concealed)} of {len(blindable)} blinded) — refusing to " - "combine (a plaintext part beside blinded ones leaks the shift)" - ) - # `.get` on the scheme so a part predating scheme binding reads as None and - # fails at _assert_draw_scheme with its explanation, not with a KeyError. - stamps = { - ( - p.metadata["blind_commitment"], - p.metadata["blind_config_digest"], - p.metadata.get("blind_draw_scheme"), - ) - for p in concealed - } - if len(stamps) != 1: - raise ValueError( - "parts carry different blind commitments — they were blinded " - "under different seeds, configs or draw schemes and must never be " - "combined: " - + "; ".join(f"({c[:12]}…, {d[:12]}…, scheme {v})" for c, d, v in stamps) - ) - ((commitment, digest, scheme),) = stamps - _assert_draw_scheme(scheme, "the blind these parts share") - labels = sorted({p.metadata["blind"] for p in concealed}) - if len(labels) != 1: - warnings.warn( - f"blindable parts share one blind (commitment {commitment[:12]}…, " - f"config {digest[:12]}…) but carry different labels {labels} — " - "assembling anyway; the label is provenance, not custody state. " - f"Stamping the assembled file with label {labels[0]!r}." - ) - return { - "concealed": True, - "blind": labels[0], - "blind_commitment": commitment, - "blind_config_digest": digest, - "blind_draw_scheme": int(scheme), - } +def _catalogues(cat_config): + return yaml.safe_load(Path(cat_config).read_text()) -# --------------------------------------------------------------------------- # -# File-level custody: blind-init / blind-part / unblind -# --------------------------------------------------------------------------- # -def verify(s, commitment): - """Problems found comparing a blinded SACC to a commitment, seedlessly. - - Returns a (possibly empty) list of human-readable strings. No seed is read, - so this cannot confirm the blind is *subtractable* — only that the file's - custody triple matches ``commitment`` (a parsed ``commitment.json``) and - that the recorded draw scheme is the one this install implements. That last - check is environment-dependent by design: a machine carrying a different - Smokescreen could not unblind the file, so it reports a problem. - """ - problems = [] - if not _concealed(s): - problems.append("file is not marked concealed") - if s.metadata.get("blind_commitment") != commitment["seed_commitment"]: - problems.append("blind_commitment does not match the committed seed commitment") - if s.metadata.get("blind_config_digest") != commitment["config_digest"]: - problems.append("blind_config_digest does not match the committed digest") - scheme = s.metadata.get("blind_draw_scheme") - if scheme != commitment.get("draw_scheme"): - problems.append( - f"blind_draw_scheme {scheme!r} does not match the committed " - f"draw_scheme {commitment.get('draw_scheme')!r}" - ) - try: - _assert_draw_scheme(scheme, "the blinded file") - except ValueError as exc: - problems.append(str(exc)) - if "seed_smokescreen" in s.metadata: - problems.append("PLAINTEXT SEED LEAKED into file metadata (seed_smokescreen)") - return problems +def _theory_stack(): + from importlib.metadata import PackageNotFoundError, version + stack = {} + for package in ("pyccl", "camb", "smokescreen"): + try: + stack[package] = version(package) + except PackageNotFoundError: + stack[package] = None + return stack -def blind_init(blind_dir, config=None, label="smokescreen", log=print): - """Fix the blind for one catalogue version: seed, commitment, seed bundle. - Draws an OS-entropy seed (never written in plaintext, never returned), - writes the repo-committable ``commitment.json`` (the custody triple plus - the label), and encrypts the seed into a Fernet bundle. Every - :func:`blind_part` and :func:`unblind_part` call reads this fixed state. +def _declared_blinded(catalogues, registry, base): + """Refuse a base ``init`` or ``share`` cannot give a blind.""" + if _custody.base_catalogue(catalogues, base) != base: + raise _custody.CustodyError( + f"{base} is a variant of {_custody.base_catalogue(catalogues, base)}; " + "a blind covers base catalogues" + ) + declared = _custody.declaration(catalogues, base) + if declared in ("unblinded", "mock"): + raise _custody.CustodyError( + f"{base} is declared {declared}; declare it blinded first" + ) + covering = [r.name for r in _custody.records(registry).values() if base in r.bases] + if covering: + raise _custody.CustodyError(f"{base} is already covered by blind {covering[0]}") - Custody caveat: the bundle and its Fernet key land in the *same* - ``blind_dir``, and anyone with both can decrypt the seed. Keep the key - out-of-band; colocation is convenience, not at-rest protection. - Returns - ------- - dict - Paths written: ``commitment``, ``bundle``, ``key``. +def init(name, bases, *, cat_config, config=None): + """Draw blind ``name`` for ``bases`` and write its record; return its directory. + + @sc blind-drawn-once + The one place a seed is drawn. Existing state is refused, never replaced, + so no second draw exists for a catalogue unless a committed record is + deleted. The seed is held in memory and written only as ciphertext. """ + from cryptography.fernet import Fernet + config = config or BlindingConfig() - paths = init_paths(blind_dir) - for path in paths.values(): - if os.path.exists(path): - raise FileExistsError( - f"refusing to overwrite existing blind state {path} — a blind " - "is a one-shot custody event; choose another directory" + registry = _custody.registry_of(cat_config) + catalogues = _catalogues(cat_config) + record, staging = registry / name, registry / f".{name}.tmp" + if not name or name[0] in "-_." or set(name) - set(_BLIND_NAME): + raise _custody.CustodyError(f"blind name {name!r}: use a-z, 0-9, - _ .") + for path in (record, staging): + if path.exists(): + raise _custody.CustodyError( + f"{path} exists; a blind is drawn once (delete a staging " + "directory an interrupted init left)" ) + for base in bases: + _declared_blinded(catalogues, registry, base) seed = secrets.token_hex(16) + key = Fernet.generate_key() + payload = { + "blind": name, + "seed": seed, + "config_digest": config.digest(), + "draw_scheme": draw_scheme(), + } + ciphertext = Fernet(key).encrypt(json.dumps(payload).encode("utf-8")) + if json.loads(Fernet(key).decrypt(ciphertext)) != payload: + raise _custody.CustodyError("the sealed seed does not round-trip") commitment = { - "label": label, - "seed_commitment": seed_commitment(seed), - "config_digest": config.config_digest(), + "blind": name, + "seed_commitment": _custody.seed_commitment(seed), + "config": config.record(), + "config_digest": config.digest(), "draw_scheme": draw_scheme(), + "theory_stack": _theory_stack(), + "created": datetime.now(timezone.utc).isoformat(timespec="seconds"), } - with open(paths["commitment"], "w", encoding="utf-8") as f: - json.dump(commitment, f, indent=2, sort_keys=True) - _write_encrypted_json(paths["bundle"], {"label": label, "seed": seed}) - - log(f"[blind-init] commitment (repo-committable): {paths['commitment']}") - log(f"[blind-init] encrypted seed bundle + key: {paths['bundle']}, {paths['key']}") - log( - "[blind-init] custody: keep the bundle key out-of-band from the bundle " - "(colocation in the blind dir is not at-rest protection)" - ) - return paths - -def _read_seed(blind_dir, config): - """Decrypt the seed bundle and verify it against the commitment. + staging.mkdir(parents=True) + (staging / "commitment.json").write_text(json.dumps(commitment, indent=2) + "\n") + (staging / "seed.fernet").write_bytes(ciphertext) + (staging / "key").write_bytes(key) + (staging / "bases").write_text("".join(f"{b}\n" for b in bases)) + for part in ("commitment.json", "seed.fernet", "key"): + os.chmod(staging / part, 0o444) + os.replace(staging, record) + print(f"[blinding] drew blind {name} for {', '.join(bases)}") + print(f"[blinding] commit {record.relative_to(registry.parent.parent)}/") + return record + + +def share(name, base, *, cat_config): + """Add ``base`` to blind ``name``'s ``bases``.""" + registry = _custody.registry_of(cat_config) + records = _custody.records(registry) + if name not in records: + raise _custody.CustodyError(f"no blind {name} in {registry}") + if records[name].revealed is not None: + raise _custody.CustodyError(f"blind {name} is revealed; draw a new one") + _declared_blinded(_catalogues(cat_config), registry, base) + with open(registry / name / "bases", "a") as f: + f.write(f"{base}\n") + print(f"[blinding] {base} shares blind {name}; commit {registry / name / 'bases'}") + + +def _parts_under(root, commitment, skip): + """SACC files under ``root`` concealed under ``commitment``, found by content.""" + for path in sorted(Path(root).rglob("*.sacc")): + if skip in path.parents: + continue + try: + stamp = _custody.read_stamp(sacc_io.load(path).metadata) + except _custody.CustodyError: + continue # not born through the door, so not under any blind + if stamp.commitment == commitment: + yield path + + +def reveal(name, *, root, cat_config): + """Publish blind ``name``'s seed and move its concealed files aside. + + @sc reveal-is-reproduction + Revealing never subtracts a shift: the seed is published, every file + concealed under the blind moves to ``/revealed//``, the + declaration is flipped by a reviewed change, and the pipeline re-measures + true products that :func:`audit` compares against the archive. No file ever + mixes shifted and true signal. + """ + registry = _custody.registry_of(cat_config) + record, commitment = _read_record(registry, name) + blind = _open(registry, name) + revealed = record / "revealed.json" + if revealed.exists(): + if json.loads(revealed.read_text())["seed"] != blind.seed: + raise _custody.CustodyError(f"{revealed} records another seed") + else: + fd = os.open(revealed, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444) + with os.fdopen(fd, "w") as f: + json.dump({"seed": blind.seed}, f) + archive = Path(root) / "revealed" / name + moved = 0 + for path in _parts_under(root, commitment["seed_commitment"], archive): + target = archive / path.relative_to(root) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.move(path, target) + moved += 1 + print(f"[blinding] published the seed of {name}; moved {moved} files to {archive}") + print( + f"[blinding] commit {revealed} and declare `blinding: unblinded` on " + f"{', '.join(_custody.records(registry)[name].bases)}, then re-run" + ) + return archive - The whole custody triple is checked before the seed is handed to any - caller, whether it is about to blind or to unblind. - Returns - ------- - tuple - ``(seed, commitment_dict)``. - """ - paths = init_paths(blind_dir) - bundle = _read_encrypted_json(paths["bundle"], paths["key"]) - with open(paths["commitment"], encoding="utf-8") as f: - commitment = json.load(f) - _assert_draw_scheme(commitment.get("draw_scheme"), f"the blind in {blind_dir}") - if seed_commitment(bundle["seed"]) != commitment["seed_commitment"]: - raise ValueError( - "bundle seed does not match the committed seed commitment — refusing " - "to proceed" - ) - if config.config_digest() != commitment["config_digest"]: - raise ValueError( - "blinding config does not match the committed config digest — " - "refusing to proceed (a wrong envelope or P(k) recipe would " - "silently produce a wrong shift)" - ) - return bundle["seed"], commitment +# A re-measurement repeats a measurement to float noise (TreeCorr's threaded +# sums reorder); the audit compares its numbers to this relative tolerance. +_REMEASURED = 1e-10 -def blind_part(part_path, blind_dir, config=None, keep_input=False, log=print): - """Blind one intermediate part SACC at birth, under the fixed blind state. +def _same(x, y): + if isinstance(x, str) or isinstance(y, str): + return x == y + return bool(np.isclose(x, y, rtol=_REMEASURED, atol=0.0) or x == y) - Reads the fixed state :func:`blind_init` wrote, conceals the part - (:func:`blind_sacc`), writes the blinded part beside the input with the - true vector escrowed into a per-part Fernet bundle, and deletes the - plaintext — only the blinded part persists. Each escrow is self-contained, - so corruption of one bundle loses one part, not all. ``keep_input=True`` - retains the plaintext part (and its custody implication). - Returns - ------- - dict - Paths written: ``blinded``, ``escrow``, ``escrow_key``. - """ - config = config or BlindingConfig() - seed, commitment = _read_seed(blind_dir, config) - paths = part_paths(part_path) - for path in paths.values(): - if os.path.exists(path): - raise FileExistsError( - f"refusing to overwrite existing blind output {path} — a blind " - "is a one-shot custody event" - ) +def _rows_match(a, b): + if len(a.data) != len(b.data): + return False + for x, y in zip(a.data, b.data): + tx, ty = ({k: v for k, v in d.tags.items() if k != "window"} for d in (x, y)) + if (x.data_type, tuple(x.tracers), set(tx)) != ( + y.data_type, + tuple(y.tracers), + set(ty), + ) or not all(_same(tx[k], ty[k]) for k in tx): + return False + return True - # The plaintext part is real, unblinded data — the blinding step is the one - # legitimate reader of the true vector, so it passes the load escape hatch. - part = sacc_io.load(part_path, allow_unblinded=True) - blinded = blind_sacc(part, seed, config=config, label=commitment["label"], log=log) - - _write_encrypted_json( - paths["escrow"], - { - "label": commitment["label"], - "seed_commitment": commitment["seed_commitment"], - "true_mean": np.asarray(part.mean, dtype=float).tolist(), - }, - ) - # The blinded file inherits the part's provenance (data vs mock); it also - # carries concealed=True (stamped by blind_sacc), so it loads without the - # escape hatch. - sacc_io.save(blinded, paths["blinded"], type=part.metadata["type"]) - if not keep_input: - os.remove(part_path) - log(f"[blind-part] deleted plaintext part {part_path}") - else: - log(f"[blind-part] plaintext part RETAINED at {part_path} (keep_input=True)") - log(f"[blind-part] wrote {paths['blinded']} (escrow beside it)") - return paths +def _covariance(s): + return None if s.covariance is None else np.asarray(s.covariance.dense) -def unblind_part(blinded_path, blind_dir, out_path, config=None, log=print): - """Unblind one blinded part (or the assembled file), verifying first. - Verifies the custody triple against ``commitment.json`` and against the - file's own stamps, then subtracts the seed-recomputed shift - (:func:`unblind_sacc`). The seed-subtracted vector is the authority. +def _same_covariance(a, b): + if a is None or b is None: + return a is None and b is None + scale = np.max(np.abs(b)) + return a.shape == b.shape and np.allclose(a, b, rtol=0.0, atol=_REMEASURED * scale) - A part's escrow bundle, when present beside the blinded file and only once - its stored ``seed_commitment`` is confirmed to be this blind's, plays two - subordinate roles: a tighter equality check (disagreement beyond ``1e-6`` - relative fails closed) and removal of the ~ulp residue float - add-then-subtract leaves, making the restore bit-for-bit. It is never the - source of correctness. The assembled file has no escrow and the - subtraction stands alone. - """ - config = config or BlindingConfig() - seed, commitment = _read_seed(blind_dir, config) - blinded = sacc_io.load(blinded_path) - part = unblind_sacc(blinded, seed, config=config, log=log) - stem, ext = os.path.splitext(blinded_path) - unblinded_stem = os.path.join( - os.path.dirname(stem), os.path.basename(stem).replace("_blinded", "") - ) - escrow = part_paths(unblinded_stem + ext) - if os.path.exists(escrow["escrow"]): - bundle = _read_encrypted_json(escrow["escrow"], escrow["escrow_key"]) - if bundle.get("seed_commitment") != commitment["seed_commitment"]: - raise ValueError( - "escrow bundle beside the blinded file was written under a " - "different seed than the commitment — refusing to trust it " - "(the seed subtraction is authoritative; this escrow is not " - "bound to this blind)" - ) - true_mean = np.asarray(bundle["true_mean"], dtype=float) - recovered = np.asarray(part.mean, dtype=float) - residual = np.nanmax( - np.abs(recovered - true_mean) / (np.abs(true_mean) + 1e-30) +def _audit_part(blinded, true, fiducial, hidden, tolerance): + """Problems with one archived part against its re-measured twin.""" + stamp, true_stamp = (_custody.read_stamp(x.metadata) for x in (blinded, true)) + if true_stamp.token != f"unblinded:{stamp.catalogue}": + return [f"the live file is stamped {true_stamp.token}"], None + problems = [] + if not _rows_match(blinded, true): + problems.append("rows, tags or tracers differ") + return problems, None + if set(blinded.tracers) != set(true.tracers): + problems.append("tracers differ") + cov_b, cov_t = _covariance(blinded), _covariance(true) + if not _same_covariance(cov_b, cov_t): + problems.append("covariances differ") + centres = [x.metadata.get("patch_centers_sha256") for x in (blinded, true)] + if centres[0] != centres[1]: + problems.append(f"patch centres differ: {centres}") + + delta = np.asarray(blinded.mean) - np.asarray(true.mean) + residual, shifted = 0.0, np.zeros(len(delta), bool) + for block, factor in factors(true, fiducial, hidden): + scale = np.max(np.abs(factor)) + residual = max(residual, np.max(np.abs(delta[block.rows] - factor)) / scale) + shifted[block.rows] = True + if residual > tolerance: + problems.append(f"blinded − true ≠ shift(seed): residual {residual:.2e}") + + sigma = None if cov_t is None else np.sqrt(np.diag(cov_t)) + e_shift = {} + for i, dp in enumerate(true.data): + if shifted[i]: + continue + kind = dp.data_type + b_mode = kind in ( + sacc_io.COSEBI_BB, + *[sacc_io.PURE_TYPES[k] for k in ("xip_B", "xim_B")], ) - if residual > 1e-6: - raise ValueError( - f"unblinded vector disagrees with the escrowed true vector " - f"(max rel {residual:.2e}) — wrong escrow for this part?" + e_mode = kind in (sacc_io.COSEBI_EE, *sacc_io.PURE_TYPES.values()) + if b_mode: + if sigma is None or abs(delta[i]) > 1e-2 * sigma[i]: + problems.append(f"B row {i} ({kind}) moved by {delta[i]:.3e}") + elif e_mode: + if sigma is not None: + e_shift[kind] = max(e_shift.get(kind, 0.0), abs(delta[i]) / sigma[i]) + elif abs(delta[i]) > 1e-8 * max(abs(dp.value), 1e-300): + problems.append( + f"row {i} ({kind}) moved, but the blind leaves it unshifted" ) - # Seed-bound escrow: clear the add-then-subtract ulp residue. - _set_values(part, range(len(true_mean)), true_mean) - log(f"[unblind] escrow verified (subtraction residual {residual:.2e})") - # unblind_sacc stripped the concealed/blind stamps, so this is the true - # revealed vector; it inherits the blinded file's provenance (data vs mock). - sacc_io.save(part, out_path, type=blinded.metadata["type"]) - log(f"[unblind] wrote {out_path}") - return out_path - - -def _write_encrypted_json(encrpt_path, payload): - """Encrypt ``payload`` (JSON) to ``encrpt_path`` + sibling ``.key``. - - Smokescreen's ``save_file`` mode names outputs from - ``basename.split('.')[0]``, which truncates our dotted catalogue-version - stems (``v1.4.6.3_…`` → ``v1.encrpt``) and collides across parts, so we - take the returned ``(ciphertext, key)`` and write them ourselves. + return problems, {"residual": residual, "e_shift_over_sigma": e_shift} + + +def audit(name, *, archive, true_root, cat_config, out=None): + """Check every archived file of blind ``name`` against its re-measured twin. + + Needs no key: the seed is the published one, checked against the + commitment. For each archived file, the live file at the same relative path + must be stamped unblinded for the same catalogue, with the same rows, tags, + tracers, covariance and patch centres (numbers to a re-measurement's float + noise), and blinded − true must equal the seed's shift on every ξ± and + Cℓ_EE block to 1e-6 of the block's largest shift (1e-3 when the theory + stack differs from the one the blind was drawn with); COSEBIs and pure-E/B + B rows may move by at most 1e-2 σ, and rows the blind leaves unshifted not + at all. """ - from smokescreen.encryption import encrypt_file + registry = _custody.registry_of(cat_config) + record, commitment = _read_record(registry, name) + report = {"blind": name, "ok": False, "problems": [], "parts": {}} + revealed = record / "revealed.json" + seed = json.loads(revealed.read_text())["seed"] if revealed.exists() else None + if seed is None or _custody.seed_commitment(seed) != commitment["seed_commitment"]: + report["problems"].append("the published seed does not match the commitment") + return _write_report(report, out) + config = BlindingConfig.from_record(commitment["config"]) + if config.digest() != commitment["config_digest"]: + report["problems"].append("the recorded config does not match its digest") + if int(commitment["draw_scheme"]) != draw_scheme(): + report["problems"].append("this install draws under another scheme") + stack = _theory_stack() + tolerance = 1e-6 + if stack != commitment.get("theory_stack"): + tolerance = 1e-3 + report["theory_stack"] = { + "record": commitment.get("theory_stack"), + "now": stack, + } + hidden = hidden_theory(seed, config) + report["shift"] = { + key: getattr(hidden, key) - getattr(config.theory, key) + for key in config.envelope + } + archive, true_root = Path(archive), Path(true_root) + for path in sorted(archive.rglob("*.sacc")): + relative = str(path.relative_to(archive)) + live = true_root / relative + if not live.exists(): + report["parts"][relative] = {"problems": ["no re-measured file"]} + continue + blinded = sacc_io.load(path) + stamp = _custody.read_stamp(blinded.metadata) + if stamp.commitment != commitment["seed_commitment"]: + report["parts"][relative] = {"problems": ["not concealed under this blind"]} + continue + problems, numbers = _audit_part( + blinded, sacc_io.load(live), config.theory, hidden, tolerance + ) + report["parts"][relative] = {"problems": problems, **(numbers or {})} + report["ok"] = ( + bool(report["parts"]) + and not report["problems"] + and not any(part["problems"] for part in report["parts"].values()) + ) + return _write_report(report, out) + + +def _write_report(report, out): + if out is not None: + Path(out).write_text(json.dumps(report, indent=2, default=float) + "\n") + return report - key_path = encrpt_path.replace(".encrpt", ".key") - plaintext = encrpt_path.replace(".encrpt", ".json") - with open(plaintext, "w", encoding="utf-8") as f: - json.dump(payload, f) - ciphertext, key = encrypt_file(plaintext, save_file=False, keep_original=False) - with open(encrpt_path, "wb") as f: - f.write(ciphertext) - with open(key_path, "wb") as f: - f.write(key) +def verify(path, *, cat_config): + """Problems with a file's stamp against the registry and declaration (seedless).""" + stamp = _custody.read_stamp(sacc_io.load(path).metadata) + registry = _custody.registry_of(cat_config) + problems = [] + if stamp.status == "blinded": + record = _custody.records(registry).get(stamp.blind) + if record is None: + return [f"no blind {stamp.blind} in {registry}"] + c = record.commitment + for what, stamped, recorded in ( + ("seed commitment", stamp.commitment, c["seed_commitment"]), + ("config digest", stamp.config_digest, c["config_digest"]), + ("draw scheme", stamp.draw_scheme, int(c["draw_scheme"])), + ): + if stamped != recorded: + problems.append(f"the {what} differs from the record") + if stamp.draw_scheme != draw_scheme(): + problems.append("this install draws under another scheme") + if stamp.catalogue not in record.bases: + problems.append(f"blind {record.name} does not cover {stamp.catalogue}") + try: + declared = _custody.custody_of( + _catalogues(cat_config), stamp.catalogue, registry=registry + ) + if declared.stamp != stamp.stamp: + problems.append(f"{stamp.catalogue} is declared {declared.token}") + except _custody.CustodyError as err: + problems.append(str(err)) + return problems -def _read_encrypted_json(encrpt_path, key_path): - """Decrypt and parse a Fernet-encrypted JSON bundle.""" - from smokescreen.encryption import decrypt_file - return json.loads(decrypt_file(encrpt_path, key_path).decode("utf-8")) +def main(argv=None): + parser = argparse.ArgumentParser( + prog="python -m sp_validation.blinding", description=__doc__.split("\n")[0] + ) + commands = parser.add_subparsers(dest="command", required=True) + for command, arguments in { + "init": ("blind", "bases+"), + "share": ("blind", "base"), + "reveal": ("blind",), + "audit": ("blind",), + "verify": ("file",), + }.items(): + sub = commands.add_parser(command) + for argument in arguments: + name, plus = argument.rstrip("+"), argument.endswith("+") + sub.add_argument(name, nargs="+" if plus else None) + sub.add_argument("--cat-config", required=True) + commands.choices["init"].add_argument( + "--config", help="JSON record of BlindingConfig fields (defaults otherwise)" + ) + commands.choices["reveal"].add_argument("--root", required=True) + commands.choices["audit"].add_argument("--archive", required=True) + commands.choices["audit"].add_argument("--true-root", required=True) + commands.choices["audit"].add_argument("--out") + a = parser.parse_args(argv) + + if a.command == "init": + config = ( + BlindingConfig.from_record(json.loads(Path(a.config).read_text())) + if a.config + else None + ) + init(a.blind, a.bases, cat_config=a.cat_config, config=config) + elif a.command == "share": + share(a.blind, a.base, cat_config=a.cat_config) + elif a.command == "reveal": + reveal(a.blind, root=a.root, cat_config=a.cat_config) + elif a.command == "audit": + report = audit( + a.blind, + archive=a.archive, + true_root=a.true_root, + cat_config=a.cat_config, + out=a.out, + ) + print(json.dumps(report, indent=2, default=float)) + return 0 if report["ok"] else 1 + elif a.command == "verify": + problems = verify(a.file, cat_config=a.cat_config) + for problem in problems: + print(f"[verify] {problem}") + print(f"[verify] {a.file}: {'ok' if not problems else 'FAILED'}") + return 1 if problems else 0 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/sp_validation/blinding_paths.py b/src/sp_validation/blinding_paths.py deleted file mode 100644 index 42ddb415..00000000 --- a/src/sp_validation/blinding_paths.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Blinding file-name conventions, with no dependencies. - -Separate from :mod:`sp_validation.blinding` so the Snakemake DAG build can -import the path layout without pulling in numpy, CCL and smokescreen. -""" - -import os - - -def init_paths(blind_dir): - """The fixed custody state ``blinding.blind_init`` writes in ``blind_dir``.""" - return { - "commitment": os.path.join(blind_dir, "commitment.json"), - "bundle": os.path.join(blind_dir, "blind_seed.encrpt"), - "key": os.path.join(blind_dir, "blind_seed.key"), - } - - -def part_paths(part_path): - """Blinded-output and escrow-bundle paths beside a part file.""" - stem, ext = os.path.splitext(str(part_path)) - return { - "blinded": f"{stem}_blinded{ext or '.fits'}", - "escrow": f"{stem}_escrow.encrpt", - "escrow_key": f"{stem}_escrow.key", - } diff --git a/src/sp_validation/blinding_theory.py b/src/sp_validation/blinding_theory.py index de758aa9..bb46e4c8 100644 --- a/src/sp_validation/blinding_theory.py +++ b/src/sp_validation/blinding_theory.py @@ -1,73 +1,31 @@ -"""Blinding theory: fiducial configuration and the two ξ± theory paths. +"""Blinding theory: the fiducial configuration and CCL's shear two-point prediction. :Name: blinding_theory.py -:Description: The blinding backend's theory surface — the fiducial - configuration (:class:`TheoryConfig`) and two independent routes to the - tomographic shear two-point prediction. +:Description: :class:`TheoryConfig` is the fiducial cosmology and model + recipe; :func:`xi_ccl` and :func:`cl_ee` are the tomographic shear ξ± and + Cℓ_EE between two bins' n(z). CCL builds the nonlinear P(k) through its + Boltzmann-CAMB HMCode2020 route and projects with its own Limber + (``angular_cl``) and FFTLog (``correlation``). ``test_camb_ccl_crosscheck`` + compares this path with a direct CAMB run. The generic cosmology machinery here is destined for ``cs_util.cosmo`` (cs_util#80). - Two independent routes to the shear two-point prediction: - - - **CCL-native path** (:func:`xi_ccl`, :func:`cl_ee`): CCL builds the - nonlinear P(k) through its Boltzmann-CAMB HMCode2020 route - (``matter_power_spectrum='camb'`` + ``extra_parameters``) and projects - to Cℓ/ξ± via its own Limber (``angular_cl``) + FFTLog - (``correlation``). This is the recipe the blinding theory backends use. - - **Independent-CAMB path** (:func:`xi_camb`): a direct ``pycamb`` run - produces the HMCode2020 ``P(k, z)`` (σ8-matched via the closed-form - A_s rescale of :func:`camb_As_for_sigma8`), wrapped in a ``ccl.Pk2D`` - and projected through the same CCL Limber + FFTLog machinery. - - Because both paths route their nonlinear P(k) through CAMB's HMCode2020 - and both project through CCL, a common Limber+FFTLog bug cancels between - them: the CAMB↔CCL cross-check test built on these two paths validates - the **P(k) recipe** and the **σ8/A_s amplitude convention**, not the - projection machinery. - - This module imports only ``numpy`` at module level; CCL and CAMB are - imported inside the functions that need them, so importing - :class:`TheoryConfig` never drags in a theory backend. + Only ``numpy`` is imported at module level; CCL is imported inside the + functions that need it, so importing :class:`TheoryConfig` never drags in a + theory backend. """ import dataclasses import numpy as np -# Fixed constants of the fiducial — load-bearing for the CAMB↔CCL amplitude -# match, so they are emitted explicitly to both stacks rather than left to -# either stack's default. Not user-facing TheoryConfig fields. +# Fixed constants of the fiducial, passed explicitly to CCL (and to the CAMB +# oracle in the tests) rather than left to either stack's default. NEFF = 3.046 T_CMB = 2.7255 -# Matter-power redshift grid shared by `make_camb_params` and `xi_camb`, so the -# CAMB run and the Pk2D built from it sample the same redshifts. -PK_ZMAX = 3.0 -PK_NZ = 48 - - -def coerce_fields(cls, overrides): - """Validate ``overrides`` against ``cls``'s fields, coercing floats. - - Unknown keys raise. Every ``float``-declared field goes through - :func:`float`, so a YAML/CLI ``w0: -1`` (int) yields the same value — and - the same config digest — as the float default ``-1.0``. Digest stability - depends on this, so it is one helper rather than three copies. - """ - by_name = {f.name: f for f in dataclasses.fields(cls)} - unknown = set(overrides) - set(by_name) - if unknown: - raise ValueError( - f"unknown {cls.__name__} fields {sorted(unknown)}; " - f"valid fields are {sorted(by_name)}" - ) - return { - name: (float(v) if by_name[name].type in (float, "float") else v) - for name, v in overrides.items() - } - # --------------------------------------------------------------------------- # # Configuration surface — the ONE place fiducial cosmology + model choices live @@ -87,9 +45,8 @@ class TheoryConfig: converted to CCL's native ``sigma8``/``Omega_c`` by :meth:`sigma8` / :meth:`omega_c`. - One nonlinear recipe is named by two tokens (``ccl_halofit_version``, - ``camb_halofit_version``) because CCL and CAMB could name it differently; - each stack is fed its own so a rename cannot silently split the recipe. + ``halofit_version`` names the nonlinear recipe CAMB runs, whether CCL + calls it or CosmoSIS does. """ # Cosmological parameters (blind axes S8, Omega_m + the rest). @@ -111,9 +68,8 @@ class TheoryConfig: # deliberate cross-check tool rather than a production setting. transfer_function: str = "boltzmann_camb" - # CAMB HMCode2020 + baryonic feedback — see the class docstring. - ccl_halofit_version: str = "mead2020_feedback" - camb_halofit_version: str = "mead2020_feedback" + # CAMB HMCode2020 + baryonic feedback. + halofit_version: str = "mead2020_feedback" hmcode_logT_AGN: float = 7.5 # values_ia.ini logT_AGN central # Intrinsic alignments: NLA. The fiducial defaults IA OFF (ia_bias=0) — @@ -145,14 +101,11 @@ def omega_c(self): return self.Omega_m - self.Omega_b - omega_nu def ccl_params(self): - """The fiducial point as a plain CCL-native parameter mapping. + """This point as a plain CCL-native parameter mapping. Exactly the keys ``Omega_c, Omega_b, h, n_s, sigma8, m_nu, - mass_split, w0, wa, Neff, T_CMB`` and no others — no CCL default - rides along. ``Neff``/``T_CMB`` are the fixed module constants. This - mapping is what the Smokescreen fork receives as ``fiducial_params`` - and what every ``theory_fn`` receives back (possibly with - ``sigma8``/``Omega_c`` overlaid by the hidden draw). + mass_split, w0, wa, Neff, T_CMB`` and no others, so no CCL default + rides along; ``Neff``/``T_CMB`` are the fixed module constants. """ return { "Omega_c": self.omega_c(), @@ -168,18 +121,13 @@ def ccl_params(self): "T_CMB": T_CMB, } - @classmethod - def from_overrides(cls, overrides): - """Build from a mapping of field overrides (fail loud on unknown keys).""" - return cls(**coerce_fields(cls, overrides)) - # --------------------------------------------------------------------------- # -# CCL-native path: cosmology construction, Cℓ_EE, ξ± +# CCL: cosmology construction, Cℓ_EE, ξ± # --------------------------------------------------------------------------- # -# The two cosmologies of a blind (fiducial + hidden) are evaluated by three -# theory backends over multiple blocks; caching the ccl.Cosmology per parameter -# point avoids re-running the CAMB P(k) computation for every block. +# The two cosmologies of a blind (fiducial and hidden) are evaluated for every +# block of a SACC; caching the ccl.Cosmology per parameter point avoids +# re-running the CAMB P(k) computation for each block. _COSMO_CACHE = {} @@ -198,7 +146,7 @@ def ccl_cosmology(params, config): key = ( tuple(sorted(params.items())), config.transfer_function, - config.ccl_halofit_version, + config.halofit_version, config.hmcode_logT_AGN, ) if key not in _COSMO_CACHE: @@ -207,7 +155,7 @@ def ccl_cosmology(params, config): "matter_power_spectrum": "camb", "extra_parameters": { "camb": { - "halofit_version": config.ccl_halofit_version, + "halofit_version": config.halofit_version, "HMCode_logT_AGN": config.hmcode_logT_AGN, } }, @@ -293,7 +241,7 @@ def cl_ee(params, config, nz_i, nz_j, ell): def xi_ccl(params, config, nz_i, nz_j, theta_arcmin, ell=None): - """CCL-native ξ± at ``theta_arcmin`` for one bin pair (Path A). + """ξ± at ``theta_arcmin`` for one bin pair. Cross Cℓ_EE on :func:`xi_ell_grid` (or ``ell``), then ``ccl.correlation`` (FFTLog Hankel transform) at θ in degrees, ``type="GG+"`` / ``"GG-"``. @@ -312,109 +260,3 @@ def xi_ccl(params, config, nz_i, nz_j, theta_arcmin, ell=None): xip = ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta_deg, type="GG+") xim = ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta_deg, type="GG-") return xip, xim - - -# --------------------------------------------------------------------------- # -# Independent-CAMB path: A_s reconciliation + P(k) → Pk2D → CCL projection -# --------------------------------------------------------------------------- # -def make_camb_params(config, As, *, nonlinear, zmax=PK_ZMAX, n_z=PK_NZ, kmax=20.0): - """A ``CAMBparams`` at ``config``'s background with amplitude ``As``. - - Every :class:`TheoryConfig` field CCL sees is fed to CAMB from the same - source — ``w0``/``wa`` via ``set_dark_energy``, ``Neff``/``T_CMB`` as the - module constants, ``m_nu``/``mass_split`` through ``set_cosmology`` — so - the independent path differs from the CCL path only in who computes P(k), - never in an unmatched background parameter. - """ - import camb - - p = camb.CAMBparams() - p.set_cosmology( - H0=config.h * 100, - ombh2=config.Omega_b * config.h**2, - omch2=config.omega_c() * config.h**2, - mnu=config.m_nu, - num_massive_neutrinos=1, - neutrino_hierarchy=config.mass_split, - nnu=NEFF, - TCMB=T_CMB, - ) - p.set_dark_energy(w=config.w0, wa=config.wa, dark_energy_model="ppf") - p.InitPower.set_params(As=As, ns=config.n_s) - p.set_matter_power(redshifts=list(np.linspace(0.0, zmax, n_z)), kmax=kmax) - if nonlinear: - p.NonLinear = camb.model.NonLinear_both - p.NonLinearModel.set_params( - halofit_version=config.camb_halofit_version, - HMCode_logT_AGN=config.hmcode_logT_AGN, - ) - else: - p.NonLinear = camb.model.NonLinear_none - return p - - -def camb_linear_sigma8(config, As, **kwargs): - """CAMB's linear σ8(z=0) at amplitude ``As``.""" - import camb - - results = camb.get_results(make_camb_params(config, As, nonlinear=False, **kwargs)) - return float(results.get_sigma8_0()) - - -def camb_As_for_sigma8(config, sigma8_target, As_seed=2.1e-9, **kwargs): - """The CAMB ``A_s`` whose linear σ8 equals ``sigma8_target``. - - Closed-form: linear σ8² ∝ A_s exactly, so one CAMB linear-σ8 evaluation - at ``As_seed`` and one rescale ``As_seed · (σ8_target/σ8_seed)²`` land on - the target — no iteration. This settles the convention subtlety that our - fiducial fixes σ8 for CCL but A_s for CAMB: a nominal ``A_s = 2.1e-9`` - leaves CAMB's σ8 ≈3% off target, enough to blow a ξ± comparison to - ~9–10%. - """ - sigma8_seed = camb_linear_sigma8(config, As_seed, **kwargs) - return As_seed * (sigma8_target / sigma8_seed) ** 2 - - -def xi_camb(config, nz, theta_arcmin, *, n_ell=300, ell_max=60000, kmax=20.0, n_k=400): - """Independent-CAMB ξ± for one bin (Path B): CAMB P(k) → Pk2D → CCL. - - A direct pycamb run produces the HMCode2020 nonlinear ``P(k, z)`` at a - σ8-matched ``A_s`` (:func:`camb_As_for_sigma8`), wrapped in a ``Pk2D`` and - projected by CCL's Limber + FFTLog with a bare tracer (IA off — this path - exists for the cross-check). ``hubble_units=False, k_hunit=False`` already - returns CCL's native units (k in 1/Mpc, P in Mpc³), so applying an - ``·h``/``/h³`` conversion here would double-count an h³ amplitude error. - - Returns - ------- - (np.ndarray, np.ndarray, float) - ``(xip, xim, As)`` — the σ8-matched amplitude is returned for - assertion by the cross-check test. - """ - import camb - import pyccl as ccl - - sigma8 = config.sigma8() - As = camb_As_for_sigma8(config, sigma8, kmax=kmax) - results = camb.get_results(make_camb_params(config, As, nonlinear=True, kmax=kmax)) - interp = results.get_matter_power_interpolator( - nonlinear=True, hubble_units=False, k_hunit=False - ) - k = np.geomspace(1e-4, kmax * config.h, n_k) # 1/Mpc - z = np.linspace(0.0, PK_ZMAX, PK_NZ) # the grid make_camb_params computed - pk = interp.P(z, k) # (n_z, n_k), Mpc^3 - a = 1.0 / (1.0 + z) - order = np.argsort(a) # Pk2D wants ascending scale factor - pk2d = ccl.Pk2D( - a_arr=a[order], lk_arr=np.log(k), pk_arr=np.log(pk[order]), is_logp=True - ) - - cosmo = ccl_cosmology(config.ccl_params(), config) - z_nz, nz_vals = nz - lens = ccl.WeakLensingTracer(cosmo, dndz=(np.asarray(z_nz), np.asarray(nz_vals))) - ells = np.unique(np.geomspace(2, ell_max, n_ell).astype(int)).astype(float) - cl = ccl.angular_cl(cosmo, lens, lens, ells, p_of_k_a=pk2d) - theta_deg = np.asarray(theta_arcmin) / 60.0 - xip = ccl.correlation(cosmo, ell=ells, C_ell=cl, theta=theta_deg, type="GG+") - xim = ccl.correlation(cosmo, ell=ells, C_ell=cl, theta=theta_deg, type="GG-") - return xip, xim, As diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 1277133f..3f9c23ce 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -14,7 +14,7 @@ _get_pte_from_scale_cut, find_conservative_scale_cut_key, ) -from ..blinding_paths import init_paths +from ..custody import custody_of, registry_of from ..statistics import chi2_and_pte from ..version import __version__ from .catalog_characterization import CatalogCharacterizationMixin @@ -132,16 +132,6 @@ class CosmologyValidation( noise debiasing, making those realizations reproducible run-to-run. cosmo_params : dict, optional Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. - run_type : {'data', 'mock'}, default 'data' - The campaign's run type, stamped as the SACC ``type`` of every part this - object writes. Custody state, not decoration: a mock campaign must be - built with ``run_type='mock'`` for its parts to assemble at all (see - ``blinding.assert_consistent_blind``). - blind_root : str, optional - Directory holding one ``blind_init`` state directory per catalogue - version. Given, the part writers stamp born-blinded and blind-irrelevant - parts under the version's blind (see :meth:`commitment_path`); ``None`` - (mock runs) leaves them plaintext. Attributes ---------- @@ -268,8 +258,6 @@ def __init__( path_onecovariance=None, cosmo_params=None, blind=None, - run_type="data", - blind_root=None, ): self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method @@ -299,8 +287,6 @@ def __init__( self.nside_mask = nside_mask self.path_onecovariance = path_onecovariance self.blind = blind - self.run_type = run_type - self.blind_root = blind_root assert self.cell_method in ["map", "catalog"], ( "cell_method must be 'map' or 'catalog'" @@ -337,6 +323,8 @@ def __init__( self.catalog_config_path = Path(catalog_config) with self.catalog_config_path.open("r") as file: self.cc = cc = yaml.load(file, Loader=yaml.FullLoader) + # The catalogues as declared, before virtual versions are materialised. + self._declared = copy.deepcopy(cc) def resolve_paths_for_version(ver): """Resolve relative paths for a version using its subdir.""" @@ -534,16 +522,16 @@ def results_objectwise(self): self._results_objectwise = self.init_results(objectwise=True) return self._results_objectwise - def commitment_path(self, version): - """The version's ``commitment.json``, or ``None`` when not blinding. + def custody(self, version): + """The custody ``version``'s catalogue is declared under. - Resolved per version rather than held as one path, because a single - ``CosmologyValidation`` can span several catalogue versions and each - has its own blind. + Read from the catalogue config this object was built from and the blind + registry beside it (:func:`sp_validation.custody.custody_of`); every + SACC this object writes for ``version`` is sealed under it. """ - if self.blind_root is None: - return None - return init_paths(os.path.join(self.blind_root, version))["commitment"] + return custody_of( + self._declared, version, registry=registry_of(self.catalog_config_path) + ) def basename(self, version, treecorr_config=None, npatch=None): cfg = treecorr_config or self.treecorr_config diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 8243f818..c3593b71 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -70,6 +70,12 @@ def plot_pseudo_cl_spectrum(datasets, spectrum, output_path): plt.close(fig) +def _spectra(part): + """The ELL/EE/EB/BB dict of a pseudo-Cl part.""" + ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(part, SACC_BIN) + return {"ELL": ell, "EE": ee, "EB": eb, "BB": bb} + + class PseudoClMixin: @property def pseudo_cls(self): @@ -520,14 +526,7 @@ def calculate_pseudo_cl(self, out_path=None): self._pseudo_cls[ver] = {} ver_out_path = out_path or self._output_path(f"pseudo_cl_{ver}.sacc") - if os.path.exists(ver_out_path): - self.print_done( - f"Skipping Pseudo-Cl's calculation, {ver_out_path} exists" - ) - self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc( - ver_out_path - ) - elif self.cell_method == "map": + if self.cell_method == "map": self.calculate_pseudo_cl_map(ver, nside, ver_out_path) elif self.cell_method == "catalog": self.calculate_pseudo_cl_catalog(ver, ver_out_path) @@ -536,15 +535,6 @@ def calculate_pseudo_cl(self, out_path=None): self.print_done("Done pseudo-Cl's") - @staticmethod - def _load_pseudo_cl_sacc(out_path): - """Read a pseudo-Cl SACC part into the ELL/EE/EB/BB dict.""" - # Readback of a part this producer just wrote — a legitimate pre-blind - # consumer, so the fail-closed load is opted out of. - s = sacc_io.load(out_path, allow_unblinded=True) - ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(s, SACC_BIN) - return {"ELL": ell, "EE": ee, "EB": eb, "BB": bb} - def calculate_pseudo_cl_map(self, ver, nside, out_path): params = get_params_rho_tau(self.cc[ver], survey=ver) @@ -610,9 +600,8 @@ def calculate_pseudo_cl_map(self, ver, nside, out_path): cl_shear = cl_shear - cl_noise self.print_cyan("Saving pseudo-Cl's...") - self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) - - self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) + sealed = self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) + self._pseudo_cls[ver]["pseudo_cl"] = _spectra(sealed) def calculate_pseudo_cl_catalog(self, ver, out_path): params = get_params_rho_tau(self.cc[ver], survey=ver) @@ -625,9 +614,8 @@ def calculate_pseudo_cl_catalog(self, ver, out_path): ) self.print_cyan("Saving pseudo-Cl's...") - self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) - - self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) + sealed = self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) + self._pseudo_cls[ver]["pseudo_cl"] = _spectra(sealed) def get_n_gal_map(self, params, nside, cat_gal): """Weighted galaxy number-density map (thin wrapper -> primitive).""" @@ -721,7 +709,8 @@ def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); the writer takes the shared bandpower window from ``wsp``. No covariance - is attached here. + is attached here. Returns the part as sealed under the version's + custody. """ s = pseudo_cl_to_sacc( self.sacc_nz(version), @@ -730,7 +719,7 @@ def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): cl_all, wsp, ) - sacc_io.save(s, out_path, type=self.run_type) + return sacc_io.save(s, out_path, custody=self.custody(version)) def plot_pseudo_cl(self): """Plot the EE/EB/BB pseudo-Cl spectra for every version.""" diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index f9f6b817..12652e81 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -55,11 +55,7 @@ def calculate_rho_tau_stats(self): def rho_tau_to_sacc_part( self, version, out_dir, base, rho_stat_handler, tau_stat_handler ): - """Write the ρ/τ SACC part for one version. - - ρ/τ carries no cosmological vector and is never blinded; on a data run - it is stamped concealed pass-through (values untouched) so the - assembly's load gate admits it. + """Write the ρ/τ SACC part for one version, under the version's custody. ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage from the handler tables. The ``CovTauTh`` theory covariance ``cov_tau_{base}_th.npy`` is passed as @@ -82,9 +78,7 @@ def rho_tau_to_sacc_part( tau_cov_th=tau_cov_th, ) out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") - sacc_io.save( - s, out_path, type=self.run_type, commitment=self.commitment_path(version) - ) + sacc_io.save(s, out_path, custody=self.custody(version)) @property def rho_stat_handler(self): diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index c410ef26..aef72b33 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -8,7 +8,7 @@ single ``{version}.sacc`` analysis file from these parts. The integration-grid ξ± part is an intermediate consumed by COSEBIs and -pure-E/B; it is blinded at birth but does not join ``{version}.sacc``. +pure-E/B; it does not join ``{version}.sacc``. Everything here is single-bin today (``bins=(0, 0)``); the interface is tomography-native so a future round supplies real bin pairs unchanged. diff --git a/src/sp_validation/custody.py b/src/sp_validation/custody.py new file mode 100644 index 00000000..8a3dcb6e --- /dev/null +++ b/src/sp_validation/custody.py @@ -0,0 +1,324 @@ +"""Custody: whether a catalogue's signal is blinded, unblinded or a mock. + +Custody is declared once per *base* catalogue, as ``blinding:`` on its entry in +``cosmo_val/cat_config.yaml``; an entry that declares nothing is blinded, so a +new catalogue fails closed. Variants share their base's custody and blind: +``_leak_corr`` and ``_seed`` versions, and entries that name their parent +with ``base:``. + +A blinded base must be covered by exactly one blind in the registry beside the +catalogue config (``cosmo_val/blinds//``: ``commitment.json``, the +``bases`` it covers, and ``revealed.json`` once its seed is published). The +registry is written only by ``python -m sp_validation.blinding``. + +Standard library only: the host Snakemake loads this file by path, and +container jobs import it, so both resolve custody from the same declaration. +""" + +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import Path + +STATUSES = ("blinded", "unblinded", "mock") + +# Top-level keys of the catalogue config that are not catalogues. +NOT_CATALOGUES = ("nz", "paths") + +# The Smokescreen fork's commitment prefix; test_blinding pins it to the fork's. +COMMITMENT_DOMAIN = b"smokescreen-seed-commitment-v1|" + +# The stamp every SACC carries: the custody it was born under. +STAMP_KEYS = ( + "blinding", + "blinding_catalogue", + "blinding_blind", + "blinding_commitment", + "blinding_config", + "blinding_scheme", +) +_BLIND_KEYS = STAMP_KEYS[2:] + +_SEED_SUFFIX = re.compile(r"_seed\d+$") +_LEAK_SUFFIX = "_leak_corr" + + +class CustodyError(ValueError): + """A catalogue's custody cannot be resolved, or a stamp is not one.""" + + +@dataclass(frozen=True) +class Custody: + """The custody of one base catalogue, and of every variant of it.""" + + status: str + catalogue: str + blind: str | None = None + commitment: str | None = None + config_digest: str | None = None + draw_scheme: int | None = None + registry: Path | None = None + + @property + def token(self): + """One string that changes whenever the custody does.""" + if self.status == "blinded": + return f"blinded:{self.catalogue}:{self.blind}:{self.commitment}" + return f"{self.status}:{self.catalogue}" + + @property + def stamp(self): + """The ``blinding*`` metadata a SACC born under this custody carries.""" + stamp = {"blinding": self.status, "blinding_catalogue": self.catalogue} + if self.status == "blinded": + stamp.update( + { + "blinding_blind": self.blind, + "blinding_commitment": self.commitment, + "blinding_config": self.config_digest, + "blinding_scheme": int(self.draw_scheme), + } + ) + return stamp + + +def seed_commitment(seed): + """The public commitment to a seed: sha256 of the domain-prefixed seed.""" + return hashlib.sha256(COMMITMENT_DOMAIN + str(seed).encode("utf-8")).hexdigest() + + +def registry_of(cat_config): + """The blind registry beside a catalogue config.""" + return Path(cat_config).resolve().parent / "blinds" + + +def entry_of(version): + """The catalogue-config entry describing ``version``'s data. + + Strips the variants CosmologyValidation materialises from an entry: + ``_leak_corr``, then ``_seed``. + """ + if version.endswith(_LEAK_SUFFIX): + version = version[: -len(_LEAK_SUFFIX)] + return _SEED_SUFFIX.sub("", version) + + +def base_catalogue(catalogues, version): + """The base catalogue of ``version``: its entry, then its ``base:`` links.""" + name, seen = entry_of(version), [] + while True: + if name in NOT_CATALOGUES: + raise CustodyError(f"{name} is not a catalogue") + if name not in catalogues: + via = f" (reached from {version} through base:)" if seen else "" + raise CustodyError(f"no catalogue {name} in the catalogue config{via}") + if name in seen: + raise CustodyError( + f"base: links form a cycle: {' -> '.join(seen + [name])}" + ) + seen.append(name) + parent = catalogues[name].get("base") + if parent is None: + return name + if "blinding" in catalogues[name]: + raise CustodyError( + f"{name} names its base {parent}; declare custody on {parent}, " + f"not on {name}" + ) + name = parent + + +def declaration(catalogues, base): + """What base catalogue ``base`` declares: a status, or ``None`` (blinded).""" + declared = catalogues[base].get("blinding") + if declared is not None and declared not in STATUSES: + raise CustodyError( + f"{base} declares blinding: {declared!r}; declare blinded, unblinded " + "or mock" + ) + return declared + + +@dataclass(frozen=True) +class Record: + """A blind's public record: its commitment, the bases it covers, and the + seed once it is revealed.""" + + name: str + commitment: dict + bases: tuple + revealed: str | None + + +def records(registry): + """Every blind in the registry, by name.""" + found = {} + if registry is None or not Path(registry).is_dir(): + return found + for path in sorted(Path(registry).iterdir()): + if not path.is_dir() or path.name.startswith("."): + continue + try: + commitment = json.loads((path / "commitment.json").read_text()) + bases = tuple((path / "bases").read_text().split()) + except (OSError, ValueError) as err: + raise CustodyError(f"blind record {path} is incomplete: {err}") from None + revealed = path / "revealed.json" + seed = json.loads(revealed.read_text())["seed"] if revealed.exists() else None + found[path.name] = Record(path.name, commitment, bases, seed) + return found + + +def _no_blind(version, base, declared, registry): + why = ( + "declared blinded" + if declared == "blinded" + else "cosmo_val/cat_config.yaml declares no custody for it, so it is blinded" + ) + repo = registry.parent.parent + cat_config = registry.parent / "cat_config.yaml" + return ( + f"{version} is blinded ({why}) and no blind covers it.\n" + "Draw one, once:\n" + f" APPTAINERENV_PYTHONPATH={repo}/src spv-container exec " + f"python -m sp_validation.blinding init {base} " + f"--cat-config {cat_config}\n" + "then commit cosmo_val/blinds//.\n" + "Re-processing a catalogue whose blind is still concealed? Share it " + f"instead: python -m sp_validation.blinding share {base} " + f"--cat-config {cat_config}\n" + "Predates blinding: declare `blinding: unblinded`. A mock: " + "`blinding: mock`. A variant: `base: `." + ) + + +def custody_of(catalogues, version, *, registry): + """The custody of ``version``, from its base's declaration and the registry. + + @sc custody-is-declared + Custody is read only here, from the parsed catalogue config and the blind + registry; no workflow config, environment variable or call argument can + change it. An entry declaring nothing is blinded, and variants share their + base's custody and blind. + + Raises + ------ + CustodyError + With the one thing to do, when the declaration and the registry + disagree: a blinded catalogue with no blind, a revealed blind still + declared blinded, an unblinded catalogue under a concealed blind, a + mock under a blind, or two blinds over one catalogue. + """ + registry = Path(registry) + base = base_catalogue(catalogues, version) + declared = declaration(catalogues, base) + covering = [r for r in records(registry).values() if base in r.bases] + if len(covering) > 1: + names = ", ".join(r.name for r in covering) + raise CustodyError(f"{base} is covered by several blinds: {names}") + record = covering[0] if covering else None + + if declared == "mock": + if record is not None: + raise CustodyError( + f"{base} is declared mock but blind {record.name} covers it; a " + "mock is never blinded" + ) + return Custody("mock", base) + + if declared == "unblinded": + if record is None: + return Custody("unblinded", base) + if record.revealed is None: + raise CustodyError( + f"{base} is declared unblinded, but blind {record.name} still " + "conceals it. Unblinding is the reveal: python -m " + f"sp_validation.blinding reveal {record.name} --root " + f"--cat-config {registry.parent / 'cat_config.yaml'}" + ) + if seed_commitment(record.revealed) != record.commitment["seed_commitment"]: + raise CustodyError( + f"the seed published for blind {record.name} does not match its " + "commitment" + ) + return Custody("unblinded", base) + + if record is None: + raise CustodyError(_no_blind(version, base, declared, registry)) + if record.revealed is not None: + raise CustodyError( + f"blind {record.name} is public; declare `blinding: unblinded` on " + f"{base} (the reveal PR does this)" + ) + c = record.commitment + return Custody( + "blinded", + base, + record.name, + c["seed_commitment"], + c["config_digest"], + int(c["draw_scheme"]), + registry, + ) + + +def summary(catalogues, versions, *, registry): + """One ``[custody]`` line per base catalogue among ``versions``.""" + by_base = {} + for version in dict.fromkeys(versions): + by_base.setdefault(base_catalogue(catalogues, version), []).append(version) + lines = [] + for base, members in by_base.items(): + custody = custody_of(catalogues, base, registry=registry) + variants = [v for v in members if v != base] + also = f" (+ {', '.join(variants)})" if variants else "" + state = ( + f"blinded under {custody.blind}" + if custody.status == "blinded" + else custody.status + ) + lines.append(f"[custody] {base}{also}: {state}") + return lines + + +def read_stamp(metadata): + """The custody a SACC's stamp records; a missing or malformed stamp raises. + + The result carries no registry: it is what the file says, to compare with + what a catalogue is declared under (``.stamp`` or ``.token``). + """ + status = metadata.get("blinding") + if status not in STATUSES: + raise CustodyError( + f"no valid custody stamp (blinding={status!r}): every SACC is born " + "through sacc_io.save" + ) + if "blinding_catalogue" not in metadata: + raise CustodyError("custody stamp names no catalogue") + present = [k for k in _BLIND_KEYS if k in metadata] + if status == "blinded" and len(present) != len(_BLIND_KEYS): + missing = sorted(set(_BLIND_KEYS) - set(present)) + raise CustodyError(f"blinded custody stamp lacks {missing}") + if status != "blinded" and present: + raise CustodyError(f"a {status} custody stamp carries blind keys {present}") + if status != "blinded": + return Custody(status, metadata["blinding_catalogue"]) + return Custody( + status, + metadata["blinding_catalogue"], + metadata["blinding_blind"], + metadata["blinding_commitment"], + metadata["blinding_config"], + int(metadata["blinding_scheme"]), + ) + + +def confirm(custody, token): + """``custody``, if its token is the one the launch resolved; else raise.""" + if custody.token != token: + raise CustodyError( + f"custody of {custody.catalogue} changed since the launch: the job " + f"resolves {custody.token}, the launch resolved {token}" + ) + return custody diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 24fee0a4..b863a9ae 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -91,6 +91,7 @@ import sacc from astropy.io import fits +from . import custody as _custody from .statistics import cov_from_one_covariance PSF_TRACER = "psf_stars" @@ -761,8 +762,8 @@ def merge(saccs): block-diagonal is out of scope here; see ``assemble_covariance``). Metadata must be consistent: keys present in several inputs must carry - equal values (a ``type: data`` file cannot merge with a ``type: mock`` - file), and the union lands on the result. This deliberately replaces the + equal values (files under two custody stamps cannot merge), and the union + lands on the result. This deliberately replaces the library's clash behaviour, which mangles clashing keys by appending labels. @@ -942,78 +943,141 @@ def update_statistic(s, sub): s.data[idx[0]].value = point.value -def save(s, path, *, type, commitment=None): - """Write ``s`` to ``path`` (FITS), overwriting any existing file. +# --------------------------------------------------------------------------- # +# The file door: every SACC is born sealed, derived with one stamp, or refused +# --------------------------------------------------------------------------- # +# Signal: the shear data types a cosmology shift reaches. +SIGNAL_PREFIX = "galaxy_shear_" +# The signal rows a blind conceals. +SHIFTABLE = (XI_PLUS, XI_MINUS, CL_EE) +# Signal rows a pure E-mode shift leaves unchanged. +UNSHIFTED = (CL_BB, CL_EB) - Parameters - ---------- - s : sacc.Sacc - Data set to write; its metadata is stamped in place. - type : {'data', 'mock'} - Provenance of the underlying catalogue, stored as the required - ``type`` metadata tag (PRD #241 §4, "Mocks vs data"). The caller — - the pipeline computing the data vector — knows whether its input - catalogue is a mock; there is deliberately no default. ``load`` - refuses ``type='data'`` files that are not blinded. - commitment : str, optional - Path to the version's ``commitment.json``. When given, the file is - stamped concealed under that blind - (:func:`sp_validation.blinding.stamp_concealed_passthrough`, values - untouched) before writing — the seam every born-blinded or - blind-irrelevant part uses to clear the fail-closed load gate. + +def _stamped(s): + return any(key in s.metadata for key in _custody.STAMP_KEYS) + + +def _mint(s, stamp): + for key in _custody.STAMP_KEYS: + s.metadata.pop(key, None) + s.metadata.update(stamp) + + +def seal(s, custody): + """A stamped copy of ``s``, concealed first when the catalogue is blinded. + + @sc born-sealed + A catalogue-born SACC leaves memory only through here. Under a blinded + custody every ξ± and Cℓ_EE row is shifted on a copy before the stamp is + minted; a SACC with no signal is stamped without opening the blind; any + other signal (COSEBIs, pure-E/B) is a derived statistic, refused here and + saved with ``derived_from``. An already-stamped SACC is re-written only as + a derivation. """ - if type not in ("data", "mock"): - raise ValueError(f"type must be 'data' or 'mock'; got {type!r}") - if s.metadata.get("type", type) != type: + if _stamped(s): raise ValueError( - f"Sacc metadata already carries type={s.metadata['type']!r}; " - f"refusing to re-stamp as {type!r}" + "this SACC is already stamped; a loaded or sealed SACC is re-written " + "only as a derivation (save(..., derived_from=[...]))" ) - s.metadata["type"] = type - if commitment is not None: + types = {dp.data_type for dp in s.data} + signal = {t for t in types if t.startswith(SIGNAL_PREFIX)} + if custody.status == "blinded" and signal - set(SHIFTABLE) - set(UNSHIFTED): + raise ValueError( + f"a blinded catalogue's {sorted(signal - set(SHIFTABLE))} rows are " + "derived statistics: save them with derived_from=[their input parts]" + ) + if custody.status == "blinded" and signal & set(SHIFTABLE): from . import blinding - blinding.stamp_concealed_passthrough(s, commitment) - s.save_fits(path, overwrite=True) + out = blinding.conceal(s, blinding.open_blind(custody)) + else: + out = s.copy() + _mint(out, custody.stamp) + return out -def load(path, *, allow_unblinded=False): - """Load a Sacc from ``path`` (FITS), failing closed on unblinded data. +def _row_key(dp): + tags = tuple(sorted((k, v) for k, v in dp.tags.items() if k != "window")) + return (dp.data_type, tuple(dp.tracers), float(dp.value), tags) - Every sacc_io file carries a ``type: data|mock`` metadata tag (stamped by - ``save``); blinded files are additionally stamped ``concealed=True`` by - Smokescreen. A ``type='data'`` file without that stamp is real, unblinded - data, and loading it raises — skipping the blind can never silently - expose the measured vector (PRD #241 §4). Mocks load freely, blinded or - not. - Parameters - ---------- - path : str - File to load. - allow_unblinded : bool, optional - Escape hatch for the two legitimate consumers of unblinded data: - the blinding step itself (which must read the true vector to conceal - it) and the unblinding/verification tooling. Nothing else — no - analysis, plotting or inference code — may pass ``True``. +def _derive(s, parts, custody): + """A copy of ``s`` under its inputs' one stamp (and ``custody``'s, if given).""" + if not parts: + raise ValueError("a derivation needs its input parts") + stamps = [_custody.read_stamp(p.metadata) for p in parts] + if len({tuple(sorted(st.stamp.items())) for st in stamps}) > 1: + raise ValueError( + "input parts carry different custody stamps: " + + "; ".join(f"part {i}: {st.token}" for i, st in enumerate(stamps)) + ) + stamp = stamps[0] + if custody is not None and custody.stamp != stamp.stamp: + raise ValueError( + f"parts are stamped {stamp.token}, but {custody.catalogue} is " + f"declared {custody.token}" + ) + inputs = {_row_key(dp) for p in parts for dp in p.data if dp.data_type in SHIFTABLE} + stray = [ + i + for i, dp in enumerate(s.data) + if dp.data_type in SHIFTABLE and _row_key(dp) not in inputs + ] + if stray: + raise ValueError( + f"{len(stray)} ξ±/Cℓ_EE rows of a derivation are not copies of its " + "inputs' rows; a catalogue's shiftable signal is born only through " + "save(custody=)" + ) + out = s.copy() + _mint(out, stamp.stamp) + return out - Returns - ------- - sacc.Sacc - The loaded data set. + +def save(s, path, *, custody=None, derived_from=None): + """Write ``s`` to ``path`` (FITS) through the one door; return what was written. + + @sc one-door + The only writer of a SACC file, and with :func:`seal` the only place a + custody stamp is minted: + + - ``save(s, path, custody=c)``: a birth, sealed by :func:`seal`; + - ``save(s, path, derived_from=parts)``: a derivation, stamped with its + inputs' one stamp; + - ``save(s, path, derived_from=parts, custody=c)``: an assembly, a + derivation whose stamp must also be ``c``'s. + + @sc derived-inherit + A derivation's inputs must share one stamp, and each of its ξ±/Cℓ_EE rows + must be a copy of an input row (type, tracers, value, tags; windows by + index), so plaintext cannot be saved under a concealed stamp. """ - s = sacc.Sacc.load_fits(path) - if ( - s.metadata["type"] == "data" - and not s.metadata.get("concealed", False) - and not allow_unblinded - ): + if derived_from is not None: + out = _derive(s, list(derived_from), custody) + elif custody is not None: + out = seal(s, custody) + else: raise ValueError( - f"{path} holds real data (type='data') without the " - "concealed=True blinding stamp — refusing to load an unblinded " - "data vector. Only the blinding/unblinding tooling may pass " - "allow_unblinded=True." + "save needs custody= (a birth) or derived_from= (a derivation); " + "a SACC is never written unstamped" ) + out.save_fits(str(path), overwrite=True) + return out + + +def load(path): + """Load the SACC at ``path``, refusing a file without a valid custody stamp. + + @sc stamped-or-refused + Every file ``save`` wrote carries one of the three stamps; anything else was + not born through the door and is refused, with no escape hatch. + """ + s = sacc.Sacc.load_fits(str(path)) + try: + _custody.read_stamp(s.metadata) + except _custody.CustodyError as err: + raise _custody.CustodyError(f"{path}: {err}") from None return s @@ -1549,42 +1613,3 @@ def covariance_blocks(cov_list, selectors, *, gaussian=True): return [ (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) ] - - -# --------------------------------------------------------------------------- # -# Terminal assembly — gather() and its blind-custody call site. -# --------------------------------------------------------------------------- # -def gather(parts, metadata=None, assemble=None): - """Assemble standalone part SACCs into the one-file ``{version}.sacc``. - - The terminal seam: every path that combines parts into the one-file - product goes through here, because this is where the one thing an - assembler cannot know about is enforced — blind custody. - :func:`sp_validation.blinding.assert_consistent_blind` runs before the - assembly and its returned shared stamp is written onto the result. The - assembler is passed *in* rather than wrapping this guard, which is what - keeps the guard un-bypassable. - - Parameters - ---------- - parts : sequence of sacc.Sacc - The part SACCs, in the assembly (covariance) order. - metadata : dict, optional - Extra key/value pairs to store on the assembled file's metadata. - assemble : callable, optional - ``assemble(parts) -> sacc.Sacc``. Defaults to :func:`merge`. Bind any - further arguments (n(z), metadata) into the callable. - - Returns - ------- - sacc.Sacc - The assembled file. - """ - from . import blinding - - parts = list(parts) - stamp = blinding.assert_consistent_blind(parts) - s = (assemble or merge)(parts) - for key, value in {**(metadata or {}), **(stamp or {})}.items(): - s.metadata[key] = value - return s diff --git a/src/sp_validation/tests/_synthetic.py b/src/sp_validation/tests/_synthetic.py new file mode 100644 index 00000000..b17bab3d --- /dev/null +++ b/src/sp_validation/tests/_synthetic.py @@ -0,0 +1,143 @@ +"""A small deterministic synthetic catalogue, on disk, with its catalogue config. + +The glue tests run the real compute seams on it: a shear catalogue +(RA/Dec/e1/e2/w), a PSF star catalogue with the columns the leakage and ρ/τ +seams read, a cs_util-readable dndz, and a ``cat_config.yaml`` beside a +``blinds/`` registry directory, as in the repository layout. Every catalogue +entry in the config describes the same files and declares its own custody. +""" + +import copy + +import numpy as np +import yaml + + +def write_synthetic_catalogs( + tmp_path, + n_gal=2000, + n_star=800, + ra_range=(10.0, 14.0), + dec_range=(10.0, 14.0), + seed=1234, + coherent_shear=False, + with_psf=False, + catalogues=None, +): + """Write the catalogue files and their config; return ``(params, version)``. + + Parameters + ---------- + coherent_shear : bool + Inject a smooth position-dependent shear on top of shape noise, so ξ± + is smooth (the pure-E/B integral needs it to be well-posed). + with_psf : bool + Add a ``psf`` block (ρ/τ and pseudo-Cℓ read it via + ``get_params_rho_tau``). + catalogues : dict, optional + ``{version: declaration}``, one catalogue entry each over the same + files. A declaration is a ``blinding`` value, or ``None`` for an entry + that declares none. Defaults to one mock, ``TestCatalog``. + + Returns + ------- + (dict, str) + ``CosmologyValidation`` keyword arguments (``catalog_config``, + ``output_dir``) and the first catalogue's version. + """ + from astropy.table import Table + + catalogues = catalogues or {"TestCatalog": "mock"} + rng = np.random.default_rng(seed) + + cat_dir = tmp_path / "catalog" + nz_dir = tmp_path / "nz" + output_dir = tmp_path / "output" + for directory in (cat_dir, nz_dir, output_dir): + directory.mkdir() + + ra = rng.uniform(*ra_range, n_gal) + dec = rng.uniform(*dec_range, n_gal) + if coherent_shear: + e1 = 0.02 * np.cos(np.radians(ra) * 40) + rng.normal(0, 0.05, n_gal) + e2 = 0.02 * np.sin(np.radians(dec) * 40) + rng.normal(0, 0.05, n_gal) + else: + e1 = rng.normal(0, 0.25, n_gal) + e2 = rng.normal(0, 0.25, n_gal) + w = rng.uniform(0.5, 1.0, n_gal) + Table({"RA": ra, "Dec": dec, "e1": e1, "e2": e2, "w": w}).write( + cat_dir / "shear.fits", overwrite=True + ) + + Table( + { + "RA": rng.uniform(*ra_range, n_star), + "Dec": rng.uniform(*dec_range, n_star), + "HSM_G1_PSF": rng.normal(0, 0.03, n_star), + "HSM_G2_PSF": rng.normal(0, 0.03, n_star), + "HSM_G1_STAR": rng.normal(0, 0.03, n_star), + "HSM_G2_STAR": rng.normal(0, 0.03, n_star), + "HSM_T_PSF": rng.uniform(0.4, 0.6, n_star), + "HSM_T_STAR": rng.uniform(0.4, 0.6, n_star), + "HSM_FLAG_PSF": np.zeros(n_star, dtype=int), + "HSM_FLAG_STAR": np.zeros(n_star, dtype=int), + } + ).write(cat_dir / "star.fits", overwrite=True) + + # cs_util.read_dndz's commented-header format: "z" holds the n+1 bin + # edges, "dn_dz" the densities. + z_edges = np.linspace(0.05, 3.0, 31) + dndz = np.exp(-(((z_edges - 0.7) / 0.3) ** 2)) + lines = ["# z dn_dz"] + [f"{zz} {nn}" for zz, nn in zip(z_edges, dndz)] + (nz_dir / "dndz_SP_A.txt").write_text("\n".join(lines) + "\n") + + entry = { + "subdir": str(cat_dir), + "pipeline": "SP", + "shear": { + "path": "shear.fits", + "redshift_path": str(nz_dir / "dndz_SP_A.txt"), + "w_col": "w", + "e1_col": "e1", + "e2_col": "e2", + "R": 1.0, + "e1_col_corrected": "e1", + "e2_col_corrected": "e2", + }, + "star": { + "path": "star.fits", + "ra_col": "RA", + "dec_col": "Dec", + "e1_col": "HSM_G1_PSF", + "e2_col": "HSM_G2_PSF", + }, + } + if with_psf: + entry["psf"] = { + "path": "star.fits", + "hdu": 1, + "ra_col": "RA", + "dec_col": "Dec", + "e1_PSF_col": "HSM_G1_PSF", + "e2_PSF_col": "HSM_G2_PSF", + "e1_star_col": "HSM_G1_STAR", + "e2_star_col": "HSM_G2_STAR", + "PSF_size": "HSM_T_PSF", + "star_size": "HSM_T_STAR", + "PSF_flag": "HSM_FLAG_PSF", + "star_flag": "HSM_FLAG_STAR", + } + + config = { + "nz": {"subdir": str(nz_dir), "dndz": {"blind": "A", "path": "dndz"}}, + "paths": {"output": str(output_dir)}, + } + for version, declaration in catalogues.items(): + config[version] = copy.deepcopy(entry) + if declaration is not None: + config[version]["blinding"] = declaration + config_path = tmp_path / "cat_config.yaml" + config_path.write_text(yaml.safe_dump(config, sort_keys=False)) + + params = {"catalog_config": str(config_path), "output_dir": str(output_dir)} + return params, next(iter(catalogues)) diff --git a/src/sp_validation/tests/test_architecture.py b/src/sp_validation/tests/test_architecture.py new file mode 100644 index 00000000..c76ed9f7 --- /dev/null +++ b/src/sp_validation/tests/test_architecture.py @@ -0,0 +1,170 @@ +"""Architecture on the resolved code tree: each assertion names its contract. + +Import boundaries are read from the ``CONTRACTS`` files (the rule lines the +scientific-software-development skill's ``check-imports`` also reads); the +rest are AST queries over every Python file outside the tests. +""" + +import ast +import fnmatch +import re +import sys +from pathlib import Path + +import pytest + +from sp_validation import custody + +REPO = Path(__file__).resolve().parents[3] +TESTS = Path(__file__).resolve().parent + +HEADER = re.compile(r"^\s*@sc(?:\s+\[[^\]]*\])?\s+([\w][\w.-]*)\s*$") +ONLY = re.compile(r"^\s*only:\s*(\S+)\s+may import\s+(.+?)\s*$") +ONLY_IMPORTER = re.compile(r"^\s*only:\s*(\S+)\s+may be imported by\s+(.+?)\s*$") + + +def _sources(): + """``{module name: path}`` for every Python file outside the tests.""" + found = {} + for root, package in ((REPO / "src" / "sp_validation", "sp_validation"),): + for path in root.rglob("*.py"): + if TESTS in path.parents: + continue + parts = path.relative_to(root).with_suffix("").parts + parts = parts[:-1] if parts[-1] == "__init__" else parts + found[".".join((package, *parts))] = path + for top in ("workflow", "scripts", "papers", "cosmo_inference"): + for path in (REPO / top).rglob("*.py"): + if "tests" in path.relative_to(REPO).parts or ".snakemake" in path.parts: + continue + found[".".join(path.relative_to(REPO).with_suffix("").parts)] = path + return found + + +SOURCES = _sources() + + +def _imports(module, path): + """Absolute names of every module ``path`` imports.""" + names = [] + for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): + if isinstance(node, ast.Import): + names += [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom): + if node.level: + package = module.split(".") + package = package if path.name == "__init__.py" else package[:-1] + base = ".".join(package[: len(package) - node.level + 1]) + target = f"{base}.{node.module}" if node.module else base + names += ( + [target] + if node.module + else [f"{target}.{alias.name}" for alias in node.names] + ) + else: + names.append(node.module) + return names + + +def _matches(name, pattern): + if pattern == "stdlib": + return name.split(".")[0] in sys.stdlib_module_names | {"__future__"} + return fnmatch.fnmatchcase(name, pattern) + + +def _rules(): + """``(contract, kind, subject, patterns)`` from every CONTRACTS file.""" + rules = [] + for contracts in sorted(REPO.rglob("CONTRACTS")): + if ".snakemake" in contracts.parts: + continue + ident = None + for line in contracts.read_text().splitlines(): + if header := HEADER.match(line): + ident = header[1] + elif rule := ONLY_IMPORTER.match(line): + rules.append((ident, "importers", rule[1], rule[2].split(", "))) + elif rule := ONLY.match(line): + rules.append((ident, "imports", rule[1], rule[2].split(", "))) + return rules + + +RULES = _rules() + + +def test_the_contracts_carry_the_boundaries(): + assert {r[0] for r in RULES} >= { + "custody-is-stdlib", + "container-is-stdlib", + "blinding-owns-smokescreen", + "blinding-owns-the-seed-cipher", + "host-importable", + } + + +@pytest.mark.parametrize( + "contract, kind, subject, patterns", RULES, ids=[r[0] for r in RULES] +) +def test_import_boundaries(contract, kind, subject, patterns): + if kind == "imports": + assert subject in SOURCES, f"[{contract}] no module {subject}" + stray = [ + name + for name in _imports(subject, SOURCES[subject]) + if not any(_matches(name, p) for p in patterns) + ] + assert not stray, f"[{contract}] {subject} imports {stray}" + else: + stray = sorted( + module + for module, path in SOURCES.items() + if not any(_matches(module, p) for p in patterns) + and any(fnmatch.fnmatchcase(n, subject) for n in _imports(module, path)) + ) + assert not stray, f"[{contract}] {subject} is imported by {stray}" + + +def _nodes(kind): + for module, path in SOURCES.items(): + for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): + if isinstance(node, kind): + yield module, node + + +def test_stamp_keys_are_spelt_only_in_custody(): + """[stamped-or-refused] Stamps are read and minted through custody.stamp and read_stamp.""" + spelt = sorted( + (module, node.value) + for module, node in _nodes(ast.Constant) + if isinstance(node.value, str) + and node.value in custody.STAMP_KEYS + and module != "sp_validation.custody" + ) + assert not spelt, spelt + + +def test_custody_is_constructed_only_where_it_is_declared(): + """[custody-is-declared] Only custody_of builds a Custody.""" + built = sorted( + module + for module, node in _nodes(ast.Call) + if ( + (isinstance(node.func, ast.Name) and node.func.id == "Custody") + or (isinstance(node.func, ast.Attribute) and node.func.attr == "Custody") + ) + and module != "sp_validation.custody" + ) + assert not built, built + + +@pytest.mark.parametrize("method", ["save_fits", "load_fits"]) +def test_sacc_files_pass_one_door(method): + """[one-door] Only sacc_io writes or reads a SACC file.""" + callers = sorted( + module + for module, node in _nodes(ast.Call) + if isinstance(node.func, ast.Attribute) + and node.func.attr == method + and module != "sp_validation.sacc_io" + ) + assert not callers, callers diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py index f17d5468..3c82c44f 100644 --- a/src/sp_validation/tests/test_assemble_sacc.py +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -19,6 +19,7 @@ from sp_validation import sacc_io as sio from sp_validation.cosmo_val import sacc_writers as sw +from sp_validation.custody import Custody def _load_assemble_module(): @@ -51,6 +52,7 @@ def _theta(n=6): META = {"catalogue_version": "vSYNTH", "npatch": 1} +MOCK = Custody("mock", "vSYNTH") def _xi_cov_txt(tmp_path, n=12, seed=21): @@ -157,7 +159,7 @@ def get_bandpower_windows(self): paths = {} for name, part in parts.items(): p = tmp_path / f"{name}.sacc" - sio.save(part, str(p), type="mock") + sio.save(part, str(p), custody=MOCK) paths[name] = str(p) return paths @@ -170,7 +172,12 @@ def test_assemble_sacc_canonical_order(tmp_path): cl_cov_path, _blocks = _pseudo_cl_cov_fits(tmp_path) out = tmp_path / "vSYNTH.sacc" s = asm.assemble_sacc( - "vSYNTH", paths, str(out), xi_cov=cov_path, pseudo_cl_cov=cl_cov_path + "vSYNTH", + paths, + str(out), + custody=MOCK, + xi_cov=cov_path, + pseudo_cl_cov=cl_cov_path, ) assert out.exists() assert type(s.covariance).__name__ == "BlockDiagonalCovariance" @@ -209,12 +216,17 @@ def test_injected_xi_covariance_replaces_the_parts_own(tmp_path): cov_path, xi_cov = _xi_cov_txt(tmp_path) cl_cov_path, _blocks = _pseudo_cl_cov_fits(tmp_path) - born = sio.load(paths["xi_reporting"], allow_unblinded=True).covariance.dense + born = sio.load(paths["xi_reporting"]).covariance.dense assert not np.allclose(born, xi_cov) # the two are distinguishable out = tmp_path / "vSYNTH.sacc" s = asm.assemble_sacc( - "vSYNTH", paths, str(out), xi_cov=cov_path, pseudo_cl_cov=cl_cov_path + "vSYNTH", + paths, + str(out), + custody=MOCK, + xi_cov=cov_path, + pseudo_cl_cov=cl_cov_path, ) tr = ("source_0", "source_0") xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) @@ -232,7 +244,7 @@ def test_assemble_sacc_injects_pseudo_cl_covariance(tmp_path): out = tmp_path / "vSYNTH.sacc" s = asm.assemble_sacc( - "vSYNTH", paths, str(out), xi_cov=cov_path, pseudo_cl_cov=cov_fits + "vSYNTH", paths, str(out), custody=MOCK, xi_cov=cov_path, pseudo_cl_cov=cov_fits ) tr = ("source_0", "source_0") cl_idx = np.concatenate( @@ -253,7 +265,7 @@ def test_missing_injected_covariance_raises(tmp_path): paths = _write_parts(tmp_path, cov_less=()) # every part born with a block out = tmp_path / "vSYNTH.sacc" with pytest.raises(ValueError, match="takes its analysis covariance from"): - asm.assemble_sacc("vSYNTH", paths, str(out)) + asm.assemble_sacc("vSYNTH", paths, str(out), custody=MOCK) def test_assemble_sacc_respects_pseudo_cl_toggle(tmp_path): @@ -262,7 +274,7 @@ def test_assemble_sacc_respects_pseudo_cl_toggle(tmp_path): assert "pseudo_cl" not in paths cov_path, _xi_cov = _xi_cov_txt(tmp_path) out = tmp_path / "vSYNTH.sacc" - s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=cov_path) + s = asm.assemble_sacc("vSYNTH", paths, str(out), custody=MOCK, xi_cov=cov_path) tr = ("source_0", "source_0") assert len(s.indices(sio.CL_EE, tr)) == 0 # Round-trips as a valid BlockDiagonalCovariance over the remaining points. @@ -284,6 +296,7 @@ def test_assemble_sacc_expected_part_missing_raises(tmp_path): "vSYNTH", paths, str(out), + custody=MOCK, expected=["xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau"], xi_cov=cov_path, ) @@ -296,5 +309,10 @@ def test_assemble_sacc_expected_rejects_unknown_name(tmp_path): out = tmp_path / "vSYNTH.sacc" with pytest.raises(ValueError, match="not assemblable statistics"): asm.assemble_sacc( - "vSYNTH", paths, str(out), expected=["cosebi"], xi_cov=cov_path + "vSYNTH", + paths, + str(out), + custody=MOCK, + expected=["cosebi"], + xi_cov=cov_path, ) diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index cdec4e8c..ac5c6bf6 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -1,31 +1,93 @@ -"""Tests for :mod:`sp_validation.blinding` — per-part Smokescreen blinding. +"""The blind and the file door: invariants I1–I11. -Acceptance criteria AC1–AC9 of the blinding PRD, plus fast unit coverage of -the config/custody surface. The theory tests (fork + CCL) are marked ``slow``; -the derived-statistics tests additionally ``importorskip`` ``cosmo_numba``. - -All fixtures are synthetic and deterministic. Each blindable intermediate is -its own standalone part SACC, as in the per-part-at-birth architecture; -derived statistics are computed downstream from the blinded integration ξ± -through the pipeline's own seams. The fork is handed no data vector at all, so -fixture ξ± values need only be smooth synthetic templates. +A blind is drawn once (``blinding init``) into a registry beside a catalogue +config; ``sacc_io.save`` is the only writer, and conceals a blinded +catalogue's ξ± and Cℓ_EE rows in memory before the file exists. Blinds here +use the fast Eisenstein–Hu theory, so the whole module runs in the fast suite; +``test_camb_ccl_crosscheck.py`` covers the production CAMB recipe. """ +import dataclasses import json -import pathlib +import os +import stat import numpy as np import pytest +import yaml from sp_validation import blinding as bd +from sp_validation import custody as cu from sp_validation import sacc_io as sio from sp_validation.blinding_theory import TheoryConfig -_NOLOG = lambda *a, **k: None # noqa: E731 +FAST = {"theory": {"transfer_function": "eisenstein_hu"}} + + +# --------------------------------------------------------------------------- # +# A registry with real blinds, and custody under them +# --------------------------------------------------------------------------- # +def _cat_config(root): + """A catalogue config declaring one catalogue per custody state.""" + entries = { + "TOY": {}, + "OTHER": {}, + "TOY_OPEN": {"blinding": "unblinded"}, + "TOY_MOCK": {"blinding": "mock"}, + "paths": {"output": str(root / "output")}, + } + path = root / "cat_config.yaml" + path.write_text(yaml.safe_dump(entries)) + (root / "fast.json").write_text(json.dumps(FAST)) + return path + + +def _init(root, blind, *bases): + bd.main( + [ + "init", + blind, + *bases, + "--cat-config", + str(root / "cat_config.yaml"), + "--config", + str(root / "fast.json"), + ] + ) + + +def _custody(root, version): + path = root / "cat_config.yaml" + return cu.custody_of( + yaml.safe_load(path.read_text()), version, registry=cu.registry_of(path) + ) + + +@pytest.fixture(scope="module") +def blinds(tmp_path_factory): + """Blinds `toy` (TOY) and `other` (OTHER), and the four custodies. + + Their seeds are fixed, so every run tests the same hidden points. + """ + root = tmp_path_factory.mktemp("registry") + _cat_config(root) + with pytest.MonkeyPatch.context() as m: + for blind, base in (("toy", "TOY"), ("other", "OTHER")): + m.setattr(bd.secrets, "token_hex", lambda n, b=blind: f"{b}-seed") + _init(root, blind, base) + return {v: _custody(root, v) for v in ("TOY", "OTHER", "TOY_OPEN", "TOY_MOCK")} + + +@pytest.fixture +def fresh(tmp_path): + """A registry of its own, for tests that edit or reveal a blind.""" + _cat_config(tmp_path) + _init(tmp_path, "toy", "TOY") + return tmp_path # --------------------------------------------------------------------------- # -# Synthetic part fixtures +# Synthetic parts # --------------------------------------------------------------------------- # def _gauss_nz(z0, sigma, n=200): z = np.linspace(0.0, 3.0, n) @@ -33,1272 +95,691 @@ def _gauss_nz(z0, sigma, n=200): return z, nz / np.trapezoid(nz, z) -def _reporting_theta(n=8): - return np.geomspace(5.0, 250.0, n) - - -def _integration_theta(n=80): - # The integration grid is the pure-E/B INTEGRATION grid, so it spans wider than - # the reporting range on both ends (production: ~0.08–300 arcmin). - return np.geomspace(0.1, 300.0, n) +NZ2 = {0: _gauss_nz(0.5, 0.15), 1: _gauss_nz(0.9, 0.2)} +PAIRS = ((0, 0), (0, 1), (1, 1)) +# The ξ± tags a part may carry; blocks never depend on them (None: no tag). +XI_TAGS = ("reporting", "integration", "cosebis", "mystery", None) def _xi_template(theta, k=0): - """Smooth synthetic ξ± for pair index ``k`` (no CCL needed).""" - theta = np.asarray(theta) xip = 1e-4 * (1 + 0.1 * k) * (theta / 10.0) ** -0.6 xim = 0.5e-4 * (1 + 0.1 * k) * (theta / 10.0) ** -0.9 return xip, xim -def _b_mode_template(theta, amplitude): - """A smooth ξ_B(θ) template. B contributes +ξ_B to ξ+, −ξ_B to ξ−.""" - return amplitude * np.exp(-((np.log(np.asarray(theta) / 30.0)) ** 2) / 2.0) +def _add_xi_rows(s, pair, theta, tag, k=0): + xip, xim = _xi_template(theta, k) + if tag is not None: + sio.add_xi(s, pair, theta, xip, xim, grid=tag) + return + tracers = sio._pair(pair) + for dtype, values in ((sio.XI_PLUS, xip), (sio.XI_MINUS, xim)): + for th, v in zip(theta, values): + s.add_data_point(dtype, tracers, float(v), theta=float(th)) -def _nz_dict(nbins): - return {i: _gauss_nz(0.5 + 0.3 * i, 0.15 + 0.02 * i) for i in range(nbins)} +def _add_cl_rows(s, pair, k=0): + ell_eff = np.array([30.0, 80.0, 150.0, 280.0, 450.0]) + w_ell = np.arange(2, 501).astype(float) + w_mat = np.exp(-0.5 * ((w_ell[:, None] - ell_eff[None, :]) / 40.0) ** 2) + w_mat /= w_mat.sum(axis=0) + ee = 1e-8 * (1 + 0.1 * k) * (ell_eff / 100.0) ** -1.2 + sio.add_pseudo_cl( + s, + pair, + ell_eff, + ee, + 0.01 * ee, + 0.02 * ee, + window_ells=w_ell, + window_weights=w_mat, + ) -def _pairs(nbins): - return [(i, j) for i in range(nbins) for j in range(i, nbins)] +def _add_rho_rows(s): + theta = np.geomspace(5.0, 250.0, 6) + sio.add_rho(s, 0, theta, np.arange(6) * 1e-7, np.arange(6) * 2e-7) + sio.add_tau(s, (0,), 0, theta, np.arange(6) * 3e-7, np.arange(6) * 4e-7) + + +def two_bin_sacc(*, xi_tags=XI_TAGS, cl=True, rho=False, covariance=True): + """ξ± on every tag in ``xi_tags`` and pseudo-Cℓ, over pairs (0,0), (0,1), (1,1).""" + s = sio.new_sacc(NZ2, metadata={"catalogue_version": "TOY"}) + for k, pair in enumerate(PAIRS): + for n, tag in enumerate(xi_tags): + _add_xi_rows(s, pair, np.geomspace(2.0, 200.0, 6) * (1 + 0.013 * n), tag, k) + if cl: + _add_cl_rows(s, pair, k) + if rho: + _add_rho_rows(s) + if covariance: + rng = np.random.default_rng(3) + s.add_covariance( + np.abs(np.asarray(s.mean)) ** 2 * rng.uniform(1, 2, len(s.mean)) + ) + return s -def make_xi_part(grid, nbins=1, b_amplitude=0.0): - """A standalone ξ± part SACC (one grid), synthetic values, eye covariance. +def cosebis_sacc(): + s = sio.new_sacc({0: NZ2[0]}, metadata={"catalogue_version": "TOY"}) + sio.add_cosebis(s, (0, 0), np.arange(1, 6) * 1e-10, (12.0, 83.0), Bn=np.ones(5)) + return s - ``b_amplitude`` injects a pure B-mode (+ξ_B to ξ+, −ξ_B to ξ−; - b_modes.py sign convention) — used on the integration part for AC4/AC9. - """ - theta = _reporting_theta() if grid == "reporting" else _integration_theta() - s = sio.new_sacc( - _nz_dict(nbins), metadata={"catalogue_version": "vTEST", "type": "mock"} - ) - blocks = [] - for k, (i, j) in enumerate(_pairs(nbins)): - xip, xim = _xi_template(theta, k) - xi_b = _b_mode_template(theta, b_amplitude) - sio.add_xi(s, (i, j), theta, xip + xi_b, xim - xi_b, grid=grid) - tr = sio._pair((i, j)) - idx = np.concatenate( - [ - s.indices(sio.XI_PLUS, tr, grid=grid), - s.indices(sio.XI_MINUS, tr, grid=grid), - ] - ) - blocks.append((idx, np.eye(len(idx)) * 1e-12)) - sio.assemble_covariance(s, blocks) + +def pure_eb_sacc(): + s = sio.new_sacc({0: NZ2[0]}, metadata={"catalogue_version": "TOY"}) + theta = np.geomspace(2.0, 200.0, 4) + sio.add_pure_eb(s, (0, 0), theta, **{k: np.ones(4) for k in sio.PURE_KEYS}) return s -def make_cl_part(nbins=1): - """A standalone pseudo-Cℓ part SACC (EE/BB/EB + bandpower windows).""" - s = sio.new_sacc( - _nz_dict(nbins), metadata={"catalogue_version": "vTEST", "type": "mock"} - ) - ell_eff = np.array([30.0, 80.0, 150.0, 280.0, 450.0]) - w_ell = np.arange(2, 501).astype(float) - w_mat = np.zeros((len(w_ell), len(ell_eff))) - for b, le in enumerate(ell_eff): - w_mat[:, b] = np.exp(-0.5 * ((w_ell - le) / 40.0) ** 2) - w_mat[:, b] /= w_mat[:, b].sum() - blocks = [] - for k, (i, j) in enumerate(_pairs(nbins)): - cl_ee = 1e-8 * (1 + 0.1 * k) * (ell_eff / 100.0) ** -1.2 - sio.add_pseudo_cl( - s, - (i, j), - ell_eff, - cl_ee, - np.zeros(5), - np.zeros(5), - window_ells=w_ell, - window_weights=w_mat, - ) - tr = sio._pair((i, j)) - idx = np.concatenate( - [s.indices(dt, tr) for dt in (sio.CL_EE, sio.CL_BB, sio.CL_EB)] - ) - blocks.append((idx, np.eye(len(idx)) * 1e-16)) - sio.assemble_covariance(s, blocks) +def rho_sacc(): + s = sio.new_sacc({0: NZ2[0]}, metadata={"catalogue_version": "TOY"}) + _add_rho_rows(s) return s -def make_rho_part(): - """A standalone ρ/τ PSF-diagnostics part SACC — never blindable.""" - ctheta = _reporting_theta() - s = sio.new_sacc( - _nz_dict(1), metadata={"catalogue_version": "vTEST", "type": "mock"} - ) - blocks = [] - for k in range(2): - sio.add_rho( - s, k, ctheta, np.arange(len(ctheta)) * 1e-7, np.arange(len(ctheta)) * 2e-7 - ) - idx = np.concatenate( - [s.indices(sio.RHO_PLUS.format(k=k)), s.indices(sio.RHO_MINUS.format(k=k))] +def _rows(s, *types): + return np.array([i for i, dp in enumerate(s.data) if dp.data_type in types]) + + +# --------------------------------------------------------------------------- # +# An independent CCL reference for the shift +# --------------------------------------------------------------------------- # +def _reference_params(seed): + """The hidden and fiducial CCL points, drawn with the fork directly.""" + from smokescreen.param_shifts import draw_param_shifts + + shift = draw_param_shifts({"S8": 0.075, "Omega_m": 0.1}, seed) + + def ccl_point(s8, om): + h, ob, mnu = 0.70, 0.0469, 0.06 + return dict( + Omega_c=om - ob - mnu / (93.14 * h**2), + Omega_b=ob, + h=h, + n_s=0.96, + sigma8=s8 / np.sqrt(om / 0.3), + m_nu=mnu, + mass_split="normal", + w0=-1.0, + wa=0.0, + Neff=3.046, + T_CMB=2.7255, ) - blocks.append((idx, np.eye(len(idx)) * 1e-18)) - sio.add_tau( - s, (0,), 0, ctheta, np.arange(len(ctheta)) * 3e-7, np.arange(len(ctheta)) * 4e-7 + + return ( + ccl_point(0.80 + shift["S8"], 0.30 + shift["Omega_m"]), + ccl_point(0.80, 0.30), ) - idx = np.concatenate( - [s.indices(sio.TAU_PLUS.format(k=0)), s.indices(sio.TAU_MINUS.format(k=0))] + + +def reference_shift(s, seed): + """t(hidden) − t(fiducial) on every ξ± and Cℓ_EE row of ``s``, from scratch.""" + import pyccl as ccl + + hidden, fiducial = _reference_params(seed) + ell = np.unique( + np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)]).astype(float) ) - blocks.append((idx, np.eye(len(idx)) * 1e-18)) - sio.assemble_covariance(s, blocks) - return s + def cosmo(p): + return ccl.Cosmology( + **p, transfer_function="eisenstein_hu", matter_power_spectrum="halofit" + ) + + def spectra(p, tracers, ells): + c = cosmo(p) + a, b = (s.tracers[t] for t in tracers) + return c, ccl.angular_cl( + c, + ccl.WeakLensingTracer(c, dndz=(a.z, a.nz)), + ccl.WeakLensingTracer(c, dndz=(b.z, b.nz)), + ells, + ) -def make_parts(nbins=1, b_amplitude=0.0, with_rho=True): - """All intermediate parts of one catalogue version, keyed by name.""" - parts = { - "xi_reporting": make_xi_part("reporting", nbins), - "xi_integration": make_xi_part("integration", nbins, b_amplitude=b_amplitude), - "cl": make_cl_part(nbins), - } - if with_rho: - parts["rho_tau"] = make_rho_part() - return parts + out = np.full(len(s.mean), np.nan) + for tracers in {dp.tracers for dp in s.data if dp.data_type in sio.SHIFTABLE}: + for dtype, kind in ((sio.XI_PLUS, "GG+"), (sio.XI_MINUS, "GG-")): + rows = [ + i + for i, dp in enumerate(s.data) + if (dp.data_type, dp.tracers) == (dtype, tracers) + ] + theta = np.array([s.data[i].tags["theta"] for i in rows]) / 60.0 + xi = [] + for p in (hidden, fiducial): + c, cl = spectra(p, tracers, ell) + xi.append(ccl.correlation(c, ell=ell, C_ell=cl, theta=theta, type=kind)) + out[rows] = xi[0] - xi[1] + rows = [ + i + for i, dp in enumerate(s.data) + if (dp.data_type, dp.tracers) == (sio.CL_EE, tracers) + ] + for i in rows: + window = s.data[i].tags["window"] + ells = np.asarray(window.values, float) + w = np.asarray(window.weight, float)[:, s.data[i].tags["window_ind"]] + out[i] = w @ ( + spectra(hidden, tracers, ells)[1] - spectra(fiducial, tracers, ells)[1] + ) + return out -def _derive_downstream(reporting_part, integration_part, nmodes=6): - """COSEBIs + pure-E/B the way the pipeline derives them downstream. +# --------------------------------------------------------------------------- # +# I1: the shift lands on every ξ± and Cℓ_EE row, from the content +# --------------------------------------------------------------------------- # +def test_shift_is_the_theory_difference_on_every_shiftable_row(blinds): + """seal(s) − s equals an independent CCL reference on every ξ± and Cℓ_EE row. - COSEBIs from the integration ξ± (full-range scale cut); pure-E/B from the - measured reporting reporting ξ± + the integration integration ξ±, with the - edge-based bounds set to the reporting grid's span — the outermost - reporting point sits at tmax with no interior support and comes back - NaN (the AC9 boundary case). + Two bins, all three pairs; ξ± rows under every tag and none, so blocks are + discovered from the data types and tracers, never from grid names. """ - from sp_validation import b_modes + s = two_bin_sacc() + sealed = sio.seal(s, blinds["TOY"]) + shift = np.asarray(sealed.mean) - np.asarray(s.mean) + expected = reference_shift(s, bd.open_blind(blinds["TOY"]).seed) + + rows = _rows(s, sio.XI_PLUS, sio.XI_MINUS, sio.CL_EE) + assert len(rows) and np.all(np.isfinite(expected[rows])) + for pair in PAIRS: + tracers = sio._pair(pair) + for types in ((sio.XI_PLUS, sio.XI_MINUS), (sio.CL_EE,)): + block = [ + i + for i in rows + if s.data[i].tracers == tracers and s.data[i].data_type in types + ] + scale = np.max(np.abs(expected[block])) + gap = np.max(np.abs(shift[block] - expected[block])) + assert scale > 0 and gap <= 1e-8 * scale, (pair, types, gap / scale) - theta_f, xip_f, xim_f = sio.get_xi(integration_part, (0, 0), grid="integration") - theta_c, xip_c, xim_c = sio.get_xi(reporting_part, (0, 0), grid="reporting") - En, Bn = b_modes.cosebis_from_xi( - theta_f, xip_f, xim_f, nmodes, scale_cut=(theta_f.min(), theta_f.max()) - ) - modes = b_modes.pure_eb_from_xi( - theta_c, - xip_c, - xim_c, - theta_f, - xip_f, - xim_f, - float(theta_c[0]), - float(theta_c[-1]), + +# --------------------------------------------------------------------------- # +# I2: nothing else moves +# --------------------------------------------------------------------------- # +def test_only_shiftable_values_move(blinds): + """BB, EB and ρ/τ values, every tag, tracer, row and the covariance are bitwise kept.""" + s = two_bin_sacc(rho=True) + sealed = sio.seal(s, blinds["TOY"]) + moved = _rows(s, sio.XI_PLUS, sio.XI_MINUS, sio.CL_EE) + kept = np.setdiff1d(np.arange(len(s.mean)), moved) + assert len(kept) and np.array_equal( + np.asarray(sealed.mean)[kept], np.asarray(s.mean)[kept] ) - return En, Bn, modes + assert np.all(np.asarray(sealed.mean)[moved] != np.asarray(s.mean)[moved]) + assert np.array_equal(sealed.covariance.dense, s.covariance.dense) + for a, b in zip(s.data, sealed.data): + assert (a.data_type, a.tracers) == (b.data_type, b.tracers) + assert {k: v for k, v in a.tags.items() if k != "window"} == { + k: v for k, v in b.tags.items() if k != "window" + } + for name, tracer in s.tracers.items(): + if hasattr(tracer, "nz"): + assert np.array_equal(sealed.tracers[name].nz, tracer.nz) + + +def test_derived_statistics_keep_their_values(blinds, tmp_path): + """COSEBIs and pure-E/B written as derivations carry their values unchanged.""" + xi = sio.seal(two_bin_sacc(cl=False), blinds["TOY"]) + for s in (cosebis_sacc(), pure_eb_sacc()): + path = tmp_path / "derived.sacc" + sio.save(s, path, derived_from=[xi]) + assert np.array_equal(np.asarray(sio.load(path).mean), np.asarray(s.mean)) # --------------------------------------------------------------------------- # -# BlindingConfig: envelope calibration + digest (fast) +# I3: the hidden point is uniform in the physical (S8, Ωm) box # --------------------------------------------------------------------------- # -def test_blinding_config_defaults(): - c = bd.BlindingConfig() - assert c.s8_half_width == 0.075 - assert c.omega_m_half_width == 0.1 - assert c.theory.S8 == 0.80 # fiducial TheoryConfig defaults - - -def test_blinding_config_overrides_fail_loud(): - c = bd.BlindingConfig.from_overrides({"s8_half_width": 0.05}) - assert c.s8_half_width == 0.05 - with pytest.raises(ValueError, match="unknown BlindingConfig fields"): - bd.BlindingConfig.from_overrides({"s8_half_width": 0.05, "bogus": 1}) - with pytest.raises(ValueError, match="unknown TheoryConfig fields"): - bd.BlindingConfig.from_overrides({"theory": {"nope": 1}}) - - -def test_blinding_config_is_frozen(): - with pytest.raises(Exception): - bd.BlindingConfig().s8_half_width = 0.2 - - -def test_envelope_calibration_maps_s8_box_to_ccl_halfwidths(): - """(S8, Ωm) half-widths → {sigma8, Omega_c} at the fiducial (exact forms).""" - c = bd.BlindingConfig() - shifts = c.shifts_dict() - assert set(shifts) == {"sigma8", "Omega_c"} - assert shifts["sigma8"] == pytest.approx( - c.s8_half_width / np.sqrt(c.theory.Omega_m / 0.3) - ) - assert shifts["Omega_c"] == c.omega_m_half_width - # every shift key must exist in the fiducial point (fork contract) - assert set(shifts) <= set(c.theory.ccl_params()) +def test_hidden_point_is_uniform_in_the_s8_om_box(): + from scipy import stats + config = bd.BlindingConfig() + fid = config.theory + hidden = [bd.hidden_theory(f"seed-{i}", config) for i in range(5000)] + s8 = np.array([h.S8 for h in hidden]) + om = np.array([h.Omega_m for h in hidden]) -def test_config_digest_stable_and_sensitive(): - """Canonical digest: byte-stable across runs, moves with every bound field.""" - c = bd.BlindingConfig() - assert c.config_digest() == bd.BlindingConfig().config_digest() - assert len(c.config_digest()) == 64 - assert bd.BlindingConfig(s8_half_width=0.05).config_digest() != c.config_digest() - assert ( - bd.BlindingConfig.from_overrides({"theory": {"S8": 0.79}}).config_digest() - != c.config_digest() - ) - # the P(k) recipe tokens are bound: a different halofit token = new digest - assert ( - bd.BlindingConfig.from_overrides( - {"theory": {"ccl_halofit_version": "takahashi"}} - ).config_digest() - != c.config_digest() - ) - # the Boltzmann backend (#280) is bound too — a different transfer - # function is a different P(k) path, so a different blind - assert ( - bd.BlindingConfig.from_overrides( - {"theory": {"transfer_function": "eisenstein_hu"}} - ).config_digest() - != c.config_digest() + assert np.all(np.abs(s8 - fid.S8) <= 0.075) and np.all( + np.abs(om - fid.Omega_m) <= 0.1 ) + assert stats.kstest((s8 - fid.S8 + 0.075) / 0.15, "uniform").pvalue > 1e-3 + assert stats.kstest((om - fid.Omega_m + 0.1) / 0.2, "uniform").pvalue > 1e-3 + for h in hidden[:50]: + p = h.ccl_params() + assert p["sigma8"] == pytest.approx(h.S8 / np.sqrt(h.Omega_m / 0.3), rel=1e-12) + omega_nu = h.m_nu / (93.14 * h.h**2) + assert p["Omega_c"] == pytest.approx( + h.Omega_m - h.Omega_b - omega_nu, rel=1e-12 + ) + assert bd.hidden_theory("seed-1", config) == hidden[1] -def test_theory_config_transfer_function_default_and_override(): - """#280: the Boltzmann backend is one config knob, CAMB by default (the - inference pipeline's Boltzmann code), overridable like any other field.""" - assert TheoryConfig().transfer_function == "boltzmann_camb" - cfg = TheoryConfig.from_overrides({"transfer_function": "eisenstein_hu"}) - assert cfg.transfer_function == "eisenstein_hu" +# --------------------------------------------------------------------------- # +# I4: the shift leaks no B-modes +# --------------------------------------------------------------------------- # +def _shape_noise_variance(left, right): + """Var ξ± per bin from shape noise at UNIONS n_eff, σ_e and area.""" + n_eff, sigma_e, area = 4.96, 0.378, 2894.0 * 3600.0 # arcmin⁻², —, arcmin² + pairs = np.pi * area * n_eff**2 * (right**2 - left**2) / 2.0 + return 2.0 * sigma_e**4 / pairs -def test_config_digest_int_float_canonical(): - """One physical cosmology has one digest, regardless of int-vs-float literals. +def test_shift_leaks_no_b_modes(blinds): + """|ΔBₙ|/σ(Bₙ) and |Δξ_B|/σ stay below 1e-2 on the production grids. - Configs come from YAML/CLI/humans, so a field can arrive as ``-1`` (int) or - ``-1.0`` (float). The digest must depend on the numeric *value*, not the - literal's Python type — otherwise an int-vs-float mismatch between the blind - and a later unblind would raise "config digest mismatch" and deny a - legitimate unblind. Every declared-float field must be canonical this way. + Single-bin n(z), the 0.08–300′ 1000-bin integration grid and the 1–250′ + 20-bin reporting grid; σ is shape noise at UNIONS depth. The shift is a pure + E-mode theory difference, so the B-modes it induces are numerical only. """ - for field, int_val, float_val in [ - ("w0", -1, -1.0), - ("wa", 0, 0.0), - ("Omega_m", 1, 1.0), - ("m_nu", 0, 0.0), - ("S8", 1, 1.0), - ]: - assert ( - bd.BlindingConfig.from_overrides( - {"theory": {field: int_val}} - ).config_digest() - == bd.BlindingConfig.from_overrides( - {"theory": {field: float_val}} - ).config_digest() - ), f"int-vs-float digest split on theory.{field}" - # the envelope half-widths (BlindingConfig's own float fields) too - assert ( - bd.BlindingConfig.from_overrides({"s8_half_width": 1}).config_digest() - == bd.BlindingConfig.from_overrides({"s8_half_width": 1.0}).config_digest() - ) - # and a full round-trip: an all-int override matches the float default digest - assert ( - bd.BlindingConfig.from_overrides( - {"theory": {"w0": -1, "Omega_m": 0, "wa": 0}} - ).config_digest() - == bd.BlindingConfig.from_overrides( - {"theory": {"w0": -1.0, "Omega_m": 0.0, "wa": 0.0}} - ).config_digest() - ) - + from sp_validation import b_modes -def test_theory_config_ccl_params_exact_keyset(): - """ccl_params() carries exactly the contracted keys — nothing rides along.""" - params = TheoryConfig().ccl_params() - assert set(params) == { - "Omega_c", - "Omega_b", - "h", - "n_s", - "sigma8", - "m_nu", - "mass_split", - "w0", - "wa", - "Neff", - "T_CMB", + grids = { + "integration": b_modes.log_bin_edges(0.08, 300.0, 1000), + "reporting": b_modes.log_bin_edges(1.0, 250.0, 20), } - assert params["sigma8"] == pytest.approx(0.80) # S8=0.80 at Ωm=0.30 - assert params["Neff"] == 3.046 and params["T_CMB"] == 2.7255 + s = sio.new_sacc({0: NZ2[0]}, metadata={"catalogue_version": "TOY"}) + theta = {} + for grid, (left, right) in grids.items(): + theta[grid] = np.sqrt(left * right) + sio.add_xi(s, (0, 0), theta[grid], *_xi_template(theta[grid]), grid=grid) + shift = np.asarray(sio.seal(s, blinds["TOY"]).mean) - np.asarray(s.mean) + + def delta(grid): + return tuple( + shift[s.indices(t, grid=grid)] for t in (sio.XI_PLUS, sio.XI_MINUS) + ) + + left, right = grids["integration"] + var = _shape_noise_variance(left, right) + (result,) = b_modes.cosebis_scan_from_xi( + theta["integration"], + *delta("integration"), + np.diag(np.concatenate([var, var])), + left, + right, + nmodes=5, + scale_cuts=[(12.0, 83.0)], + ).values() + sigma_b = np.sqrt(np.diag(result["cov"])[5:]) + assert np.max(np.abs(result["En"])) > 1e2 * np.max(np.abs(result["Bn"])) + assert np.max(np.abs(result["Bn"]) / sigma_b) <= 1e-2 + + rep_left, rep_right = grids["reporting"] + modes = b_modes.pure_eb_from_xi( + theta["reporting"], + *delta("reporting"), + theta["integration"], + *delta("integration"), + rep_left[0], + rep_right[-1], + ) + sigma = np.sqrt(_shape_noise_variance(rep_left, rep_right)) + for key in ("xip_B", "xim_B"): + finite = np.isfinite(modes[key]) + assert finite.sum() > 10 + assert np.max(np.abs(modes[key][finite]) / sigma[finite]) <= 1e-2, key # --------------------------------------------------------------------------- # -# Commitment + the fork's draw (fast; smokescreen import is light) +# I5: the seal table # --------------------------------------------------------------------------- # -def test_commitment_is_the_forks_domain_separated_digest(): - """One definition of the commitment, and it is the fork's.""" - import smokescreen - - seed = "the-secret" - assert bd.seed_commitment(seed) == smokescreen.seed_commitment(seed) - assert bd.seed_commitment("right") != bd.seed_commitment("wrong") - +@pytest.mark.parametrize("version", ["TOY_OPEN", "TOY_MOCK"]) +def test_unblinded_and_mock_births_are_stamped_untouched(blinds, version): + s = two_bin_sacc(rho=True) + sealed = sio.seal(s, blinds[version]) + assert np.array_equal(np.asarray(sealed.mean), np.asarray(s.mean)) + assert cu.read_stamp(sealed.metadata).stamp == blinds[version].stamp + assert "blinding" not in s.metadata # the input is not stamped in place -def test_commitment_does_not_embed_the_rng_seed(): - """The published commitment must not carry the effective RNG seed. - The fork derives the base seed for its per-key RNG from the *undomained* - sha256 of the seed string, taking the digest's first 8 bytes. A commitment - hashed over the bare seed would therefore publish that base seed verbatim in - its first 16 hex characters, and anyone holding the (public) commitment and - the (public) fiducial config could redraw the hidden cosmology and subtract - the blind. The domain prefix is what breaks that identity — this is the - regression guard for it. - """ - import secrets +def test_a_blinded_birth_is_concealed_and_stamped(blinds): + s = two_bin_sacc(cl=False) + sealed = sio.seal(s, blinds["TOY"]) + assert not np.array_equal(np.asarray(sealed.mean), np.asarray(s.mean)) + assert cu.read_stamp(sealed.metadata).stamp == blinds["TOY"].stamp - from smokescreen.param_shifts import _normalize_seed - # The third seed is drawn exactly as blind_init draws a production one. - for seed in ("my_secret_seed", "the-secret", secrets.token_hex(16)): - commitment = bd.seed_commitment(seed) - assert int(commitment[:16], 16) != _normalize_seed(seed) +@pytest.mark.parametrize( + "make", [cosebis_sacc, pure_eb_sacc], ids=["cosebis", "pure_eb"] +) +def test_a_blinded_birth_of_a_derived_statistic_is_refused(blinds, make): + with pytest.raises(ValueError, match="derived_from"): + sio.seal(make(), blinds["TOY"]) -def test_hidden_params_deterministic_and_in_envelope(): - """Same (seed, config) ⇒ same hidden point; draws respect the envelope.""" - import secrets +def test_a_blinded_birth_without_signal_never_opens_the_blind(blinds, monkeypatch): + def refuse(custody): + raise AssertionError("ρ/τ opened the blind") - c = bd.BlindingConfig() - fid = c.theory.ccl_params() - h1, h2 = bd.hidden_params("a-seed", c), bd.hidden_params("a-seed", c) - assert h1 == h2 - assert bd.hidden_params("другой", c) != h1 - shifts = c.shifts_dict() - for _ in range(50): - h = bd.hidden_params(secrets.token_hex(8), c) - for key, half in shifts.items(): - assert abs(h[key] - fid[key]) <= half - # only the enveloped keys move - assert all(h[k] == fid[k] for k in fid if k not in shifts) + monkeypatch.setattr(bd, "open_blind", refuse) + s = rho_sacc() + sealed = sio.seal(s, blinds["TOY"]) + assert np.array_equal(np.asarray(sealed.mean), np.asarray(s.mean)) + assert cu.read_stamp(sealed.metadata).stamp == blinds["TOY"].stamp -def test_hidden_params_no_global_rng_state(): - """The fork's draw uses a local RNG — global numpy state is untouched.""" - np.random.seed(0) - before = np.random.get_state()[1].copy() - bd.hidden_params("whatever", bd.BlindingConfig()) - assert np.array_equal(before, np.random.get_state()[1]) +def test_a_stamped_sacc_is_not_born_again(blinds, tmp_path): + sealed = sio.seal(two_bin_sacc(cl=False), blinds["TOY_OPEN"]) + with pytest.raises(ValueError, match="already stamped"): + sio.seal(sealed, blinds["TOY"]) + path = tmp_path / "part.sacc" + sio.save(two_bin_sacc(cl=False), path, custody=blinds["TOY_OPEN"]) + with pytest.raises(ValueError, match="already stamped"): + sio.save(sio.load(path), path, custody=blinds["TOY_OPEN"]) # --------------------------------------------------------------------------- # -# Per-part merge + provenance with a monkeypatched factor (fast — no CCL) +# I6: the gates # --------------------------------------------------------------------------- # -def _patch_constant_factor(monkeypatch, value=1e-6): - def fake(part, indices, factory, config, seed): - return np.arange(len(indices), dtype=float) * value + value - - monkeypatch.setattr(bd, "_concealing_factor", fake) - return fake - - -def test_merge_places_shift_at_recorded_indices_only(monkeypatch): - """AC5 (merge half), per part: the shift lands exactly on the blindable - rows, in stored order; covariance, n(z), and every tag are untouched.""" - _patch_constant_factor(monkeypatch) - for name, part in make_parts(nbins=2, with_rho=False).items(): - orig = np.array(part.mean) - orig_cov = part.covariance.dense.copy() - orig_nz = part.tracers["source_0"].nz.copy() - - blinded = bd.blind_sacc(part, "seed", log=_NOLOG) - - blocks = bd._blindable_blocks(part) - assert len(blocks) == 1, f"{name}: a part carries exactly one block" - shifted = np.zeros(len(orig), dtype=bool) - for _, indices, _ in blocks: - expected = orig[indices] + (np.arange(len(indices)) * 1e-6 + 1e-6) - assert np.array_equal(np.array(blinded.mean)[indices], expected) - shifted[indices] = True - assert np.array_equal(np.array(blinded.mean)[~shifted], orig[~shifted]) - assert np.array_equal(blinded.covariance.dense, orig_cov) - assert np.array_equal(blinded.tracers["source_0"].nz, orig_nz) - # row-order preservation: type/tracers/tags sequence is bitwise unchanged - for a, b in zip(part.data, blinded.data): - assert a.data_type == b.data_type - assert a.tracers == b.tracers - assert a.tags == b.tags - - -def test_provenance_metadata_contract(monkeypatch): - """Blinded parts carry concealed/blind/commitment/digest; no seed key.""" - _patch_constant_factor(monkeypatch) - s = make_xi_part("reporting") - s.metadata["seed_smokescreen"] = "leaked!" # must be stripped - c = bd.BlindingConfig() - blinded = bd.blind_sacc(s, "seed", config=c, label="B", log=_NOLOG) - assert blinded.metadata["concealed"] is True - assert blinded.metadata["blind"] == "B" - assert blinded.metadata["blind_commitment"] == bd.seed_commitment("seed") - assert blinded.metadata["blind_config_digest"] == c.config_digest() - assert blinded.metadata["blind_draw_scheme"] == bd.draw_scheme() - assert "seed_smokescreen" not in blinded.metadata - assert blinded.metadata["catalogue_version"] == "vTEST" - - -def test_blind_refuses_double_blind(monkeypatch): - _patch_constant_factor(monkeypatch) - s = make_xi_part("reporting") - blinded = bd.blind_sacc(s, "seed", log=_NOLOG) - with pytest.raises(ValueError, match="already concealed"): - bd.blind_sacc(blinded, "seed2", log=_NOLOG) - - -def test_blind_refuses_non_blindable_part(): - """A ρ/τ diagnostic part must never see a blind call — loud refusal.""" - with pytest.raises(ValueError, match="no blindable block"): - bd.blind_sacc(make_rho_part(), "seed", log=_NOLOG) - - -def test_unblind_fails_closed_on_wrong_seed_or_config(monkeypatch): - """AC6 (in-memory half): wrong seed and wrong config both refuse loudly.""" - _patch_constant_factor(monkeypatch) - s = make_xi_part("reporting") - blinded = bd.blind_sacc(s, "right-seed", log=_NOLOG) - with pytest.raises(ValueError, match="blind_commitment"): - bd.unblind_sacc(blinded, "wrong-seed", log=_NOLOG) - with pytest.raises(ValueError, match="blind_config_digest"): - bd.unblind_sacc( - blinded, - "right-seed", - config=bd.BlindingConfig(s8_half_width=0.01), - log=_NOLOG, - ) - with pytest.raises(ValueError, match="not concealed"): - bd.unblind_sacc(s, "right-seed", log=_NOLOG) +def test_save_needs_a_custody_or_inputs(tmp_path): + with pytest.raises(ValueError, match="custody"): + sio.save(two_bin_sacc(), tmp_path / "x.sacc") + assert not (tmp_path / "x.sacc").exists() + + +def _raw(tmp_path, name, **metadata): + s = two_bin_sacc(cl=False) + s.metadata.update(metadata) + path = tmp_path / f"{name}.sacc" + s.save_fits(str(path), overwrite=True) + return path + + +def test_load_refuses_unstamped_and_malformed_files(blinds, tmp_path): + stamp = blinds["TOY"].stamp + for name, metadata in { + "unstamped": {}, + "unknown": {**stamp, "blinding": "open"}, + "no_catalogue": {"blinding": "unblinded"}, + "no_commitment": {k: v for k, v in stamp.items() if k != "blinding_commitment"}, + "mixed": {**blinds["TOY_OPEN"].stamp, "blinding_blind": "toy"}, + }.items(): + with pytest.raises(ValueError, match="stamp"): + sio.load(_raw(tmp_path, name, **metadata)) + + +@pytest.mark.parametrize("version", ["TOY", "TOY_OPEN", "TOY_MOCK"]) +def test_every_custody_loads(blinds, tmp_path, version): + path = tmp_path / "part.sacc" + written = sio.save(two_bin_sacc(), path, custody=blinds[version]) + loaded = sio.load(path) + assert cu.read_stamp(loaded.metadata).stamp == blinds[version].stamp + assert np.array_equal(np.asarray(loaded.mean), np.asarray(written.mean)) # --------------------------------------------------------------------------- # -# The vector core: full-length theory in, block slice out (fast — fake backend) +# I7: derivations inherit one stamp; assembly equals the declaration # --------------------------------------------------------------------------- # -def _fake_factory(indices, hole=None): - """A backend that fills ``indices`` with ``sigma8 * arange`` and nothing else. +def test_a_derivation_inherits_its_inputs_stamp(blinds, tmp_path): + xi = sio.seal(two_bin_sacc(cl=False), blinds["TOY"]) + written = sio.save(cosebis_sacc(), tmp_path / "c.sacc", derived_from=[xi, xi]) + assert cu.read_stamp(written.metadata).stamp == blinds["TOY"].stamp - Mimics the real backends' contract — a block's ``theory_fn`` reads its - layout off the SACC it is given and returns a full-length vector, NaN on - every row outside its own block — without importing CCL. ``hole`` leaves - one row of the block itself unfilled. - """ - - def factory(s, theory): - def theory_fn(params): - out = np.full(len(s.mean), np.nan) - out[indices] = params["sigma8"] * np.arange(len(indices)) - if hole is not None: - out[hole] = np.nan - return out - return theory_fn +def test_inputs_under_two_stamps_are_refused(blinds, tmp_path): + a = sio.seal(two_bin_sacc(cl=False), blinds["TOY_OPEN"]) + b = sio.seal(two_bin_sacc(cl=False), blinds["TOY_MOCK"]) + with pytest.raises(ValueError, match="stamps"): + sio.save(cosebis_sacc(), tmp_path / "c.sacc", derived_from=[a, b]) - return factory +def test_plaintext_cannot_be_laundered_under_a_concealed_stamp(blinds, tmp_path): + """A derivation's ξ± and Cℓ_EE rows must be copies of its inputs' rows.""" + plain = two_bin_sacc() + concealed = sio.seal(plain, blinds["TOY"]) + with pytest.raises(ValueError, match="not copies"): + sio.save(plain, tmp_path / "x.sacc", derived_from=[concealed]) + assert not (tmp_path / "x.sacc").exists() + sio.save(concealed.copy(), tmp_path / "y.sacc", derived_from=[concealed]) -def test_concealing_factor_slices_its_own_block_from_a_full_length_vector(): - """The factor is the theory difference at the block's rows, and the rows - the backend does not fill are never read. - This is the shape of the whole path after the sub-SACC carving came out: - the backend is driven off the assembled SACC directly, returns a - full-length vector that is NaN everywhere but its own block, and - ``_concealing_factor`` returns exactly that block's rows. Checked against - :func:`hidden_params`, which reaches the same hidden point by an - independent route. - """ - ordered = ["xi_reporting", "cl", "rho_tau", "xi_integration"] - s = sio.gather([make_parts(nbins=1)[k] for k in ordered]) - blocks = bd._blindable_blocks(s) - assert len(blocks) == 3, "assembled file carries all three blindable blocks" - - cfg = bd.BlindingConfig() - delta = bd.hidden_params("seed", cfg)["sigma8"] - cfg.theory.ccl_params()["sigma8"] - assert delta != 0.0 - for _, indices, _ in blocks: - factor = bd._concealing_factor(s, indices, _fake_factory(indices), cfg, "seed") - assert factor.shape == (len(indices),) - assert np.allclose(factor, delta * np.arange(len(indices))) - - -def test_concealing_factor_refuses_a_row_its_backend_cannot_fill(): - """A NaN on a row the block *claims* is a layout the backend cannot cover. - - Slicing to the block drops the NaNs outside it by construction; this is - the converse guard, and it has to be explicit — without it a row the - backend silently skipped would be shifted by NaN, destroying that point - with no error anywhere. - """ - part = make_xi_part("reporting") - ((_, indices, _),) = bd._blindable_blocks(part) - with pytest.raises(ValueError, match="unfilled"): - bd._concealing_factor( - part, - indices, - _fake_factory(indices, hole=indices[0]), - bd.BlindingConfig(), - "seed", +@pytest.mark.parametrize( + "parts_under, declared", + [ + ("TOY", "TOY_OPEN"), # a stale concealed part after a reveal + ("OTHER", "TOY"), # another blind and catalogue + ("TOY_MOCK", "TOY_OPEN"), # a mock inside data + ("TOY_OPEN", "TOY"), # unblinded inside blinded + ], +) +def test_assembly_refuses_parts_under_another_custody( + blinds, tmp_path, parts_under, declared +): + part = sio.seal(two_bin_sacc(cl=False), blinds[parts_under]) + with pytest.raises(ValueError, match="declared"): + sio.save( + part.copy(), + tmp_path / "a.sacc", + derived_from=[part], + custody=blinds[declared], ) + assert not (tmp_path / "a.sacc").exists() -# --------------------------------------------------------------------------- # -# Draw-scheme binding: the blind is (seed, config, draw semantics) -# --------------------------------------------------------------------------- # -def test_draw_scheme_is_the_installed_fork_constant(): - """The recorded scheme is read from Smokescreen, not hardcoded here.""" - import smokescreen +def test_assembly_under_the_declaration(blinds, tmp_path): + part = sio.seal(two_bin_sacc(cl=False), blinds["TOY"]) + written = sio.save( + part.copy(), tmp_path / "a.sacc", derived_from=[part], custody=blinds["TOY"] + ) + assert cu.read_stamp(written.metadata).stamp == blinds["TOY"].stamp - assert bd.draw_scheme() == int(smokescreen.DRAW_SCHEME) - assert isinstance(bd.draw_scheme(), int) +# --------------------------------------------------------------------------- # +# I8: the digest binds the whole config; the commitment is the fork's +# --------------------------------------------------------------------------- # +def _variants(config): + """Configs differing from ``config`` in exactly one field, at any depth.""" + for key, value in config.envelope.items(): + yield dataclasses.replace( + config, envelope={**config.envelope, key: value + 0.01} + ) + rest = {k: v for k, v in config.envelope.items() if k != key} + yield dataclasses.replace(config, envelope={**rest, key + "_": value}) + yield dataclasses.replace(config, envelope={**config.envelope, "h": 0.01}) + for field in dataclasses.fields(TheoryConfig): + value = getattr(config.theory, field.name) + changed = value + "x" if isinstance(value, str) else value + 0.01 + yield dataclasses.replace( + config, theory=dataclasses.replace(config.theory, **{field.name: changed}) + ) -def test_assert_draw_scheme_message_names_both_versions(monkeypatch): - """A scheme mismatch says which scheme made the blind and which is installed.""" - monkeypatch.setattr(bd, "draw_scheme", lambda: 2) - bd._assert_draw_scheme(2, "the blind") # matching scheme is silent - with pytest.raises(ValueError, match=r"DRAW_SCHEME=1.*DRAW_SCHEME=2"): - bd._assert_draw_scheme(1, "the blind") - with pytest.raises(ValueError, match="no draw-scheme record"): - bd._assert_draw_scheme(None, "the blind") +def test_the_digest_changes_with_every_field(): + config = bd.BlindingConfig() + digests = [v.digest() for v in _variants(config)] + assert len(digests) == len(set(digests)) and config.digest() not in digests + assert len(digests) > len(dataclasses.fields(TheoryConfig)) -def test_unblind_refuses_a_blind_drawn_under_another_scheme(monkeypatch): - """The finding this closes: a blind added under one draw scheme and - subtracted under another silently produces a wrong data vector, because - the seed hash, the config digest and the escrow check all still pass. The - scheme must be part of the custody state, checked before any subtraction. - """ - _patch_constant_factor(monkeypatch) - blinded = bd.blind_sacc(make_xi_part("reporting"), "seed", log=_NOLOG) - assert blinded.metadata["blind_draw_scheme"] == bd.draw_scheme() - # everything else about this file is valid — only the draw semantics moved - monkeypatch.setattr( - bd, "draw_scheme", lambda: blinded.metadata["blind_draw_scheme"] + 1 +def test_the_digest_ignores_int_versus_float(): + a = bd.BlindingConfig(envelope={"S8": 0.075, "Omega_m": 0.1}) + b = bd.BlindingConfig( + envelope={"S8": 0.075, "Omega_m": 0.1}, + theory=TheoryConfig(w0=-1, wa=0, ia_bias=0), ) - with pytest.raises(ValueError, match="DRAW_SCHEME"): - bd.unblind_sacc(blinded, "seed", log=_NOLOG) - - -def test_unblind_refuses_a_file_predating_scheme_binding(monkeypatch): - """A blinded file with no scheme record fails closed, not open.""" - _patch_constant_factor(monkeypatch) - blinded = bd.blind_sacc(make_xi_part("reporting"), "seed", log=_NOLOG) - del blinded.metadata["blind_draw_scheme"] - with pytest.raises(ValueError, match="no draw-scheme record"): - bd.unblind_sacc(blinded, "seed", log=_NOLOG) + assert a.digest() == b.digest() + assert bd.BlindingConfig.from_record(json.loads(json.dumps(a.record()))) == a -def test_unblind_strips_the_scheme_stamp_with_the_rest(monkeypatch): - _patch_constant_factor(monkeypatch) - blinded = bd.blind_sacc(make_xi_part("reporting"), "seed", log=_NOLOG) - part = bd.unblind_sacc(blinded, "seed", log=_NOLOG) - assert "blind_draw_scheme" not in part.metadata - - -def test_read_seed_fails_closed_on_scheme_drift(tmp_path, monkeypatch): - """blind_part reads the seed through _read_seed, so a scheme change between - blinding part 1 and part 2 is caught before the second part is shifted.""" - bd.blind_init(str(tmp_path), log=_NOLOG) - monkeypatch.setattr(bd, "draw_scheme", lambda: 99) - with pytest.raises(ValueError, match="DRAW_SCHEME"): - bd._read_seed(str(tmp_path), bd.BlindingConfig()) +def test_the_commitment_is_the_forks(): + import smokescreen + assert cu.COMMITMENT_DOMAIN == smokescreen.COMMITMENT_DOMAIN + for seed in ("the-secret", "2112"): + assert cu.seed_commitment(seed) == smokescreen.seed_commitment(seed) -def test_stamp_passthrough_carries_the_committed_scheme(tmp_path, monkeypatch): - """A pass-through part inherits the blind's scheme, and cannot be stamped - from an install that draws differently.""" - paths = bd.blind_init(str(tmp_path), log=_NOLOG) - s = bd.stamp_concealed_passthrough(make_rho_part(), paths["commitment"]) - assert s.metadata["blind_draw_scheme"] == bd.draw_scheme() - monkeypatch.setattr(bd, "draw_scheme", lambda: 99) - with pytest.raises(ValueError, match="DRAW_SCHEME"): - bd.stamp_concealed_passthrough(make_rho_part(), paths["commitment"]) +def test_the_commitment_does_not_embed_the_rng_seed(): + """The fork seeds its RNG from the undomained sha256 of the seed. + A commitment over the bare seed would publish that RNG seed in its first + 16 hex characters, and with the public config anyone could redraw the + hidden cosmology; the domain prefix breaks that identity. + """ + import secrets -def test_assert_consistent_blind_refuses_divergent_or_foreign_schemes(monkeypatch): - """Parts blinded under different schemes never assemble; nor does a set - that agrees with itself but not with the installed fork.""" - parts = make_parts(nbins=1, with_rho=False) - for p in parts.values(): - _stamp(p) - parts["cl"].metadata["blind_draw_scheme"] = bd.draw_scheme() + 1 - with pytest.raises(ValueError, match="different blind commitments"): - bd.assert_consistent_blind(list(parts.values())) + from smokescreen.param_shifts import _normalize_seed - parts = make_parts(nbins=1, with_rho=False) - for p in parts.values(): - _stamp(p) - p.metadata["blind_draw_scheme"] = bd.draw_scheme() + 1 - with pytest.raises(ValueError, match="DRAW_SCHEME"): - bd.assert_consistent_blind(list(parts.values())) + for seed in ("my_secret_seed", "the-secret", secrets.token_hex(16)): + assert int(cu.seed_commitment(seed)[:16], 16) != _normalize_seed(seed) # --------------------------------------------------------------------------- # -# blind-init custody + assembly hash assertion (fast — encryption only) +# I9: the seed never touches disk # --------------------------------------------------------------------------- # -def test_blind_init_writes_commitment_and_encrypted_bundle_only(tmp_path): - """AC6 (init): commitment.json + encrypted bundle; never a plaintext seed.""" - paths = bd.blind_init(str(tmp_path), log=_NOLOG) - with open(paths["commitment"], encoding="utf-8") as f: - commitment = json.load(f) - assert set(commitment) == { - "label", - "seed_commitment", - "config_digest", - "draw_scheme", - } - assert len(commitment["seed_commitment"]) == 64 - assert commitment["config_digest"] == bd.BlindingConfig().config_digest() - # exactly the three custody outputs, no plaintext bundle - assert {p.name for p in tmp_path.iterdir()} == { - "commitment.json", - "blind_seed.encrpt", - "blind_seed.key", - } - # the decrypted seed matches the public commitment - bundle = bd._read_encrypted_json(paths["bundle"], paths["key"]) - assert bd.seed_commitment(bundle["seed"]) == commitment["seed_commitment"] - # one-shot custody: a second init in the same dir refuses - with pytest.raises(FileExistsError, match="refusing to overwrite"): - bd.blind_init(str(tmp_path), log=_NOLOG) +SEED = "5eed" * 8 -def test_read_seed_fails_closed_on_drifted_config(tmp_path): - bd.blind_init(str(tmp_path), log=_NOLOG) - with pytest.raises(ValueError, match="config digest"): - bd._read_seed(str(tmp_path), bd.BlindingConfig(s8_half_width=0.01)) +def _files_containing(root, needle): + return [ + p for p in root.rglob("*") if p.is_file() and needle.encode() in p.read_bytes() + ] -def _stamp(s, seed="s", label="A", config=None): - bd._stamp_provenance( - s, - bd.seed_commitment(seed), - label, - (config or bd.BlindingConfig()).config_digest(), - ) - return s +@pytest.mark.parametrize("fault", [None, "after_key", "encrypt"]) +def test_init_never_writes_the_seed(tmp_path, monkeypatch, fault): + """A normal init, or one interrupted anywhere, leaves no file holding the seed.""" + from cryptography import fernet + _cat_config(tmp_path) + monkeypatch.setattr(bd.secrets, "token_hex", lambda n: SEED) + if fault == "after_key": -def test_assert_consistent_blind_shared_stamp(): - """One commitment across all blindable parts ⇒ the shared stamp returns; - ρ/τ parts are exempt from the assertion.""" - parts = make_parts(nbins=1) - for name in ("xi_reporting", "xi_integration", "cl"): - _stamp(parts[name]) - stamp = bd.assert_consistent_blind(list(parts.values())) - assert stamp == { - "concealed": True, - "blind": "A", - "blind_commitment": bd.seed_commitment("s"), - "blind_config_digest": bd.BlindingConfig().config_digest(), - "blind_draw_scheme": bd.draw_scheme(), - } + def replace(src, dst): + raise OSError("interrupted after the key was written") + monkeypatch.setattr(bd.os, "replace", replace) + elif fault == "encrypt": -def test_assert_consistent_blind_fails_closed(): - """AC6 (assembly): mismatched commitments and mixed states both refuse.""" - parts = make_parts(nbins=1, with_rho=False) - _stamp(parts["xi_reporting"], seed="one") - _stamp(parts["xi_integration"], seed="one") - _stamp(parts["cl"], seed="two") # different seed ⇒ different commitment - with pytest.raises(ValueError, match="different blind commitments"): - bd.assert_consistent_blind(list(parts.values())) - - parts = make_parts(nbins=1, with_rho=False) - _stamp(parts["xi_reporting"]) # blinded beside plaintext blindable parts - with pytest.raises(ValueError, match="mixed"): - bd.assert_consistent_blind(list(parts.values())) - - -def test_assert_consistent_blind_all_plaintext_is_none(): - """A declared-mock plaintext assembly (nothing blinded) asserts nothing.""" - assert bd.assert_consistent_blind(list(make_parts().values())) is None - - -def test_assert_consistent_blind_unconcealed_data_fails_closed(): - """PRD §4 "Mocks vs data": an unconcealed blindable part may assemble only - if its metadata declares ``type == "mock"`` — an unconcealed ``data`` part, - or one missing the tag, fails closed (skipping the blind can never - silently expose real data). Mixed concealed/plaintext still fails on the - mixed guard regardless of type.""" - parts = make_parts(nbins=1, with_rho=False) - parts["xi_integration"].metadata["type"] = "data" - with pytest.raises(ValueError, match="type"): - bd.assert_consistent_blind(list(parts.values())) - - parts = make_parts(nbins=1, with_rho=False) - del parts["cl"].metadata["type"] # missing tag counts as not-a-mock - with pytest.raises(ValueError, match=""): - bd.assert_consistent_blind(list(parts.values())) - - # concealed data parts assemble integration (that is the whole point of the blind) - parts = make_parts(nbins=1, with_rho=False) - for p in parts.values(): - p.metadata["type"] = "data" - _stamp(p) - assert bd.assert_consistent_blind(list(parts.values()))["concealed"] is True - - # mixed concealed/plaintext fails on the mixed guard even for mocks - parts = make_parts(nbins=1, with_rho=False) - _stamp(parts["xi_reporting"]) - with pytest.raises(ValueError, match="mixed"): - bd.assert_consistent_blind(list(parts.values())) - - -def test_gather_fails_closed_on_unconcealed_data_part(): - """The type guard reaches the terminal gather surface too.""" - parts = make_parts(nbins=1, with_rho=False) - parts["xi_reporting"].metadata["type"] = "data" - with pytest.raises(ValueError, match="type"): - sio.gather(list(parts.values())) - - -def test_assert_consistent_blind_differing_labels_warn_not_fail(): - """Same seed+config, different --label ⇒ assemble cleanly with a warning. - - The label is provenance, not custody state: parts blinded under one blind - but tagged with different labels must not be misread as different blinds. - The assembly succeeds (keyed on commitment+digest); a distinct warning - surfaces the label divergence rather than a false "different commitments". - """ - parts = make_parts(nbins=1, with_rho=False) - _stamp(parts["xi_reporting"], seed="one", label="A") - _stamp(parts["xi_integration"], seed="one", label="A") - _stamp(parts["cl"], seed="one", label="B") # same blind, different label - with pytest.warns(UserWarning, match="different labels"): - stamp = bd.assert_consistent_blind(list(parts.values())) - assert stamp["blind_commitment"] == bd.seed_commitment("one") - assert stamp["blind"] == "A" # deterministic: sorted-first label - - -def test_gather_assembles_parts_and_stamps_blind(): - """The sacc_io gather combines parts (points, tags, covariance blocks), - calls the assembly assertion, and stamps the shared blind.""" - parts = make_parts(nbins=1) - for name in ("xi_reporting", "xi_integration", "cl"): - _stamp(parts[name]) - ordered = [parts[k] for k in ("xi_reporting", "cl", "rho_tau", "xi_integration")] - s = sio.gather(ordered, metadata={"catalogue_version": "vTEST", "type": "mock"}) - - assert len(s.mean) == sum(len(p.mean) for p in ordered) - assert np.array_equal(np.array(s.mean), np.concatenate([p.mean for p in ordered])) - # covariance: block-diagonal of the parts, in order - cursor = 0 - for p in ordered: - n = len(p.mean) - assert np.array_equal( - s.covariance.dense[cursor : cursor + n, cursor : cursor + n], - p.covariance.dense, - ) - cursor += n - # the integration rows are addressable by the grid tag in the assembled file - assert len(s.indices(sio.XI_PLUS, grid="integration")) == len( - parts["xi_integration"].indices(sio.XI_PLUS, grid="integration") - ) - # bandpower windows survive assembly - assert s.get_bandpower_windows(s.indices(sio.CL_EE)) is not None - # blind stamp on the assembled file - assert s.metadata["concealed"] is True - assert s.metadata["blind_commitment"] == bd.seed_commitment("s") - assert s.metadata["catalogue_version"] == "vTEST" + def encrypt(self, data): + raise RuntimeError("interrupted while encrypting") + monkeypatch.setattr(fernet.Fernet, "encrypt", encrypt) -def test_gather_fails_closed_on_mismatched_blinds(): - parts = make_parts(nbins=1, with_rho=False) - _stamp(parts["xi_reporting"], seed="one") - _stamp(parts["xi_integration"], seed="two") - _stamp(parts["cl"], seed="one") - with pytest.raises(ValueError, match="different blind commitments"): - sio.gather(list(parts.values())) + if fault is None: + _init(tmp_path, "toy", "TOY") + assert (tmp_path / "blinds" / "toy" / "key").exists() + else: + with pytest.raises((OSError, RuntimeError)): + _init(tmp_path, "toy", "TOY") + assert _files_containing(tmp_path, SEED) == [] -def test_cli_blind_init_refuses_existing_state(tmp_path): - """The blind-init CLI refuses to overwrite a previous blind's state.""" - import importlib.util +def test_init_refuses_existing_state(fresh): + with pytest.raises(cu.CustodyError, match="exists"): + _init(fresh, "toy", "OTHER") + with pytest.raises(cu.CustodyError, match="already covered"): + _init(fresh, "again", "TOY") + with pytest.raises(cu.CustodyError, match="blinded first"): + _init(fresh, "open", "TOY_OPEN") - script = ( - pathlib.Path(__file__).resolve().parents[3] / "scripts" / "blind_data_vector.py" - ) - spec = importlib.util.spec_from_file_location("_blind_cli", script) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) - (tmp_path / "commitment.json").write_text("{}") - with pytest.raises(SystemExit, match="refusing to overwrite"): - cli.main(["blind-init", str(tmp_path)]) +def test_the_record_is_read_only(fresh): + record = fresh / "blinds" / "toy" + for name in ("commitment.json", "seed.fernet", "key"): + assert not os.stat(record / name).st_mode & ( + stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH + ) # --------------------------------------------------------------------------- # -# AC2 + AC3 + AC7: the shift itself (slow — fork + CCL) +# I10: an opened blind is the one committed # --------------------------------------------------------------------------- # -@pytest.mark.slow -@pytest.mark.parametrize( - "transfer_function", - ["boltzmann_camb", "eisenstein_hu"], - ids=["default-camb", "non-default-eh"], -) -def test_ac2_on_file_shift_equals_theory_difference_per_part(transfer_function): - """AC2: per-row shift on each blinded part == theory_fn(hidden) − - theory_fn(fiducial), hidden recovered by re-running the fork's draw — - for all three parts, on a two-bin fixture; and the recovered hidden - cosmology is identical across the three parts (one seed → one hidden). - - Scope: this verifies fork-draw recovery + placement (the shift on the - file is exactly what re-running the same backend at the recovered - hidden/fiducial points predicts, at the recorded rows). It is NOT a - backend-correctness test: both sides run the identical ``theory_fn``, so - any wrong-cosmology dependence cancels and a wrong backend would still - pass here. Backend correctness is carried by AC3. - - Parametrized over the Boltzmann backend (#280): the default CAMB route - and one non-default (Eisenstein–Hu — cheap, no CAMB run) both thread the - same ``transfer_function`` knob through all three theory backends.""" - cfg = bd.BlindingConfig.from_overrides( - {"theory": {"transfer_function": transfer_function}} - ) - seed = "ac2-seed" - parts = make_parts(nbins=2, with_rho=False) - - hiddens, worst = [], 0.0 - for name, part in parts.items(): - blinded = bd.blind_sacc(part, seed, config=cfg, log=_NOLOG) - hidden = bd.hidden_params(seed, cfg) # recovered per part - hiddens.append(hidden) - fiducial = cfg.theory.ccl_params() - ((block_name, indices, factory),) = bd._blindable_blocks(part) - # Independent of the blinding path: the factory is driven directly off - # the part, at the hidden point recovered by hidden_params, and the two - # theory vectors are differenced here rather than by the fork. The - # factory fills only its own block, so slice to it. - theory = factory(part, cfg.theory) - expected = (theory(hidden) - theory(fiducial))[indices] - actual = np.array(blinded.mean)[indices] - np.array(part.mean)[indices] - gap = np.max(np.abs(actual - expected)) - scale = np.max(np.abs(expected)) - worst = max(worst, gap / scale) - assert gap <= 1e-10 * max(scale, 1e-30), f"{name}/{block_name}: |Δ|={gap:.3e}" - # one seed → one hidden cosmology across all parts - assert hiddens[0] == hiddens[1] == hiddens[2] - print(f"\nAC2 max relative shift mismatch across parts: {worst:.3e}") - - -@pytest.mark.slow -def test_ac3_cross_backend_against_independent_ccl_reference(): - """AC3: the realized shift matches an independently written direct-CCL - reference (self-contained here; does not touch the blinding backends).""" - import pyccl as ccl +def _edit(path, edit): + os.chmod(path, 0o644) + record = json.loads(path.read_text()) + edit(record) + path.write_text(json.dumps(record)) - cfg = bd.BlindingConfig() - seed = "ac3-seed" - reporting = make_xi_part("reporting", nbins=2) - cl_part = make_cl_part(nbins=2) - blinded_xi = bd.blind_sacc(reporting, seed, config=cfg, log=_NOLOG) - blinded_cl = bd.blind_sacc(cl_part, seed, config=cfg, log=_NOLOG) - hidden = bd.hidden_params(seed, cfg) - fiducial = cfg.theory.ccl_params() - - # ----- independent reference (from scratch; same fixture n(z), θ, ℓ) ---- - def ref_cosmo(params): - return ccl.Cosmology( - **params, - matter_power_spectrum="camb", - extra_parameters={ - "camb": {"halofit_version": "mead2020_feedback", "HMCode_logT_AGN": 7.5} - }, - ) - ell = np.unique( - np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)]).astype(float) - ) +def test_open_blind_verifies_the_record(fresh): + blind = bd.open_blind(_custody(fresh, "TOY")) + assert blind.name == "toy" and blind.seed not in repr(blind) - def ref_xi(params, nz_i, nz_j, theta): - cosmo = ref_cosmo(params) - ti = ccl.WeakLensingTracer(cosmo, dndz=nz_i) - tj = ccl.WeakLensingTracer(cosmo, dndz=nz_j) - cl = ccl.angular_cl(cosmo, ti, tj, ell) - xip = ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta / 60.0, type="GG+") - xim = ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta / 60.0, type="GG-") - return xip, xim - - worst = 0.0 - for i, j in bd.xi_pairs(reporting, "reporting"): - tr = sio._pair((i, j)) - theta = sio._tag(reporting, sio.XI_PLUS, tr, "theta", grid="reporting") - nz_i, nz_j = sio.get_nz(reporting, i), sio.get_nz(reporting, j) - xip_h, xim_h = ref_xi(hidden, nz_i, nz_j, theta) - xip_f, xim_f = ref_xi(fiducial, nz_i, nz_j, theta) - for dt, ref_shift in ( - (sio.XI_PLUS, xip_h - xip_f), - (sio.XI_MINUS, xim_h - xim_f), - ): - idx = reporting.indices(dt, tr, grid="reporting") - realized = np.array(blinded_xi.mean)[idx] - np.array(reporting.mean)[idx] - worst = max(worst, np.max(np.abs(realized - ref_shift))) - print(f"\nAC3 max |realized − independent reference| (reporting ξ±): {worst:.3e}") - assert worst < 1e-8 # observed ~1e-10; factor magnitudes ~1e-6 - - # pseudo-Cℓ: W @ ΔCℓ_EE against the same independent reference - for i, j in bd.cl_pairs(cl_part): - tr = sio._pair((i, j)) - idx = cl_part.indices(sio.CL_EE, tr) - window = cl_part.get_bandpower_windows(idx) - w_ell = np.asarray(window.values, dtype=float) - w_mat = np.asarray(window.weight, dtype=float) - nz_i, nz_j = sio.get_nz(cl_part, i), sio.get_nz(cl_part, j) - - def ref_cl(params): - cosmo = ref_cosmo(params) - ti = ccl.WeakLensingTracer(cosmo, dndz=nz_i) - tj = ccl.WeakLensingTracer(cosmo, dndz=nz_j) - return ccl.angular_cl(cosmo, ti, tj, w_ell) - - ref_shift = w_mat.T @ (ref_cl(hidden) - ref_cl(fiducial)) - realized = np.array(blinded_cl.mean)[idx] - np.array(cl_part.mean)[idx] - assert np.max(np.abs(realized - ref_shift)) < 1e-12 # bandpowers ~1e-9 - - -@pytest.mark.slow -def test_ac7_reproducibility_same_seed_same_shift(): - """AC7: two blind runs of the same part with the same (seed, config) - produce identical shifts; a different seed produces a different one.""" - part = make_xi_part("reporting") - b1 = bd.blind_sacc(part, "repro-seed", log=_NOLOG) - b2 = bd.blind_sacc(part, "repro-seed", log=_NOLOG) - assert np.array_equal(np.array(b1.mean), np.array(b2.mean)) - b3 = bd.blind_sacc(part, "other-seed", log=_NOLOG) - assert not np.array_equal(np.array(b1.mean), np.array(b3.mean)) + +@pytest.mark.parametrize( + "tamper", + ["wrong_key", "config", "config_and_digest", "scheme", "name"], +) +def test_open_blind_refuses_a_tampered_record(fresh, tamper): + from cryptography.fernet import Fernet + + record = fresh / "blinds" / "toy" + commitment = record / "commitment.json" + if tamper == "wrong_key": + os.chmod(record / "key", 0o644) + (record / "key").write_bytes(Fernet.generate_key()) + elif tamper == "config": + _edit(commitment, lambda r: r["config"]["envelope"].update(S8=0.3)) + elif tamper == "config_and_digest": + + def edit(r): + r["config"]["envelope"]["S8"] = 0.3 + r["config_digest"] = bd.BlindingConfig.from_record(r["config"]).digest() + + _edit(commitment, edit) + elif tamper == "scheme": + _edit(commitment, lambda r: r.update(draw_scheme=r["draw_scheme"] + 1)) + elif tamper == "name": + os.rename(record, fresh / "blinds" / "toy2") + _edit( + fresh / "blinds" / "toy2" / "commitment.json", + lambda r: r.update(blind="toy2"), + ) + (fresh / "blinds" / "toy2" / "bases").write_text("TOY\n") + with pytest.raises(cu.CustodyError): + bd.open_blind(_custody(fresh, "TOY")) # --------------------------------------------------------------------------- # -# AC1, AC4, AC5, AC9: born-blinded derived statistics (slow + cosmo_numba) +# I11: the audit proves blinded − true = shift(seed) # --------------------------------------------------------------------------- # -@pytest.mark.slow -def test_ac1_zero_shift_is_identity(): - """AC1: a zero envelope reproduces every part exactly, and the integration part - run downstream through the b_modes seams yields COSEBIs and pure-E/B - identical to the unblinded run — the per-part plumbing and the - born-blinded derivation path are the identity at zero shift.""" - pytest.importorskip("cosmo_numba") - zero = bd.BlindingConfig(s8_half_width=0.0, omega_m_half_width=0.0) - parts = make_parts(nbins=1, with_rho=False) - blinded = { - name: bd.blind_sacc(p, "any-seed", config=zero, log=_NOLOG) - for name, p in parts.items() - } - for name, part in parts.items(): - assert np.array_equal(np.array(part.mean), np.array(blinded[name].mean)), ( - f"zero-shift blind changed {name}" - ) - # derived statistics downstream: identical inputs ⇒ identical numbers - En_t, Bn_t, modes_t = _derive_downstream( - parts["xi_reporting"], parts["xi_integration"] - ) - En_b, Bn_b, modes_b = _derive_downstream( - blinded["xi_reporting"], blinded["xi_integration"] - ) - assert np.array_equal(En_t, En_b) and np.array_equal(Bn_t, Bn_b) - for key in modes_t: - t, b = modes_t[key], modes_b[key] - both_nan = np.isnan(t) & np.isnan(b) - assert np.array_equal(t[~both_nan], b[~both_nan]), key - assert np.array_equal(np.isnan(t), np.isnan(b)), key - - -@pytest.mark.slow -def test_ac4_b_mode_invariance_and_leakage_floor(): - """AC4: the ΔBₙ induced by deriving B-modes from the blinded integration ξ± is - independent of the injected B amplitude (a fixed absolute E→B leakage - offset, not fractional). The magnitude is measured and reported, never - asserted against a constant.""" - pytest.importorskip("cosmo_numba") - seed = "ac4-seed" - deltas, reports = [], [] - for amp in (2e-6, 2e-5): - reporting = make_xi_part("reporting") - integration = make_xi_part("integration", b_amplitude=amp) - blinded_reporting = bd.blind_sacc(reporting, seed, log=_NOLOG) - blinded_integration = bd.blind_sacc(integration, seed, log=_NOLOG) - _, Bn_t, modes_t = _derive_downstream(reporting, integration) - _, Bn_b, modes_b = _derive_downstream(blinded_reporting, blinded_integration) - d_bn = Bn_b - Bn_t - d_xib = modes_b["xip_B"] - modes_t["xip_B"] - finite = np.isfinite(d_xib) - deltas.append((d_bn, d_xib[finite])) - reports.append( - f"B={amp:.0e}: max|ΔBₙ|={np.max(np.abs(d_bn)):.3e} " - f"(ΔBₙ/Bₙ={np.max(np.abs(d_bn)) / np.max(np.abs(Bn_t)):.2e}), " - f"max|Δξ+_B|={np.max(np.abs(d_xib[finite])):.3e} " - f"({np.max(np.abs(d_xib[finite])) / amp:.2%} of injected B)" - ) - print("\nAC4 " + "\nAC4 ".join(reports)) - - (d_bn_1, d_xib_1), (d_bn_2, d_xib_2) = deltas - scale = max(np.max(np.abs(d_bn_1)), 1e-30) - gap = np.max(np.abs(d_bn_1 - d_bn_2)) - print( - f"AC4 ΔBₙ amplitude-independence: max|ΔBₙ(2e-6) − ΔBₙ(2e-5)| = " - f"{gap:.3e} ({gap / scale:.2e} of |ΔBₙ|)" - ) - assert gap <= 1e-9 * scale + 1e-24, ( - "ΔBₙ depends on the injected B amplitude — the shift is not pure E" - ) - # The pure-ξ_B leakage is also amplitude-independent, but only to the - # adaptive-quadrature floor: cosmo_numba's Schneider integrals subdivide - # adaptively, so the estimator is not bit-linear in its inputs and the - # two runs differ at a small fraction of the (tiny) leakage itself. The - # COSEBIs assertion above carries the exact-identity criterion; this one - # bounds the quadrature wobble. - scale_x = max(np.max(np.abs(d_xib_1)), 1e-30) - gap_x = np.max(np.abs(d_xib_1 - d_xib_2)) - print( - f"AC4 Δξ+_B amplitude-independence: {gap_x:.3e} " - f"({gap_x / scale_x:.2e} of the leakage)" - ) - assert gap_x <= 0.05 * scale_x - - -@pytest.mark.slow -def test_ac5_untouched_blocks_and_row_order(): - """AC5: each part's covariance byte-identical; the Cℓ BB/EB rows and the - ρ/τ part never blinded; every part's shifted rows land at their original - within-part indices (order-preservation).""" - parts = make_parts(nbins=1) - seed = "ac5-seed" - blinded = { - name: bd.blind_sacc(parts[name], seed, log=_NOLOG) - for name in ("xi_reporting", "xi_integration", "cl") - } - for name, b in blinded.items(): - part = parts[name] - assert np.array_equal(b.covariance.dense, part.covariance.dense), name - # row order: the identity of every row (type/tracers/tags) unchanged - for a, c in zip(part.data, b.data): - assert (a.data_type, a.tracers, a.tags) == (c.data_type, c.tracers, c.tags) - # BB/EB rows of the Cℓ part untouched (pure E-mode shift) - for dt in (sio.CL_BB, sio.CL_EB): - idx = parts["cl"].indices(dt) - assert np.array_equal( - np.array(blinded["cl"].mean)[idx], np.array(parts["cl"].mean)[idx] - ), f"{dt} was touched by the blind" - # the blindable rows did move (the blind actually blinded) - for name, grid in ( - ("xi_reporting", "reporting"), - ("xi_integration", "integration"), +def _reveal_pair(root, *, patch_centers=("a", "a")): + """A concealed part in an archive and its true twin at the same path.""" + blinded = _custody(root, "TOY") + seed = bd.open_blind(blinded).seed + (root / "blinds" / "toy" / "revealed.json").write_text(json.dumps({"seed": seed})) + true = cu.Custody("unblinded", "TOY") + s = two_bin_sacc() + for tree, custody, centers in ( + ("archive", blinded, patch_centers[0]), + ("live", true, patch_centers[1]), ): - idx = bd._xi_indices(parts[name], grid) - assert not np.allclose( - np.array(blinded[name].mean)[idx], np.array(parts[name].mean)[idx], atol=0 - ) - # ρ/τ: refused by blind_sacc (test_blind_refuses_non_blindable_part) and - # exempt in assembly — pass through gather untouched - s = sio.gather( - [ - blinded["xi_reporting"], - parts["rho_tau"], - blinded["cl"], - blinded["xi_integration"], - ] - ) - idx = s.indices(sio.RHO_PLUS.format(k=0)) - assert np.array_equal( - np.array(s.mean)[idx], - np.array(parts["rho_tau"].mean)[ - parts["rho_tau"].indices(sio.RHO_PLUS.format(k=0)) - ], + part = s.copy() + part.metadata["patch_centers_sha256"] = centers + (root / tree / "sub").mkdir(parents=True) + sio.save(part, root / tree / "sub" / "part.sacc", custody=custody) + return seed + + +def _audit(root): + return bd.audit( + "toy", + archive=root / "archive", + true_root=root / "live", + cat_config=root / "cat_config.yaml", ) -@pytest.mark.slow -def test_ac9_pure_eb_nan_parity_under_blind(): - """AC9: the pure-E/B NaN pattern born from the blinded parts is identical - to the true parts' — blinding never moves a NaN. - - The Schneider estimator returns NaN wherever a reporting point lacks - interior support against the edge-based integration bounds. Whatever - that pattern is on the true parts (the fixture puts the outermost - reporting point at the boundary, so it is non-empty here; on production - files it is empty), the blinded derivation must reproduce it bit-for-bit: - the blind is a pure shift of the estimator's inputs, not a change of - estimator support. The finite values move (the ξ± shifted); the NaN mask - does not.""" - pytest.importorskip("cosmo_numba") - seed = "ac9-seed" - reporting, integration = make_xi_part("reporting"), make_xi_part("integration") - _, _, modes_t = _derive_downstream(reporting, integration) - _, _, modes_b = _derive_downstream( - bd.blind_sacc(reporting, seed, log=_NOLOG), - bd.blind_sacc(integration, seed, log=_NOLOG), +def test_the_audit_passes_on_a_true_reveal(fresh): + _reveal_pair(fresh) + report = _audit(fresh) + assert report["ok"], report + ((path, part),) = report["parts"].items() + assert path == "sub/part.sacc" and part["residual"] <= 1e-6 + assert ( + abs(report["shift"]["S8"]) <= 0.075 and abs(report["shift"]["Omega_m"]) <= 0.1 ) - for key in modes_t: - t, b = modes_t[key], modes_b[key] - assert np.array_equal(np.isnan(t), np.isnan(b)), ( - f"blinding moved the pure-E/B NaN pattern for {key}" - ) - # the finite values did move (the blind actually shifted the ξ±) - t, b = modes_t["xip_E"], modes_b["xip_E"] - finite = np.isfinite(t) - assert finite.any() and not np.allclose(t[finite], b[finite], atol=0) -# --------------------------------------------------------------------------- # -# AC6 + AC8: end-to-end custody through the file surface (slow + cosmo_numba) -# --------------------------------------------------------------------------- # -@pytest.mark.slow -def test_ac6_ac8_end_to_end_init_parts_gather_unblind(tmp_path): - """AC8: blind-init → blind-part on each intermediate part → terminal - gather (hash assertion passes, stamp lands) → unblind restores each part - bit-for-bit, and the derived statistics re-derived from the unblinded - integration part reproduce the truth. AC6: no plaintext part or seed survives - on disk, no ``seed_smokescreen`` key; unblind fails closed on a tampered - commitment.""" - pytest.importorskip("cosmo_numba") - parts = make_parts(nbins=1) - En_true, Bn_true, modes_true = _derive_downstream( - parts["xi_reporting"], parts["xi_integration"] - ) +def test_the_audit_fails_on_a_wrong_seed(fresh): + _reveal_pair(fresh) + (fresh / "blinds" / "toy" / "revealed.json").write_text(json.dumps({"seed": "x"})) + assert not _audit(fresh)["ok"] - blind_dir = tmp_path / "blind" - blind_dir.mkdir() - init = bd.blind_init(str(blind_dir), log=_NOLOG) - - part_files, out_paths = {}, {} - for name in ("xi_reporting", "xi_integration", "cl"): - path = tmp_path / f"{name}.fits" - sio.save(parts[name], str(path), type="mock") - out_paths[name] = bd.blind_part(str(path), str(blind_dir), log=_NOLOG) - part_files[name] = path - - # -- custody hygiene (AC6) --------------------------------------------- -- - for name, path in part_files.items(): - assert not path.exists(), f"plaintext part {name} was not deleted" - blinded = sio.load(out_paths[name]["blinded"]) - assert blinded.metadata["concealed"] is True - assert "seed_smokescreen" not in blinded.metadata - assert pathlib.Path(out_paths[name]["escrow"]).exists() - assert not np.array_equal(np.array(blinded.mean), np.array(parts[name].mean)) - with open(init["commitment"], encoding="utf-8") as f: - commitment = json.load(f) - assert set(commitment) == { - "label", - "seed_commitment", - "config_digest", - "draw_scheme", - } - blinded_parts = {n: sio.load(p["blinded"]) for n, p in out_paths.items()} - for b in blinded_parts.values(): - assert b.metadata["blind_commitment"] == commitment["seed_commitment"] - # no plaintext json anywhere beside the blind outputs - assert not list(tmp_path.rglob("*escrow.json")) - assert not (blind_dir / "blind_seed.json").exists() - - # -- terminal assembly: hash assertion + stamp (AC8) -------------------- -- - assembled = sio.gather( - [ - blinded_parts["xi_reporting"], - blinded_parts["cl"], - parts["rho_tau"], - blinded_parts["xi_integration"], - ], - metadata={"catalogue_version": "vTEST", "type": "mock"}, - ) - assert assembled.metadata["blind_commitment"] == commitment["seed_commitment"] - # born-blinded derived statistics from the blinded parts differ from truth - En_b, Bn_b, _ = _derive_downstream( - blinded_parts["xi_reporting"], blinded_parts["xi_integration"] - ) - assert not np.allclose(En_b, En_true, atol=0) - - # -- fail-closed on tampered commitment (AC6) --------------------------- -- - tampered = dict(commitment, seed_commitment="0" * 64) - with open(init["commitment"], "w", encoding="utf-8") as f: - json.dump(tampered, f) - with pytest.raises(ValueError, match="committed seed commitment"): - bd.unblind_part( - out_paths["xi_integration"]["blinded"], - str(blind_dir), - str(tmp_path / "never.fits"), - log=_NOLOG, - ) - with open(init["commitment"], "w", encoding="utf-8") as f: - json.dump(commitment, f) - with pytest.raises(ValueError, match="config digest"): - bd.unblind_part( - out_paths["xi_integration"]["blinded"], - str(blind_dir), - str(tmp_path / "never.fits"), - config=bd.BlindingConfig(s8_half_width=0.01), - log=_NOLOG, - ) - # -- bit-for-bit restoration per part (AC8) ----------------------------- -- - restored = {} - for name in ("xi_reporting", "xi_integration", "cl"): - out = tmp_path / f"{name}_restored.fits" - bd.unblind_part( - out_paths[name]["blinded"], str(blind_dir), str(out), log=_NOLOG - ) - restored[name] = sio.load(str(out)) - assert np.array_equal( - np.array(restored[name].mean), np.array(parts[name].mean) - ), f"{name} not restored bit-for-bit" - assert not restored[name].metadata.get("concealed", False) - assert "blind_commitment" not in restored[name].metadata - - # unblinding then re-deriving reproduces the true derived statistics - En_r, Bn_r, modes_r = _derive_downstream( - restored["xi_reporting"], restored["xi_integration"] - ) - assert np.array_equal(En_r, En_true) and np.array_equal(Bn_r, Bn_true) - for key in modes_true: - t, r = modes_true[key], modes_r[key] - both_nan = np.isnan(t) & np.isnan(r) - assert np.array_equal(t[~both_nan], r[~both_nan]), key - assert np.array_equal(np.isnan(t), np.isnan(r)), key - - -def test_ac8_dotted_versioned_part_names_escrow_and_restore(tmp_path): - """AC8 under the canonical catalogue-version naming (dotted stems). - - Production part files carry the versioned name ``v1.4.6.3_xi_reporting.fits`` - etc. ``smokescreen.encryption.encrypt_file`` names its outputs from - ``basename.split('.')[0]``, so both these parts would misfile onto - ``v1.encrpt``/``v1.key`` and the second would silently overwrite the - first's escrowed truth. Guard: the escrow lands at the exact - :func:`part_paths` name, two dot-prefix-sharing parts do not collide, and - each restores bit-for-bit.""" - parts = make_parts(nbins=1) - blind_dir = tmp_path / "blind" - blind_dir.mkdir() - bd.blind_init(str(blind_dir), log=_NOLOG) - - version = "v1.4.6.3" - out_paths, part_files = {}, {} - for name in ("xi_reporting", "xi_integration"): - path = tmp_path / f"{version}_{name}.fits" - sio.save(parts[name], str(path), type="mock") - out_paths[name] = bd.blind_part(str(path), str(blind_dir), log=_NOLOG) - part_files[name] = path - - # escrow bundles landed at the declared names (no split('.') truncation), - # and the two dot-prefix-sharing parts did not collide onto one bundle. - escrow_files = {n: p["escrow"] for n, p in out_paths.items()} - assert escrow_files["xi_reporting"] != escrow_files["xi_integration"] - for name, path in part_files.items(): - assert not path.exists(), f"plaintext part {name} was not deleted" - assert pathlib.Path(out_paths[name]["escrow"]).exists(), name - assert pathlib.Path(out_paths[name]["escrow_key"]).exists(), name - # the truncated-name collision target must not exist - assert not (tmp_path / "v1.encrpt").exists() - assert not (tmp_path / "v1.key").exists() - assert not list(tmp_path.rglob("*escrow.json")) - - # each part restores bit-for-bit via its own escrow (not subtraction-only) - for name in ("xi_reporting", "xi_integration"): - out = tmp_path / f"{version}_{name}_restored.fits" - bd.unblind_part( - out_paths[name]["blinded"], str(blind_dir), str(out), log=_NOLOG - ) - restored = sio.load(str(out)) - assert np.array_equal(np.array(restored.mean), np.array(parts[name].mean)), ( - f"{name} not restored bit-for-bit" - ) +def test_the_audit_fails_on_other_patch_centres(fresh): + _reveal_pair(fresh, patch_centers=("a", "b")) + assert not _audit(fresh)["ok"] diff --git a/src/sp_validation/tests/test_blinding_wiring.py b/src/sp_validation/tests/test_blinding_wiring.py deleted file mode 100644 index c5e01029..00000000 --- a/src/sp_validation/tests/test_blinding_wiring.py +++ /dev/null @@ -1,389 +0,0 @@ -"""Tests for the Snakemake blind-at-birth wiring, independent of a live cluster. - -Covers ``workflow/common.py``'s part-path and run-type helpers, and the -data-run fail-closed assembly: ``assemble_sacc`` must refuse an unblinded -``type='data'`` part and succeed once every part is concealed under one -commitment. A candide-only test additionally asserts the blinding subgraph -resolves in the cosmo_val DAG dry-run. -""" - -import importlib.util -import json -import os -import re -import subprocess -import sys -import types -from pathlib import Path - -import numpy as np -import pytest - -from sp_validation import blinding -from sp_validation import sacc_io as sio -from sp_validation.cosmo_val import sacc_writers as sw - - -def _repo_root(): - return next( - p for p in Path(__file__).resolve().parents if (p / "pyproject.toml").exists() - ) - - -def _load_module(rel_path, name): - """Import a workflow module/script by file path (off the package path).""" - path = _repo_root() / rel_path - spec = importlib.util.spec_from_file_location(name, path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -common = _load_module("workflow/common.py", "wf_common") -asm = _load_module("workflow/scripts/assemble_sacc.py", "assemble_sacc") - - -# --------------------------------------------------------------------------- # -# 1. common.py part-path and run-type helpers -# --------------------------------------------------------------------------- # -_STEMS = [ - "SP_v1.4.6.3_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=100", - "SP_v1.4.6.3_leak_corr_xi_minsep=0.08_maxsep=300.0_nbins=1000_npatch=1", - "pseudo_cl_analysis_SP_v1.4.6.3_powspace_nbins=32", - "pseudo_cl_analysis_SP_v1.4.6.3_leak_corr_powspace_nbins=32", -] - - -@pytest.mark.parametrize( - "stem,expected", - [ - (_STEMS[0], "SP_v1.4.6.3"), - (_STEMS[1], "SP_v1.4.6.3_leak_corr"), - (_STEMS[2], "SP_v1.4.6.3"), - (_STEMS[3], "SP_v1.4.6.3_leak_corr"), - ], -) -def test_version_of_extracts_catalogue_version(stem, expected): - assert common.version_of(stem) == expected - - -def test_version_of_raises_without_version(): - with pytest.raises(ValueError, match="no catalogue version"): - common.version_of("cosebis_no_version_here") - - -def test_blindable_part_switches_on_run_type(monkeypatch): - part = "/out/SP_v1.4.6.3_xi_minsep=0.08_maxsep=300_nbins=1000_npatch=1.sacc" - monkeypatch.setattr(common, "RUN_TYPE", "data") - assert common.blindable_part(part) == common.blinded_path(part) - monkeypatch.setattr(common, "RUN_TYPE", "mock") - assert common.blindable_part(part) == part - - -# --------------------------------------------------------------------------- # -# 2. Data-run fail-closed assembly (#252 terminal custody gate) -# --------------------------------------------------------------------------- # -META = {"catalogue_version": "vSYNTH", "npatch": 1} -# Two arbitrary-but-consistent hex stamps standing in for a real blind's -# seed commitment / config digest; the assembly only checks they agree across parts. -_COMMIT = "a" * 64 -_DIGEST = "b" * 64 - - -def _nz(): - return np.linspace(0.01, 2.0, 40), np.random.default_rng(0).uniform(0.1, 1.0, 40) - - -def _spd(n, seed): - a = np.random.default_rng(seed).normal(size=(n, n)) - return a @ a.T + n * np.eye(n) - - -def _xi_cov(tmp_path, n_theta=6): - """The analytic ξ± covariance assembly injects, as a CosmoCov-format .txt. - - Custody is what these tests are about, and the injected covariance carries - none of it — it is a theory product — so any SPD block of the right size - stands in. - """ - path = tmp_path / "xi_cov_processed.txt" - np.savetxt(str(path), _spd(2 * n_theta, 77)) - return str(path) - - -def _data_parts(tmp_path, *, conceal, one_plaintext=False, run_type="data"): - """Write the five per-statistic parts, stamped ``type=run_type``. - - ``conceal`` stamps every part with the shared blind (concealed=True). With - ``one_plaintext`` the ξ± reporting part is left unconcealed — a blinded / - plaintext mix the assembly must refuse. ``run_type='mock'`` writes the parts - as a mock campaign's producers do, which is the only way an unconcealed - blindable part is allowed through the assembly. - """ - nz = {0: _nz()} - theta = np.geomspace(1.0, 100.0, 6) - - xi = sw.xi_to_sacc( - nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) - xi.add_covariance(_spd(len(xi.mean), 1)) - co = sw.cosebis_to_sacc( - nz, - META, - { - "En": np.arange(1, 6) * 1e-6, - "Bn": np.arange(1, 6) * 1e-7, - "cov": _spd(10, 3), - }, - (1.0, 100.0), - ) - eb_arrays = {k: np.arange(6) * (i + 1) * 1e-6 for i, k in enumerate(sio.PURE_KEYS)} - eb = sw.pure_eb_to_sacc(nz, META, theta, eb_arrays, covariance=_spd(36, 4)) - rho = {"theta": theta} - tau = {"theta": theta} - rng = np.random.default_rng(5) - for k in sw.RHO_K: - for s in ("p", "m"): - rho[f"rho_{k}_{s}"] = rng.normal(size=6) * 1e-6 - rho[f"varrho_{k}_{s}"] = rng.uniform(1e-14, 1e-13, 6) - for k in sw.TAU_K: - for s in ("p", "m"): - tau[f"tau_{k}_{s}"] = rng.normal(size=6) * 1e-6 - tau[f"vartau_{k}_{s}"] = rng.uniform(1e-14, 1e-13, 6) - rt = sw.rho_tau_to_sacc(nz, META, rho, tau) - - parts = {"xi_reporting": xi, "cosebis": co, "pure_eb": eb, "rho_tau": rt} - paths = {} - for name, part in parts.items(): - if conceal and not (one_plaintext and name == "xi_reporting"): - blinding._stamp_provenance(part, _COMMIT, "A", _DIGEST) - p = tmp_path / f"{name}.sacc" - sio.save(part, str(p), type=run_type) - paths[name] = str(p) - return paths - - -def test_data_assemble_fails_closed_on_unblinded_part(tmp_path): - """A data run refuses to assemble an unconcealed real part (fail closed).""" - paths = _data_parts(tmp_path, conceal=False) - with pytest.raises(ValueError, match="refusing to load an unblinded"): - asm.assemble_sacc( - "vSYNTH", - paths, - str(tmp_path / "vSYNTH.sacc"), - xi_cov=_xi_cov(tmp_path), - ) - - -def test_data_assemble_passes_on_blinded_parts(tmp_path): - """With every part concealed under one blind, the data-run assembly succeeds - and stamps the shared commitment on the terminal file.""" - paths = _data_parts(tmp_path, conceal=True) - out = tmp_path / "vSYNTH.sacc" - s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=_xi_cov(tmp_path)) - assert s.metadata["concealed"] is True - assert s.metadata["blind_commitment"] == _COMMIT - assert s.metadata["blind_config_digest"] == _DIGEST - # Round-trips through the fail-closed load gate without an escape hatch. - assert sio.load(str(out)).metadata["concealed"] is True - - -def test_data_assemble_refuses_blinded_plaintext_mix(tmp_path): - """A concealed ξ± beside a plaintext one is a custody violation — refuse.""" - paths = _data_parts(tmp_path, conceal=True, one_plaintext=True) - # The plaintext ξ± reporting part fails the load gate first (data + not - # concealed), so the mix can never even reach assembly. - with pytest.raises(ValueError, match="refusing to load an unblinded"): - asm.assemble_sacc( - "vSYNTH", - paths, - str(tmp_path / "vSYNTH.sacc"), - xi_cov=_xi_cov(tmp_path), - ) - - -def test_data_assemble_runs_behind_the_custody_guard(tmp_path, monkeypatch): - """The custody guard is reached *through* the production assembly. - - Every part here is concealed, so every part clears the fail-closed load - gate — the earlier tests all stop there. Only - ``blinding.assert_consistent_blind`` can catch what is wrong with this - assembly: the install's Smokescreen draws shifts under a different scheme - than the one that made the blind, so it could never unblind what it is - about to write. That ``assemble_sacc`` raises is the check that it runs - through :func:`sacc_io.gather` like every other assembly path, rather than - reimplementing the custody wrapper around its own assembler. - """ - paths = _data_parts(tmp_path, conceal=True) - monkeypatch.setattr(blinding, "draw_scheme", lambda: 99) - with pytest.raises(ValueError, match="DRAW_SCHEME"): - asm.assemble_sacc( - "vSYNTH", - paths, - str(tmp_path / "vSYNTH.sacc"), - xi_cov=_xi_cov(tmp_path), - ) - - -def test_data_assemble_stamps_the_draw_scheme_on_the_terminal_file(tmp_path): - """The assembled file carries the blind's draw scheme, like its parts.""" - paths = _data_parts(tmp_path, conceal=True) - out = tmp_path / "vSYNTH.sacc" - asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=_xi_cov(tmp_path)) - assert sio.load(str(out)).metadata["blind_draw_scheme"] == blinding.draw_scheme() - - -def test_mock_assemble_succeeds_without_a_blind(tmp_path): - """A mock campaign assembles its plaintext parts — the gate's mock branch is - reachable from real producers, not only from test fixtures. - - Every part writer stamps the campaign's run type (CosmologyValidation's - ``run_type``, ``run_2pcf``'s ``run_type=`` / ``--run-type``), so a ``mock`` campaign's parts declare themselves mocks and - ``assert_consistent_blind`` lets them through unconcealed. The same parts - stamped ``type='data'`` fail closed — that is - ``test_data_assemble_fails_closed_on_unblinded_part``. - """ - paths = _data_parts(tmp_path, conceal=False, run_type="mock") - out = tmp_path / "vSYNTH.sacc" - s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=_xi_cov(tmp_path)) - assert "concealed" not in s.metadata - # No escape hatch: a mock part is not gated by the fail-closed loader. - assert sio.load(str(out)).metadata["type"] == "mock" - - -def test_rho_tau_part_is_stamped_concealed_from_the_commitment(tmp_path): - """ρ/τ carries no cosmological vector, but a data run's assembly still opens - it through the fail-closed load gate — so its writer must stamp it. - - This drives the real writer (``PSFSystematicsMixin.rho_tau_to_sacc_part``) - with the commitment the ``rho_tau_stats`` rule binds on a data run, and - checks the emitted part loads without the escape hatch. Without the stamp a - data run's ``assemble_sacc`` dies on the ρ/τ part before custody is ever - checked. - """ - from sp_validation.cosmo_val.core import CosmologyValidation - from sp_validation.cosmo_val.psf_systematics import PSFSystematicsMixin - - root = tmp_path / "blind" - blind_dir = root / "vSYNTH" - blind_dir.mkdir(parents=True) - commitment = blinding.blind_init(str(blind_dir), log=lambda *_: None)["commitment"] - theta = np.geomspace(1.0, 100.0, 6) - rng = np.random.default_rng(11) - rho = {"theta": theta} - tau = {"theta": theta} - for k in sw.RHO_K: - for sign in ("p", "m"): - rho[f"rho_{k}_{sign}"] = rng.normal(size=6) * 1e-6 - rho[f"varrho_{k}_{sign}"] = rng.uniform(1e-14, 1e-13, 6) - for k in sw.TAU_K: - for sign in ("p", "m"): - tau[f"tau_{k}_{sign}"] = rng.normal(size=6) * 1e-6 - tau[f"vartau_{k}_{sign}"] = rng.uniform(1e-14, 1e-13, 6) - - class _Writer(PSFSystematicsMixin): - """The writer's collaborators, stubbed — the method under test is real.""" - - run_type = "data" - blind_root = str(root) - # The real per-version resolution is part of what this test covers. - commitment_path = CosmologyValidation.commitment_path - - def sacc_nz(self, version): - return {0: _nz()} - - def sacc_metadata(self, version): - return dict(META) - - def print_magenta(self, *args, **kwargs): - pass - - out_dir = tmp_path / "rho_tau_stats" - out_dir.mkdir() - _Writer().rho_tau_to_sacc_part( - "vSYNTH", - str(out_dir), - "vSYNTH", - types.SimpleNamespace(rho_stats=rho), - types.SimpleNamespace(tau_stats=tau), - ) - - written = sio.load(str(out_dir / "rho_tau_vSYNTH.sacc")) - assert written.metadata["concealed"] is True - assert written.metadata["blind_draw_scheme"] == blinding.draw_scheme() - with open(commitment, encoding="utf-8") as f: - committed = json.load(f) - assert written.metadata["blind_commitment"] == committed["seed_commitment"] - assert written.metadata["blind_config_digest"] == committed["config_digest"] - - -def test_assert_consistent_blind_rejects_divergent_commitments(tmp_path): - """Two ξ± parts blinded under different commitments must never combine.""" - nz = {0: _nz()} - theta = np.geomspace(1.0, 100.0, 6) - a = sw.xi_to_sacc( - nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) - b = sw.xi_to_sacc( - nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="integration" - ) - blinding._stamp_provenance(a, _COMMIT, "A", _DIGEST) - blinding._stamp_provenance(b, "c" * 64, "A", _DIGEST) - with pytest.raises(ValueError, match="different blind commitments"): - blinding.assert_consistent_blind([a, b]) - - -# --------------------------------------------------------------------------- # -# 3. The blinding subgraph resolves in the cosmo_val DAG (candide-only) -# --------------------------------------------------------------------------- # -requires_candide_data = pytest.mark.skipif( - not Path("/n17data/cdaley/unions").exists(), - reason="candide-local workflow config/data (/n17data) absent — off-cluster", -) - - -@requires_candide_data -def test_blinding_subgraph_in_cosmo_val_dry_run(): - """A data-run cosmo_val assemble pulls blind_init + blind_part, and binds the - ξ± / pseudo-Cℓ parts to their *_blinded siblings.""" - env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} - env.pop("SNAKEMAKE_PROFILE", None) - result = subprocess.run( - [ - sys.executable, - "-m", - "snakemake", - "assemble_sacc_all", - "--dry-run", - "--cores", - "1", - "--configfile", - "config/config.yaml", - ], - cwd=_repo_root() / "papers/cosmo_val", - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=180, - check=False, - ) - assert result.returncode == 0, result.stdout - out = result.stdout - assert "rule blind_init:" in out, out - assert "rule blind_part:" in out, out - # assemble consumes the blinded ξ± reporting and pseudo-Cℓ. The integration - # ξ± is not gathered into the terminal file (per the #247 ruling), but the - # COSEBIs / pure-E/B consumers now bind the blinded integration part (via - # blindable_part) to re-derive their concealed E-modes, so blind_part enters - # the subgraph for it too. - assert "_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=100_blinded.sacc" in out - # maxsep=300.0, not 300: the grid table canonicalises before it names files. - assert "_xi_minsep=0.08_maxsep=300.0_nbins=1000_npatch=1_blinded.sacc" in out - assert "pseudo_cl_analysis_SP_v1.4.6.3_powspace_nbins=32_blinded.sacc" in out - # Every blindable plaintext part is temp() on a data run, the analysis - # pseudo-Cℓ included (its own rule exists so it can be). - assert "Would remove temporary output" in out, out - assert re.search( - r"Would remove temporary output \S*pseudo_cl_analysis_\S+\.sacc", out - ), out diff --git a/src/sp_validation/tests/test_camb_ccl_crosscheck.py b/src/sp_validation/tests/test_camb_ccl_crosscheck.py index 3f286474..02414f5d 100644 --- a/src/sp_validation/tests/test_camb_ccl_crosscheck.py +++ b/src/sp_validation/tests/test_camb_ccl_crosscheck.py @@ -1,140 +1,166 @@ -"""CAMB↔CCL theory cross-check (blinding PRD AC10–14). - -The blinding shift is a difference of CCL theory vectors; downstream -inference runs CAMB (CosmoSIS). The shift only means what it is intended to -mean if CCL and CAMB predict the same ξ± at a fixed cosmology on our θ grid. -This module asserts that agreement between the two independent ξ± paths in -:mod:`sp_validation.blinding_theory`: - -- **Path A** (:func:`~sp_validation.blinding_theory.xi_ccl`): CCL-native — CCL's - Boltzmann-CAMB HMCode2020 P(k) route, projected by CCL Limber + FFTLog. -- **Path B** (:func:`~sp_validation.blinding_theory.xi_camb`): an independent - pycamb run produces the HMCode2020 ``P(k, z)`` (σ8-matched ``A_s``), - wrapped in a ``ccl.Pk2D`` and projected through the same CCL machinery. - -Because both paths route their nonlinear P(k) through CAMB's HMCode2020 and -both project through CCL, a common Limber+FFTLog bug cancels: this test -validates the **P(k) recipe** and the **σ8/A_s amplitude convention**, not -the projection. The one convention subtlety it settles: the fiducial fixes -σ8 for CCL but A_s for CAMB; a nominal ``A_s = 2.1e-9`` leaves CAMB's σ8 -≈3% off target — enough to blow a ξ± comparison to ~9–10%. +"""The blinding theory against an independent CAMB oracle (AC10–13). + +The blinding shift is a difference of CCL theory vectors; inference runs CAMB +(CosmoSIS). The shift means what it is meant to only if CCL and CAMB predict the +same ξ± at a fixed cosmology on our θ grid. This module compares +:func:`sp_validation.blinding_theory.xi_ccl` (CCL's Boltzmann-CAMB HMCode2020 +P(k), projected by CCL's Limber + FFTLog) with an oracle written here: a direct +pycamb run of the HMCode2020 ``P(k, z)`` at a σ8-matched ``A_s``, wrapped in a +``ccl.Pk2D`` and projected by the same CCL machinery. + +Both paths route P(k) through CAMB and project through CCL, so a shared +projection bug cancels: the comparison validates the P(k) recipe and the σ8/A_s +amplitude convention. The fiducial fixes σ8 for CCL but A_s for CAMB; a nominal +``A_s = 2.1e-9`` leaves CAMB's σ8 ≈3% off target, enough to move ξ± by ~10%. """ +import dataclasses import pathlib import re import numpy as np -import pytest -from sp_validation import blinding_theory as cm +from sp_validation import blinding_theory as bt -# Tolerances (AC11/AC12). Observed floor on this fixture: see the printed -# numbers in the slow tests — the tolerances sit above the floor with -# headroom; version bumps move the floor and that is not a regression. -XIP_RTOL = 0.005 # 0.5 % -XIM_RTOL = 0.010 # 1.0 % -# ξ− crosses zero on this grid: the relative assertion applies only where -# |ξ−| exceeds an absolute floor set from the fixture's peak |ξ−|. +XIP_RTOL = 0.005 +XIM_RTOL = 0.010 +# ξ− crosses zero on this grid: the relative bound applies only where |ξ−| +# exceeds this fraction of its peak. XIM_FLOOR_FRAC = 0.05 +THETA_ARCMIN = np.geomspace(5.0, 250.0, 12) +PK_ZMAX, PK_NZ = 3.0, 48 + -# --------------------------------------------------------------------------- # -# Deterministic fixture: one Gaussian source bin, 12-point θ grid -# --------------------------------------------------------------------------- # def _gauss_nz(n=400): z = np.linspace(0.01, 3.0, n) nz = np.exp(-0.5 * ((z - 0.7) / 0.2) ** 2) return z, nz / np.trapezoid(nz, z) -THETA_ARCMIN = np.geomspace(5.0, 250.0, 12) +# --------------------------------------------------------------------------- # +# The CAMB oracle +# --------------------------------------------------------------------------- # +def camb_params(config, As, *, nonlinear, kmax=20.0): + """``CAMBparams`` at ``config``'s background, every field fed from one source.""" + import camb + + p = camb.CAMBparams() + p.set_cosmology( + H0=config.h * 100, + ombh2=config.Omega_b * config.h**2, + omch2=config.omega_c() * config.h**2, + mnu=config.m_nu, + num_massive_neutrinos=1, + neutrino_hierarchy=config.mass_split, + nnu=bt.NEFF, + TCMB=bt.T_CMB, + ) + p.set_dark_energy(w=config.w0, wa=config.wa, dark_energy_model="ppf") + p.InitPower.set_params(As=As, ns=config.n_s) + p.set_matter_power(redshifts=list(np.linspace(0.0, PK_ZMAX, PK_NZ)), kmax=kmax) + if nonlinear: + p.NonLinear = camb.model.NonLinear_both + p.NonLinearModel.set_params( + halofit_version=config.halofit_version, + HMCode_logT_AGN=config.hmcode_logT_AGN, + ) + else: + p.NonLinear = camb.model.NonLinear_none + return p -def _both_paths(config, **camb_kwargs): - z, nz = _gauss_nz() - xip_a, xim_a = cm.xi_ccl( - config.ccl_params(), config, (z, nz), (z, nz), THETA_ARCMIN +def camb_sigma8(config, As): + import camb + + return float( + camb.get_results(camb_params(config, As, nonlinear=False)).get_sigma8_0() ) - xip_b, xim_b, As = cm.xi_camb(config, (z, nz), THETA_ARCMIN, **camb_kwargs) - return (xip_a, xim_a), (xip_b, xim_b), As -def _assert_xi_agreement(a, b, label): - (xip_a, xim_a), (xip_b, xim_b) = a, b - assert np.all(xip_a > 0) and np.all(xip_b > 0) # sensible cosmic shear - rel_p = np.abs(xip_b - xip_a) / np.abs(xip_a) - assert rel_p.max() < XIP_RTOL, ( - f"{label}: ξ+ max rel diff {rel_p.max():.3%} ≥ {XIP_RTOL:.1%}" - ) - floor = XIM_FLOOR_FRAC * np.max(np.abs(xim_a)) - above = np.abs(xim_a) > floor - rel_m = np.abs(xim_b - xim_a)[above] / np.abs(xim_a)[above] - assert rel_m.max() < XIM_RTOL, ( - f"{label}: ξ− max rel diff {rel_m.max():.3%} ≥ {XIM_RTOL:.1%} (on |ξ−| > floor)" +def camb_As_for_sigma8(config, target, As_seed=2.1e-9): + """σ8² ∝ A_s exactly, so one evaluation and one rescale land on target.""" + return As_seed * (target / camb_sigma8(config, As_seed)) ** 2 + + +def xi_camb(config, nz, theta_arcmin, *, n_ell=300, ell_max=60000, kmax=20.0, n_k=400): + """ξ± from a direct CAMB P(k), projected by CCL; returns ``(xip, xim, As)``.""" + import camb + import pyccl as ccl + + As = camb_As_for_sigma8(config, config.sigma8()) + results = camb.get_results(camb_params(config, As, nonlinear=True, kmax=kmax)) + # CCL's native units already: k in 1/Mpc, P in Mpc³. + interp = results.get_matter_power_interpolator( + nonlinear=True, hubble_units=False, k_hunit=False ) - # near the zero crossing: absolute agreement at the floor scale - abs_m = np.abs(xim_b - xim_a)[~above] - if len(abs_m): - assert abs_m.max() < XIM_RTOL * floor, ( - f"{label}: ξ− absolute diff {abs_m.max():.3e} near zero crossing" - ) - print( - f"\n{label}: ξ+ max rel {rel_p.max():.3%}; " - f"ξ− max rel {rel_m.max():.3%} (above floor, " - f"{above.sum()}/{len(above)} points)" + k = np.geomspace(1e-4, kmax * config.h, n_k) + z = np.linspace(0.0, PK_ZMAX, PK_NZ) + a = 1.0 / (1.0 + z) + order = np.argsort(a) + pk2d = ccl.Pk2D( + a_arr=a[order], + lk_arr=np.log(k), + pk_arr=np.log(interp.P(z, k)[order]), + is_logp=True, ) + cosmo = bt.ccl_cosmology(config.ccl_params(), config) + lens = ccl.WeakLensingTracer(cosmo, dndz=nz) + ells = np.unique(np.geomspace(2, ell_max, n_ell).astype(int)).astype(float) + cl = ccl.angular_cl(cosmo, lens, lens, ells, p_of_k_a=pk2d) + theta_deg = np.asarray(theta_arcmin) / 60.0 + xip = ccl.correlation(cosmo, ell=ells, C_ell=cl, theta=theta_deg, type="GG+") + xim = ccl.correlation(cosmo, ell=ells, C_ell=cl, theta=theta_deg, type="GG-") + return xip, xim, As # --------------------------------------------------------------------------- # -# AC10: σ8/A_s reconciliation +# AC10–12 # --------------------------------------------------------------------------- # -@pytest.mark.slow -def test_ac10_sigma8_As_reconciliation(): - """(a) nominal A_s leaves CAMB's σ8 >2% off target — the convention - offset is real; (b) the closed-form rescale lands on target to <1e-4.""" - cfg = cm.TheoryConfig() - target = cfg.sigma8() +def _assert_agreement(config, label): + nz = _gauss_nz() + xip_a, xim_a = bt.xi_ccl(config.ccl_params(), config, nz, nz, THETA_ARCMIN) + xip_b, xim_b, _ = xi_camb(config, nz, THETA_ARCMIN) + assert np.all(xip_a > 0) and np.all(xip_b > 0) + rel_p = np.abs(xip_b - xip_a) / np.abs(xip_a) + assert rel_p.max() < XIP_RTOL, f"{label}: ξ+ max rel diff {rel_p.max():.3%}" + floor = XIM_FLOOR_FRAC * np.max(np.abs(xim_a)) + above = np.abs(xim_a) > floor + rel_m = np.abs(xim_b - xim_a)[above] / np.abs(xim_a)[above] + assert rel_m.max() < XIM_RTOL, f"{label}: ξ− max rel diff {rel_m.max():.3%}" + assert np.all(np.abs(xim_b - xim_a)[~above] < XIM_RTOL * floor), label - nominal = cm.camb_linear_sigma8(cfg, 2.1e-9) - offset = abs(nominal / target - 1) - print(f"\nAC10 nominal-A_s σ8 offset: {offset:.4f}") - assert offset > 0.02 - As = cm.camb_As_for_sigma8(cfg, target) - matched = cm.camb_linear_sigma8(cfg, As) - print(f"AC10 σ8-matched residual: {abs(matched - target):.2e} (A_s={As:.4e})") - assert abs(matched - target) < 1e-4 +def test_ac10_sigma8_As_reconciliation(): + """Nominal A_s misses σ8 by >2%; the closed-form rescale lands within 1e-4.""" + cfg = bt.TheoryConfig() + assert abs(camb_sigma8(cfg, 2.1e-9) / cfg.sigma8() - 1) > 0.02 + assert ( + abs(camb_sigma8(cfg, camb_As_for_sigma8(cfg, cfg.sigma8())) - cfg.sigma8()) + < 1e-4 + ) -# --------------------------------------------------------------------------- # -# AC11 + AC12: ξ± agreement at and off the fiducial -# --------------------------------------------------------------------------- # -@pytest.mark.slow def test_ac11_xi_agreement_at_fiducial(): - cfg = cm.TheoryConfig() - a, b, _ = _both_paths(cfg) - _assert_xi_agreement(a, b, "AC11 fiducial") + _assert_agreement(bt.TheoryConfig(), "fiducial") -@pytest.mark.slow def test_ac12_xi_agreement_off_fiducial(): - """A representative in-envelope offset — the *shift* (a difference of two - theory vectors) must not inherit a stack-disagreement bias.""" - cfg = cm.TheoryConfig.from_overrides({"S8": 0.80 + 0.075, "Omega_m": 0.30 - 0.05}) - a, b, _ = _both_paths(cfg) - _assert_xi_agreement(a, b, "AC12 off-fiducial") + """An in-envelope point: the shift must not inherit a stack disagreement.""" + cfg = dataclasses.replace(bt.TheoryConfig(), S8=0.80 + 0.075, Omega_m=0.30 - 0.05) + _assert_agreement(cfg, "off-fiducial") # --------------------------------------------------------------------------- # -# AC13: halofit token pinned to the inference config (fast) +# AC13: the nonlinear recipe is the inference config's # --------------------------------------------------------------------------- # def test_ac13_halofit_token_matches_inference_config(): - """The blinding fiducial's CCL halofit token equals the CosmoSIS - inference config's ``halofit_version`` — asserted against the config - file itself. All three blinding backends share one recipe by - construction and would agree with each other while jointly diverging - from the inference stack, so this cannot be caught by the cross-backend - test and is asserted independently here.""" + """The blinding recipe is the CosmoSIS pipeline's, read from its config file. + + The CCL path and the CAMB oracle share the recipe by construction, so they + would agree while jointly diverging from inference; only this lineage check + catches that. + """ ini = ( pathlib.Path(__file__).resolve().parents[3] / "cosmo_inference" @@ -144,32 +170,6 @@ def test_ac13_halofit_token_matches_inference_config(): ) match = re.search(r"^halofit_version\s*=\s*(\S+)", ini.read_text(), re.MULTILINE) assert match, f"no halofit_version in {ini}" - inference_token = match.group(1) - cfg = cm.TheoryConfig() - assert cfg.ccl_halofit_version == inference_token - # the two stack tokens denote ONE recipe; a divergence is a config bug - assert cfg.camb_halofit_version == cfg.ccl_halofit_version - # #280: the shipped Boltzmann backend is CAMB-through-CCL, matching the - # CosmoSIS+CAMB inference stack — one power-spectrum path. The cross-check - # tests above (AC10–12, 14) all run at this default configuration. + cfg = bt.TheoryConfig() + assert cfg.halofit_version == match.group(1) assert cfg.transfer_function == "boltzmann_camb" - - -# --------------------------------------------------------------------------- # -# AC14: fast smoke — broken wiring caught in the fast suite -# --------------------------------------------------------------------------- # -def test_ac14_crosscheck_smoke(): - """Both paths run at coarse resolution: finite, positive, - few-percent-agreeing ξ+, and a σ8-matched A_s in a sane range.""" - cfg = cm.TheoryConfig() - z, nz = _gauss_nz(n=150) - theta = np.geomspace(10.0, 100.0, 4) - xip_a, _ = cm.xi_ccl(cfg.ccl_params(), cfg, (z, nz), (z, nz), theta) - xip_b, _, As = cm.xi_camb( - cfg, (z, nz), theta, n_ell=120, ell_max=30000, kmax=10.0, n_k=200 - ) - assert np.all(np.isfinite(xip_a)) and np.all(np.isfinite(xip_b)) - assert np.all(xip_a > 0) and np.all(xip_b > 0) - assert 1e-9 < As < 3e-9 - rel = np.abs(xip_b - xip_a) / np.abs(xip_a) - assert rel.max() < 0.05, f"smoke ξ+ rel diff {rel.max():.3%} unexpectedly large" diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 5e248797..53eabf23 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -14,6 +14,7 @@ import numpy as np import pytest import yaml +from _synthetic import write_synthetic_catalogs from sp_validation.cosmo_val import CosmologyValidation @@ -273,141 +274,6 @@ def test_v1_4_6_glass_mock_default_seed(self, base_config): # shear_psf_leakage, cosmo_numba), i.e. they run in the container. # ------------------------------------------------------------------ - @staticmethod - def _write_synthetic_catalogs( - tmp_path, - n_gal=2000, - n_star=800, - ra_range=(10.0, 14.0), - dec_range=(10.0, 14.0), - seed=1234, - coherent_shear=False, - with_psf=False, - ): - """Write small deterministic FITS catalogs + dndz, return a config dict. - - Builds a synthetic shear catalog (RA/Dec/e1/e2/w), a PSF star catalog - with the columns the leakage/rho-tau seams read, and a cs_util-readable - dndz file. Returns ``(params, version)`` ready to hand to - ``CosmologyValidation``. - - Parameters - ---------- - coherent_shear : bool - If True, inject a smooth position-dependent shear pattern on top of - shape noise so that xi+/- is smooth (needed for the pure-E/B - integral to be numerically well-posed). - with_psf : bool - If True, add a ``psf`` config block (rho/tau / pseudo-Cl read it via - ``get_params_rho_tau``). - """ - from astropy.table import Table - - rng = np.random.default_rng(seed) - version = "TestCatalog" - - cat_dir = tmp_path / "catalog" - cat_dir.mkdir() - nz_dir = tmp_path / "nz" - nz_dir.mkdir() - output_dir = tmp_path / "output" - output_dir.mkdir() - - ra = rng.uniform(*ra_range, n_gal) - dec = rng.uniform(*dec_range, n_gal) - if coherent_shear: - # Smooth E-mode-like pattern so xi+/- is smooth, plus shape noise. - e1 = 0.02 * np.cos(np.radians(ra) * 40) + rng.normal(0, 0.05, n_gal) - e2 = 0.02 * np.sin(np.radians(dec) * 40) + rng.normal(0, 0.05, n_gal) - else: - e1 = rng.normal(0, 0.25, n_gal) - e2 = rng.normal(0, 0.25, n_gal) - w = rng.uniform(0.5, 1.0, n_gal) - - shear_path = cat_dir / "shear.fits" - Table({"RA": ra, "Dec": dec, "e1": e1, "e2": e2, "w": w}).write( - shear_path, overwrite=True - ) - - star_path = cat_dir / "star.fits" - Table( - { - "RA": rng.uniform(*ra_range, n_star), - "Dec": rng.uniform(*dec_range, n_star), - "HSM_G1_PSF": rng.normal(0, 0.03, n_star), - "HSM_G2_PSF": rng.normal(0, 0.03, n_star), - "HSM_G1_STAR": rng.normal(0, 0.03, n_star), - "HSM_G2_STAR": rng.normal(0, 0.03, n_star), - "HSM_T_PSF": rng.uniform(0.4, 0.6, n_star), - "HSM_T_STAR": rng.uniform(0.4, 0.6, n_star), - "HSM_FLAG_PSF": np.zeros(n_star, dtype=int), - "HSM_FLAG_STAR": np.zeros(n_star, dtype=int), - } - ).write(star_path, overwrite=True) - - # dndz in the commented-header format cs_util.read_dndz expects: - # column "z" holds bin edges (n+1), "dn_dz" the densities. - z_edges = np.linspace(0.05, 3.0, 31) - dndz = np.exp(-(((z_edges - 0.7) / 0.3) ** 2)) - dndz_lines = ["# z dn_dz"] + [f"{zz} {nn}" for zz, nn in zip(z_edges, dndz)] - (nz_dir / "dndz_SP_A.txt").write_text("\n".join(dndz_lines) + "\n") - - shear_cfg = { - "path": "shear.fits", - "redshift_path": str(nz_dir / "dndz_SP_A.txt"), - "w_col": "w", - "e1_col": "e1", - "e2_col": "e2", - "R": 1.0, - "e1_col_corrected": "e1", - "e2_col_corrected": "e2", - } - star_cfg = { - "path": "star.fits", - "ra_col": "RA", - "dec_col": "Dec", - "e1_col": "HSM_G1_PSF", - "e2_col": "HSM_G2_PSF", - } - version_cfg = { - "subdir": str(cat_dir), - "pipeline": "SP", - "shear": shear_cfg, - "star": star_cfg, - } - if with_psf: - version_cfg["psf"] = { - "path": "star.fits", - "hdu": 1, - "ra_col": "RA", - "dec_col": "Dec", - "e1_PSF_col": "HSM_G1_PSF", - "e2_PSF_col": "HSM_G2_PSF", - "e1_star_col": "HSM_G1_STAR", - "e2_star_col": "HSM_G2_STAR", - "PSF_size": "HSM_T_PSF", - "star_size": "HSM_T_STAR", - "PSF_flag": "HSM_FLAG_PSF", - "star_flag": "HSM_FLAG_STAR", - } - - config_data = { - "nz": { - "subdir": str(nz_dir), - "dndz": {"blind": "A", "path": "dndz"}, - }, - "paths": {"output": str(output_dir)}, - version: version_cfg, - } - config_path = tmp_path / "synthetic_config.yaml" - config_path.write_text(yaml.dump(config_data, sort_keys=False)) - - params = { - "catalog_config": str(config_path), - "output_dir": str(output_dir), - } - return params, version - def test_calculate_2pcf_runs_on_synthetic_catalog(self, tmp_path): """calculate_2pcf wires catalog+config into treecorr GGCorrelation. @@ -417,7 +283,7 @@ def test_calculate_2pcf_runs_on_synthetic_catalog(self, tmp_path): committed reference later for value-drift coverage). """ pytest.importorskip("treecorr") - params, version = self._write_synthetic_catalogs(tmp_path) + params, version = write_synthetic_catalogs(tmp_path) nbins = 8 cv = CosmologyValidation( @@ -459,7 +325,7 @@ def test_xi_part_carries_the_covariance_the_measurement_estimated( run_2pcf = importlib.util.module_from_spec(spec) spec.loader.exec_module(run_2pcf) - params, version = self._write_synthetic_catalogs(tmp_path) + params, version = write_synthetic_catalogs(tmp_path) part = tmp_path / "part.sacc" gg = run_2pcf.run_2pcf( ver=version, @@ -472,7 +338,7 @@ def test_xi_part_carries_the_covariance_the_measurement_estimated( sacc_out=str(part), ) - cov = sacc_io.load(str(part), allow_unblinded=True).covariance + cov = sacc_io.load(str(part)).covariance if npatch > 1: assert isinstance(cov, sacc.covariance.FullCovariance) np.testing.assert_array_equal(cov.dense, gg.cov) @@ -491,7 +357,7 @@ def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): """ import treecorr - params, version = self._write_synthetic_catalogs( + params, version = write_synthetic_catalogs( tmp_path, n_gal=4000, coherent_shear=True ) cv = CosmologyValidation( @@ -527,7 +393,7 @@ def test_calculate_scale_dependent_leakage_runs_on_synthetic_catalog( tightened to allclose vs. a committed reference later). """ pytest.importorskip("treecorr") - params, version = self._write_synthetic_catalogs(tmp_path) + params, version = write_synthetic_catalogs(tmp_path) nbins = 8 cv = CosmologyValidation( @@ -576,7 +442,7 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( from sp_validation import b_modes # Coherent shear -> smooth xi+/-, so the pure-E/B integral is well-posed. - params, version = self._write_synthetic_catalogs( + params, version = write_synthetic_catalogs( tmp_path, n_gal=4000, coherent_shear=True ) diff --git a/src/sp_validation/tests/test_custody.py b/src/sp_validation/tests/test_custody.py new file mode 100644 index 00000000..ec10a27a --- /dev/null +++ b/src/sp_validation/tests/test_custody.py @@ -0,0 +1,260 @@ +"""Custody is declared per base catalogue and resolved in one place (I12, I13). + +``sp_validation.custody.custody_of`` reads a catalogue's declaration and the +blind registry beside the catalogue config. These tests pin every row of its +table on hand-written registries (the host never decrypts, so a record needs +no real seed), and that the host Snakemake and a container job resolve the same +custody for every version. +""" + +import importlib.util +import json +from pathlib import Path + +import pytest +import yaml + +from sp_validation import custody as cu + +REPO = Path(__file__).resolve().parents[3] + + +def _write_blind(registry, name, bases, *, revealed=None): + """A registry record as ``blinding init`` writes it, with a stand-in seed.""" + record = registry / name + record.mkdir(parents=True) + seed = f"seed-of-{name}" + (record / "commitment.json").write_text( + json.dumps( + { + "blind": name, + "seed_commitment": cu.seed_commitment(seed), + "config_digest": "d" * 64, + "draw_scheme": 2, + } + ) + ) + (record / "bases").write_text("".join(f"{b}\n" for b in bases)) + if revealed is not None: + (record / "revealed.json").write_text(json.dumps({"seed": revealed})) + return seed + + +def _catalogues(**declarations): + """A parsed catalogue config: one entry per ``name=declaration``.""" + entries = {"nz": {"subdir": "/nz"}, "paths": {"output": "./output"}} + for name, declaration in declarations.items(): + entries[name] = { + "shear": { + "path": f"{name}.fits", + "e1_col_corrected": "e1_corrected", + "e2_col_corrected": "e2_corrected", + } + } + if isinstance(declaration, dict): + entries[name].update(declaration) + elif declaration is not None: + entries[name]["blinding"] = declaration + return entries + + +# --------------------------------------------------------------------------- # +# I12: the custody table +# --------------------------------------------------------------------------- # +def test_undeclared_catalogue_without_a_blind_fails_with_the_command(tmp_path): + cats = _catalogues(SP_v9=None) + with pytest.raises(cu.CustodyError) as err: + cu.custody_of(cats, "SP_v9", registry=tmp_path / "blinds") + message = str(err.value) + assert "declares no custody for it, so it is blinded" in message + assert "python -m sp_validation.blinding init" in message + assert "share" in message + + +def test_declared_blinded_without_a_blind_fails(tmp_path): + cats = _catalogues(SP_v9="blinded") + with pytest.raises(cu.CustodyError, match="no blind covers it"): + cu.custody_of(cats, "SP_v9", registry=tmp_path / "blinds") + + +@pytest.mark.parametrize("declaration", [None, "blinded"]) +def test_blinded_under_the_covering_blind(tmp_path, declaration): + registry = tmp_path / "blinds" + seed = _write_blind(registry, "y3", ["SP_v9"]) + c = cu.custody_of(_catalogues(SP_v9=declaration), "SP_v9", registry=registry) + assert c.status == "blinded" and c.catalogue == "SP_v9" and c.blind == "y3" + assert c.commitment == cu.seed_commitment(seed) + assert c.token == f"blinded:SP_v9:y3:{c.commitment}" + assert c.stamp["blinding"] == "blinded" + assert c.stamp["blinding_commitment"] == c.commitment + + +def test_blinded_under_a_revealed_blind_fails(tmp_path): + registry = tmp_path / "blinds" + seed = _write_blind(registry, "y3", ["SP_v9"]) + (registry / "y3" / "revealed.json").write_text(json.dumps({"seed": seed})) + with pytest.raises(cu.CustodyError, match="declare `blinding: unblinded`"): + cu.custody_of(_catalogues(SP_v9=None), "SP_v9", registry=registry) + + +def test_unblinded_without_a_blind(tmp_path): + c = cu.custody_of( + _catalogues(SP_v9="unblinded"), "SP_v9", registry=tmp_path / "blinds" + ) + assert (c.status, c.token) == ("unblinded", "unblinded:SP_v9") + assert c.stamp == {"blinding": "unblinded", "blinding_catalogue": "SP_v9"} + + +def test_unblinding_a_concealed_catalogue_is_the_reveal(tmp_path): + registry = tmp_path / "blinds" + _write_blind(registry, "y3", ["SP_v9"]) + with pytest.raises(cu.CustodyError, match="blinding reveal y3"): + cu.custody_of(_catalogues(SP_v9="unblinded"), "SP_v9", registry=registry) + + +def test_unblinded_after_a_reveal_whose_seed_matches(tmp_path): + registry = tmp_path / "blinds" + seed = _write_blind(registry, "y3", ["SP_v9"]) + (registry / "y3" / "revealed.json").write_text(json.dumps({"seed": seed})) + c = cu.custody_of(_catalogues(SP_v9="unblinded"), "SP_v9", registry=registry) + assert c.token == "unblinded:SP_v9" + + +def test_a_published_seed_that_misses_the_commitment_fails(tmp_path): + registry = tmp_path / "blinds" + _write_blind(registry, "y3", ["SP_v9"], revealed="not-the-seed") + with pytest.raises(cu.CustodyError, match="commitment"): + cu.custody_of(_catalogues(SP_v9="unblinded"), "SP_v9", registry=registry) + + +def test_mock(tmp_path): + registry = tmp_path / "blinds" + c = cu.custody_of(_catalogues(SP_v9="mock"), "SP_v9", registry=registry) + assert (c.status, c.token) == ("mock", "mock:SP_v9") + _write_blind(registry, "y3", ["SP_v9"]) + with pytest.raises(cu.CustodyError, match="mock"): + cu.custody_of(_catalogues(SP_v9="mock"), "SP_v9", registry=registry) + + +def test_two_covering_blinds_fail(tmp_path): + registry = tmp_path / "blinds" + _write_blind(registry, "a", ["SP_v9"]) + _write_blind(registry, "b", ["SP_v9"]) + with pytest.raises(cu.CustodyError, match="a, b"): + cu.custody_of(_catalogues(SP_v9=None), "SP_v9", registry=registry) + + +def test_a_variant_entry_may_not_declare_custody(tmp_path): + cats = _catalogues( + SP_v9="unblinded", SP_v9_ecut07={"base": "SP_v9", "blinding": "unblinded"} + ) + with pytest.raises(cu.CustodyError, match="declare custody on SP_v9"): + cu.custody_of(cats, "SP_v9_ecut07", registry=tmp_path / "blinds") + + +def test_an_unknown_declaration_fails(tmp_path): + with pytest.raises(cu.CustodyError, match="blinded, unblinded or mock"): + cu.custody_of(_catalogues(SP_v9="open"), "SP_v9", registry=tmp_path / "blinds") + + +def test_variants_share_their_base(tmp_path): + """`_leak_corr`, `_seed` and `base:` entries resolve to the base.""" + registry = tmp_path / "blinds" + _write_blind(registry, "y3", ["SP_v9"]) + cats = _catalogues(SP_v9=None, SP_v9_ecut07={"base": "SP_v9"}) + base = cu.custody_of(cats, "SP_v9", registry=registry) + for version in ( + "SP_v9_leak_corr", + "SP_v9_seed00042", + "SP_v9_seed00042_leak_corr", + "SP_v9_ecut07", + "SP_v9_ecut07_leak_corr", + ): + assert cu.custody_of(cats, version, registry=registry) == base, version + + +def test_base_links_are_followed_and_checked(): + cats = _catalogues( + A="unblinded", B={"base": "A"}, C={"base": "B"}, D={"base": "nowhere"} + ) + assert cu.base_catalogue(cats, "C_leak_corr") == "A" + with pytest.raises(cu.CustodyError, match="nowhere"): + cu.base_catalogue(cats, "D") + loop = _catalogues(E={"base": "F"}, F={"base": "E"}) + with pytest.raises(cu.CustodyError, match="cycle"): + cu.base_catalogue(loop, "E") + with pytest.raises(cu.CustodyError, match="not a catalogue"): + cu.base_catalogue(cats, "paths") + + +def test_summary_is_one_line_per_base(tmp_path): + registry = tmp_path / "blinds" + _write_blind(registry, "y3", ["SP_v9"]) + cats = _catalogues(SP_v9=None, SP_v8="unblinded") + lines = cu.summary( + cats, ["SP_v9", "SP_v9_leak_corr", "SP_v8_leak_corr"], registry=registry + ) + assert lines == [ + "[custody] SP_v9 (+ SP_v9_leak_corr): blinded under y3", + "[custody] SP_v8 (+ SP_v8_leak_corr): unblinded", + ] + + +def test_every_catalogue_in_the_repository_resolves(): + """Each entry of the committed cat_config has a custody, from the repo registry.""" + path = REPO / "cosmo_val" / "cat_config.yaml" + cats = yaml.safe_load(path.read_text()) + registry = cu.registry_of(path) + statuses = { + name: cu.custody_of(cats, name, registry=registry).status + for name in cats + if name not in cu.NOT_CATALOGUES + } + assert statuses and set(statuses.values()) <= set(cu.STATUSES), statuses + + +# --------------------------------------------------------------------------- # +# I13: the host and a job resolve the same custody +# --------------------------------------------------------------------------- # +def _toy_checkout(tmp_path): + """A checkout-shaped tree: workflow/common.py, src/, cosmo_val/{cat_config,blinds}.""" + (tmp_path / "workflow").mkdir() + (tmp_path / "workflow" / "common.py").write_text( + (REPO / "workflow" / "common.py").read_text() + ) + (tmp_path / "src").symlink_to(REPO / "src") + cats = _catalogues( + TOY=None, + TOY_OPEN="unblinded", + TOY_MOCK="mock", + TOY_ecut07={"base": "TOY"}, + ) + for entry in (cats["TOY"], cats["TOY_OPEN"], cats["TOY_MOCK"], cats["TOY_ecut07"]): + entry["subdir"] = str(tmp_path) + (tmp_path / "cosmo_val").mkdir() + (tmp_path / "cosmo_val" / "cat_config.yaml").write_text(yaml.safe_dump(cats)) + _write_blind(tmp_path / "cosmo_val" / "blinds", "toy", ["TOY"]) + return tmp_path + + +def test_host_and_job_resolve_the_same_custody(tmp_path): + """`common.custody_token(v)` equals `CosmologyValidation.custody(v).token`.""" + from sp_validation.cosmo_val import CosmologyValidation + + root = _toy_checkout(tmp_path) + spec = importlib.util.spec_from_file_location( + "toy_common", root / "workflow" / "common.py" + ) + common = importlib.util.module_from_spec(spec) + spec.loader.exec_module(common) + common.CATALOG_CONFIG = yaml.safe_load(Path(common.CAT_CONFIG).read_text()) + + versions = ["TOY", "TOY_leak_corr", "TOY_OPEN", "TOY_MOCK", "TOY_ecut07"] + cv = CosmologyValidation( + versions=versions, + catalog_config=common.CAT_CONFIG, + output_dir=str(tmp_path / "out"), + ) + for version in versions: + assert common.custody_token(version) == cv.custody(version).token, version + assert common.custody_token("TOY").startswith("blinded:TOY:toy:") diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py new file mode 100644 index 00000000..cd81b083 --- /dev/null +++ b/src/sp_validation/tests/test_custody_e2e.py @@ -0,0 +1,445 @@ +"""End to end: one blinded catalogue through the rule scripts, then its reveal. + +A synthetic catalogue is declared three ways (``TOY`` blinded, ``TOY_OPEN`` +unblinded, ``TOY_MOCK`` mock). A blind is drawn for ``TOY`` with ``blinding +init``; the rule scripts run as Snakemake runs them (``runpy`` with a +``snakemake`` object) for ``TOY`` and ``TOY_leak_corr``: both ξ± grids, a +two-bin writer, pseudo-Cℓ on an nside-32 NaMaster workspace, ρ/τ, COSEBIs, +pure-E/B and assembly. The blind is then revealed, the declaration flipped, the +chain re-run, and the audit must prove blinded − true = shift(seed) on every +part the reveal archived. +""" + +import json +import os +import runpy +import types +from pathlib import Path + +import numpy as np +import pytest +import yaml +from _synthetic import write_synthetic_catalogs + +from sp_validation import blinding as bd +from sp_validation import custody as cu +from sp_validation import sacc_io as sio +from sp_validation.cosmo_val import CosmologyValidation + +REPO = Path(__file__).resolve().parents[3] +SCRIPTS = REPO / "workflow" / "scripts" + +GRIDS = { + "reporting": {"min_sep": 5.0, "max_sep": 60.0, "nbins": 6, "npatch": 1}, + "integration": {"min_sep": 1.0, "max_sep": 150.0, "nbins": 120, "npatch": 1}, +} +SCALE_CUT = [12.0, 60.0] +PARTS = ("xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau") + + +class Named(list): + """The shape of Snakemake's ``input``/``output``/``params``: a list with names.""" + + def __init__(self, **items): + super().__init__(items.values()) + self._items = items + + def __getitem__(self, key): + return self._items[key] if isinstance(key, str) else super().__getitem__(key) + + def __getattr__(self, name): + try: + return self._items[name] + except KeyError: + raise AttributeError(name) from None + + def get(self, key, default=None): + return self._items.get(key, default) + + +def run_rule(script, *, input=None, output=None, params=None): + """Run a rule script as Snakemake runs a job: outputs removed, then ``script:``.""" + for path in (output or {}).values(): + Path(path).unlink(missing_ok=True) + smk = types.SimpleNamespace( + input=Named(**(input or {})), + output=Named(**(output or {})), + params=Named(**(params or {})), + ) + runpy.run_path( + str(SCRIPTS / script), init_globals={"snakemake": smk}, run_name="__main__" + ) + + +def declared(cat_config, version): + """The custody a host Snakemake resolves for ``version``.""" + return cu.custody_of( + yaml.safe_load(Path(cat_config).read_text()), + version, + registry=cu.registry_of(cat_config), + ) + + +def _namaster_part_inputs(): + """``(ell_eff, cl_all, workspace)`` on an nside-32 full-sky workspace.""" + import healpy as hp + import pymaster as nmt + + nside = 32 + npix = hp.nside2npix(nside) + field = nmt.NmtField(np.ones(npix), [np.zeros(npix), np.zeros(npix)], spin=2) + binning = nmt.NmtBin.from_nside_linear(nside, 8) + workspace = nmt.NmtWorkspace() + workspace.compute_coupling_matrix(field, field, binning) + ell = binning.get_effective_ells() + ee = 1e-8 * (ell / 30.0) ** -1.2 + return ell, np.array([ee, 0.01 * ee, 0.01 * ee, 0.02 * ee]), workspace + + +def _rho_tau_handlers(): + rng = np.random.default_rng(5) + theta = np.geomspace(5.0, 60.0, 6) + rho, tau = {"theta": theta}, {"theta": theta} + for k in range(6): + for sfx in ("p", "m"): + rho[f"rho_{k}_{sfx}"] = rng.normal(size=6) * 1e-6 + rho[f"varrho_{k}_{sfx}"] = rng.uniform(1e-14, 1e-13, 6) + for k in (0, 2, 5): + for sfx in ("p", "m"): + tau[f"tau_{k}_{sfx}"] = rng.normal(size=6) * 1e-6 + tau[f"vartau_{k}_{sfx}"] = rng.uniform(1e-14, 1e-13, 6) + return types.SimpleNamespace(rho_stats=rho), types.SimpleNamespace(tau_stats=tau) + + +def _covariances(root): + """CosmoCov-format ξ± covariances on both grids, and a NaMaster Cℓ FITS.""" + from astropy.io import fits + + paths = {} + for grid, b in GRIDS.items(): + paths[grid] = root / f"cov_{grid}.txt" + np.savetxt(paths[grid], np.diag(np.full(2 * b["nbins"], 1e-10))) + nbp = len(_namaster_part_inputs()[0]) + paths["pseudo_cl"] = root / "pseudo_cl_cov.fits" + fits.HDUList( + [fits.PrimaryHDU()] + + [ + fits.ImageHDU(np.eye(nbp) * 1e-20, name=name) + for name in ("COVAR_EE_EE", "COVAR_BB_BB", "COVAR_EB_EB") + ] + ).writeto(paths["pseudo_cl"], overwrite=True) + return paths + + +def run_chain(cat_config, version, out, cov, *, grid_parts=None): + """The cosmo_val chain for one version, into ``out``; returns the part paths.""" + out.mkdir(parents=True, exist_ok=True) + token = declared(cat_config, version).token + xi = {} + for grid, b in GRIDS.items(): + xi[grid] = out / f"{version}_xi_{grid}.sacc" + binning = ( + f"minsep={b['min_sep']}_maxsep={b['max_sep']}_nbins={b['nbins']}" + f"_npatch={b['npatch']}" + ) + run_rule( + "run_2pcf.py", + output={ + "txt": str(out / f"{version}_xi_{binning}.txt"), + "sacc": str(xi[grid]), + }, + params={ + "ver": version, + **b, + "cat_config": str(cat_config), + "output_dir": str(out), + "grid": grid, + "custody": token, + }, + ) + if grid_parts == "only": + return xi + + cv = CosmologyValidation( + versions=[version], catalog_config=str(cat_config), output_dir=str(out) + ) + paths = { + "xi_reporting": xi["reporting"], + "pseudo_cl": out / f"pseudo_cl_{version}.sacc", + "cosebis": out / f"{version}_cosebis.sacc", + "pure_eb": out / f"{version}_pure_eb.sacc", + "rho_tau": out / f"rho_tau_{version}.sacc", + } + cv.pseudo_cl_to_sacc_part( + version, str(paths["pseudo_cl"]), *_namaster_part_inputs() + ) + cv.rho_tau_to_sacc_part(version, str(out), version, *_rho_tau_handlers()) + + integ = GRIDS["integration"] + run_rule( + "cv_cosebis.py", + input={"xi": str(xi["integration"]), "cov": str(cov["integration"])}, + output={ + "npz": str(out / f"{version}_cosebis.npz"), + "sacc": str(paths["cosebis"]), + "figure_modes": str(out / f"{version}_cosebis_modes.png"), + "figure_covariance": str(out / f"{version}_cosebis_cov.png"), + "figure_scalecut_ptes": str(out / f"{version}_cosebis_ptes.png"), + }, + params={ + "version": version, + "min_sep": integ["min_sep"], + "max_sep": integ["max_sep"], + "nbins": integ["nbins"], + "nmodes": 5, + "scale_cuts": [SCALE_CUT], + "fiducial_scale_cut": SCALE_CUT, + }, + ) + rep = GRIDS["reporting"] + run_rule( + "cv_pure_eb.py", + input={ + "xi_reporting": str(xi["reporting"]), + "xi_integration": str(xi["integration"]), + "cov_integration": str(cov["integration"]), + }, + output={ + "npz": str(out / f"{version}_pure_eb.npz"), + "sacc": str(paths["pure_eb"]), + "figure_integration_vs_reporting": str(out / f"{version}_eb_ivr.png"), + "figure_xis": str(out / f"{version}_eb_xis.png"), + "figure_ptes": str(out / f"{version}_eb_ptes.png"), + "figure_covariance": str(out / f"{version}_eb_cov.png"), + }, + params={ + "version": version, + "min_sep": rep["min_sep"], + "max_sep": rep["max_sep"], + "nbins": rep["nbins"], + "n_samples": 10, + "cosmo_params": {"transfer_function": "eisenstein_hu"}, + "fiducial_scale_cut": SCALE_CUT, + }, + ) + assemble(cat_config, version, out, paths, cov, token) + return paths + + +def assemble(cat_config, version, out, paths, cov, token): + run_rule( + "assemble_sacc.py", + input={ + **{k: str(v) for k, v in paths.items()}, + "xi_cov": str(cov["reporting"]), + "pseudo_cl_cov": str(cov["pseudo_cl"]), + }, + output={"sacc": str(out / f"{version}.sacc")}, + params={ + "version": version, + "expected": list(PARTS), + "custody": token, + "cat_config": str(cat_config), + }, + ) + + +def stamps(root): + """``{relative path: stamp}`` of every SACC under ``root``.""" + return { + str(p.relative_to(root)): cu.read_stamp(sio.load(p).metadata).stamp + for p in sorted(root.rglob("*.sacc")) + if "revealed" not in p.relative_to(root).parts + } + + +@pytest.fixture +def toy(tmp_path, monkeypatch): + """The catalogue config, a blind for TOY, and the covariances the rules read.""" + monkeypatch.syspath_prepend(str(SCRIPTS)) + import cv_runner + import matplotlib + + # Rule jobs line-buffer their own streams; under pytest they are captured. + monkeypatch.setattr(cv_runner, "_unbuffer_streams", lambda: None) + # The figures render with matplotlib's own text engine, whatever LaTeX + # setup the invoking user's matplotlibrc asks for. + monkeypatch.setitem(matplotlib.rcParams, "text.usetex", False) + params, _ = write_synthetic_catalogs( + tmp_path, + n_gal=20000, + coherent_shear=True, + with_psf=True, + catalogues={"TOY": None, "TOY_OPEN": "unblinded", "TOY_MOCK": "mock"}, + ) + (tmp_path / "fast.json").write_text( + json.dumps({"theory": {"transfer_function": "eisenstein_hu"}}) + ) + cat_config = Path(params["catalog_config"]) + bd.main( + [ + "init", + "toy", + "TOY", + "--cat-config", + str(cat_config), + "--config", + str(tmp_path / "fast.json"), + ] + ) + return types.SimpleNamespace( + root=tmp_path, cat_config=cat_config, cov=_covariances(tmp_path) + ) + + +def test_a_blinded_catalogue_from_birth_to_audit(toy, monkeypatch): + out = toy.root / "cosmo_val" + blinded = declared(toy.cat_config, "TOY") + assert blinded.status == "blinded" + + # --- a blinded run ------------------------------------------------------- + for version in ("TOY", "TOY_leak_corr"): + run_chain(toy.cat_config, version, out, toy.cov) + born = stamps(out) + assert len(born) == 2 * (2 + len(PARTS)), sorted(born) + assert all(s == blinded.stamp for s in born.values()), born # one commitment + assert ( + bd.main(["verify", str(out / "TOY.sacc"), "--cat-config", str(toy.cat_config)]) + == 0 + ) + + # A two-bin writer conceals every pair by passing the custody. + two_bin = sio.new_sacc( + { + 0: sio.get_nz(sio.load(out / "TOY_xi_reporting.sacc"), 0), + 1: sio.get_nz(sio.load(out / "TOY_xi_reporting.sacc"), 0), + }, + ) + theta = np.geomspace(5.0, 60.0, 6) + for pair in ((0, 0), (0, 1), (1, 1)): + sio.add_xi(two_bin, pair, theta, 1e-5 / theta, 1e-6 / theta, grid="tomo") + written = sio.save(two_bin, out / "two_bin.sacc", custody=blinded) + assert np.all(np.asarray(written.mean) != np.asarray(two_bin.mean)) + + # The true vector of TOY is TOY_OPEN's, the same files declared unblinded. + open_parts = run_chain( + toy.cat_config, "TOY_OPEN", toy.root / "open", toy.cov, grid_parts="only" + ) + with pytest.raises(ValueError, match="stamps"): + assemble( + toy.cat_config, + "TOY", + toy.root / "swap", + { + "xi_reporting": open_parts["reporting"], + **{ + k: out / Path(v).name + for k, v in { + "pseudo_cl": "pseudo_cl_TOY.sacc", + "cosebis": "TOY_cosebis.sacc", + "pure_eb": "TOY_pure_eb.sacc", + "rho_tau": "rho_tau_TOY.sacc", + }.items() + }, + }, + toy.cov, + blinded.token, + ) + plain = sio.load(open_parts["reporting"]) + for key in cu.STAMP_KEYS: + plain.metadata.pop(key, None) + with pytest.raises(ValueError, match="not copies"): + sio.save( + plain, + out / "laundered.sacc", + derived_from=[sio.load(out / "TOY_xi_reporting.sacc")], + ) + assert not (out / "laundered.sacc").exists() + + # --- an interrupted birth leaves no part --------------------------------- + from sp_validation.cosmo_val import sacc_writers + + def after_compute(*args, **kwargs): + raise RuntimeError("interrupted after the measurement") + + with monkeypatch.context() as m: + m.setattr(sacc_writers, "xi_to_sacc", after_compute) + with pytest.raises(RuntimeError): + run_chain( + toy.cat_config, + "TOY", + toy.root / "interrupted", + toy.cov, + grid_parts="only", + ) + assert not list((toy.root / "interrupted").glob("*.sacc")) + + calls = [] + real_factor = bd._factor + + def second_block_fails(*args, **kwargs): + calls.append(1) + if len(calls) == 2: + raise RuntimeError("interrupted inside conceal") + return real_factor(*args, **kwargs) + + with monkeypatch.context() as m: + m.setattr(bd, "_factor", second_block_fails) + with pytest.raises(RuntimeError, match="inside conceal"): + sio.save( + two_bin, toy.root / "interrupted" / "two_bin.sacc", custody=blinded + ) + assert not (toy.root / "interrupted" / "two_bin.sacc").exists() + + # --- the reveal: archive, flip the declaration, re-measure, audit -------- + (out / "two_bin.sacc").unlink() + assert ( + bd.main( + ["reveal", "toy", "--root", str(out), "--cat-config", str(toy.cat_config)] + ) + == 0 + ) + archive = out / "revealed" / "toy" + assert sorted( + str(p.relative_to(archive)) for p in archive.rglob("*.sacc") + ) == sorted(born) + assert not list(out.glob("*.sacc")) + + config = yaml.safe_load(toy.cat_config.read_text()) + config["TOY"]["blinding"] = "unblinded" + toy.cat_config.write_text(yaml.safe_dump(config, sort_keys=False)) + true = declared(toy.cat_config, "TOY") + assert true.token == "unblinded:TOY" + + for version in ("TOY", "TOY_leak_corr"): + run_chain(toy.cat_config, version, out, toy.cov) + assert all(s == true.stamp for s in stamps(out).values()) + + report = bd.audit("toy", archive=archive, true_root=out, cat_config=toy.cat_config) + assert report["ok"], json.dumps(report, indent=1, default=str) + assert set(report["parts"]) == set(born) + assert ( + abs(report["shift"]["S8"]) <= 0.075 and abs(report["shift"]["Omega_m"]) <= 0.1 + ) + + revealed = toy.cat_config.parent / "blinds" / "toy" / "revealed.json" + published = revealed.read_text() + os.chmod(revealed, 0o644) + revealed.write_text(json.dumps({"seed": "not-the-seed"})) + assert not bd.audit( + "toy", archive=archive, true_root=out, cat_config=toy.cat_config + )["ok"] + revealed.write_text(published) + + +def test_a_mock_never_opens_a_blind(toy, monkeypatch): + def refuse(custody): + raise AssertionError(f"a mock opened a blind: {custody}") + + monkeypatch.setattr(bd, "open_blind", refuse) + out = toy.root / "mock" + run_chain(toy.cat_config, "TOY_MOCK", out, toy.cov) + mock = declared(toy.cat_config, "TOY_MOCK") + found = stamps(out) + assert len(found) == 2 + len(PARTS) + assert all(s == mock.stamp for s in found.values()), found diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index fd39c8e2..cdfec1f8 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -150,6 +150,7 @@ def _write_synthetic_config(tmp_path): "star": {**psf_cfg}, "psf": psf_cfg, "patch_number": 150, + "blinding": "mock", }, } config_path = tmp_path / "config.yaml" @@ -523,7 +524,7 @@ def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): cv.calculate_pseudo_cl_catalog(ver, out_path) assert os.path.exists(out_path) - s = sacc_io.load(out_path, allow_unblinded=True) + s = sacc_io.load(out_path) ell, ee, bb, eb, window = sacc_io.get_pseudo_cl(s, SACC_BIN) assert window is not None # the shared BandpowerWindow rides the part diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 8f8564ae..381aeefb 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -9,6 +9,7 @@ import pytest from sp_validation import sacc_io as sio +from sp_validation.custody import Custody # --------------------------------------------------------------------------- # @@ -62,9 +63,12 @@ def _cosebi_block(s, tr): # --------------------------------------------------------------------------- # # 1. Per-writer round-trip (arrays / tags / windows / NZ bitwise) # --------------------------------------------------------------------------- # +MOCK = Custody("mock", "vTEST") + + def _roundtrip(s, tmp_path, name="rt"): path = tmp_path / f"{name}.sacc" - sio.save(s, str(path), type="mock") + sio.save(s, str(path), custody=MOCK) return sio.load(str(path)) @@ -465,7 +469,7 @@ def test_end_to_end_one_file_layout(tmp_path): s, [(xi_c, _spd(len(xi_c), 1)), (co, _spd(len(co), 2)), (xi_f, fine_block)], ) - sio.save(s, str(tmp_path / f"{version}.sacc"), type="mock") + sio.save(s, str(tmp_path / f"{version}.sacc"), custody=MOCK) a = sio.load(str(tmp_path / f"{version}.sacc")) @@ -504,7 +508,7 @@ def test_one_file_layout_diagonal_fine_fallback(tmp_path): xi_f = _xi_block(s, tr, grid="integration") variances = np.concatenate([np.arange(1, 51) * 1e-12, np.arange(1, 51) * 2e-12]) sio.assemble_covariance(s, [(xi_c, _spd(len(xi_c), 1)), (xi_f, np.diag(variances))]) - sio.save(s, str(tmp_path / "vDIAG.sacc"), type="mock") + sio.save(s, str(tmp_path / "vDIAG.sacc"), custody=MOCK) a = sio.load(str(tmp_path / "vDIAG.sacc")) assert np.array_equal(np.diag(a.covariance.dense[np.ix_(xi_f, xi_f)]), variances) @@ -612,76 +616,6 @@ def test_tomographic_xi_covariance_one_contiguous_block(): assert np.array_equal(s.mean[idx_p], xip) -# --------------------------------------------------------------------------- # -# 12. type stamping + fail-closed load (data/mock x concealed/not; escape -# hatch for the blinding/unblinding tooling) -# --------------------------------------------------------------------------- # -def _saved(tmp_path, name, *, type, concealed=None): - s = _base_sacc() - _add_xi(s) - if concealed is not None: - s.metadata["concealed"] = concealed - path = str(tmp_path / f"{name}.sacc") - sio.save(s, path, type=type) - return path - - -def test_load_mock_unconcealed(tmp_path): - s = sio.load(_saved(tmp_path, "m0", type="mock")) - assert s.metadata["type"] == "mock" - - -def test_load_mock_concealed(tmp_path): - s = sio.load(_saved(tmp_path, "m1", type="mock", concealed=True)) - assert s.metadata["concealed"] - - -def test_load_data_concealed(tmp_path): - s = sio.load(_saved(tmp_path, "d1", type="data", concealed=True)) - assert s.metadata["type"] == "data" - - -def test_load_data_unconcealed_fails_closed(tmp_path): - path = _saved(tmp_path, "d0", type="data") - with pytest.raises(ValueError, match="unblinded"): - sio.load(path) - # concealed=False is as unblinded as no stamp at all - path_f = _saved(tmp_path, "d0f", type="data", concealed=False) - with pytest.raises(ValueError, match="unblinded"): - sio.load(path_f) - - -def test_load_data_unconcealed_escape_hatch(tmp_path): - s = sio.load(_saved(tmp_path, "d0h", type="data"), allow_unblinded=True) - assert s.metadata["type"] == "data" - - -def test_save_requires_valid_type(tmp_path): - s = _base_sacc() - with pytest.raises(TypeError): - sio.save(s, str(tmp_path / "x.sacc")) # type is required - with pytest.raises(ValueError, match="'data' or 'mock'"): - sio.save(s, str(tmp_path / "x.sacc"), type="simulation") - - -def test_save_refuses_type_restamp(tmp_path): - s = _base_sacc() - sio.save(s, str(tmp_path / "x.sacc"), type="mock") - with pytest.raises(ValueError, match="re-stamp"): - sio.save(s, str(tmp_path / "x.sacc"), type="data") - - -def test_load_requires_type_tag(tmp_path): - import sacc as sacc_lib - - s = _base_sacc() # never stamped - path = str(tmp_path / "untyped.sacc") - s.save_fits(path, overwrite=True) - with pytest.raises(KeyError): - sio.load(path) - assert sacc_lib.Sacc.load_fits(path) is not None # raw loader still works - - # --------------------------------------------------------------------------- # # 13. merge(): per-statistic files combine into one; shared tracers stored # once; covariance block-diagonal (all-or-none); metadata union with @@ -702,18 +636,18 @@ def _cosebi_sacc(metadata=None): def test_merge_per_statistic_files(tmp_path): - meta = {"version": "vM", "type": "mock"} + meta = {"version": "vM", "survey": "UNIONS"} s_xi, s_co = _xi_sacc(meta), _cosebi_sacc(meta) merged = sio.merge([s_xi, s_co]) # shared tracers stored once; all points present, xi first assert set(merged.tracers) == {"source_0", sio.PSF_TRACER} assert len(merged.mean) == len(s_xi.mean) + len(s_co.mean) assert np.array_equal(merged.mean, np.concatenate([s_xi.mean, s_co.mean])) - assert merged.metadata["version"] == "vM" and merged.metadata["type"] == "mock" + assert merged.metadata["version"] == "vM" and merged.metadata["survey"] == "UNIONS" # inputs untouched assert s_xi.metadata["version"] == "vM" # readers work on the merged file after a round-trip - sio.save(merged, str(tmp_path / "vM.sacc"), type="mock") + sio.save(merged, str(tmp_path / "vM.sacc"), custody=MOCK) merged_rt = sio.load(str(tmp_path / "vM.sacc")) _, p, _ = sio.get_xi(merged_rt, (0, 0), grid="reporting") assert np.array_equal(p, np.arange(6) * 1e-5) @@ -748,7 +682,7 @@ def test_merge_block_diagonal_covariance_stays_block_diagonal(tmp_path): assert type(s_xi.covariance).__name__ == "BlockDiagonalCovariance" merged = sio.merge([s_xi, s_co]) assert type(merged.covariance).__name__ == "BlockDiagonalCovariance" - sio.save(merged, str(tmp_path / "vBLK.sacc"), type="mock") + sio.save(merged, str(tmp_path / "vBLK.sacc"), custody=MOCK) merged_rt = sio.load(str(tmp_path / "vBLK.sacc")) assert type(merged_rt.covariance).__name__ == "BlockDiagonalCovariance" n_xi = len(s_xi.mean) @@ -768,8 +702,8 @@ def test_merge_mixed_covariance_fails(): def test_merge_conflicting_metadata_fails(): - s_xi = _xi_sacc({"type": "data"}) - s_co = _cosebi_sacc({"type": "mock"}) + s_xi = _xi_sacc({"survey": "UNIONS"}) + s_co = _cosebi_sacc({"survey": "KiDS"}) with pytest.raises(ValueError, match="conflicting metadata"): sio.merge([s_xi, s_co]) diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index c4963b33..159a3e92 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -16,6 +16,7 @@ from sp_validation import sacc_io as sio from sp_validation.cosmo_val import sacc_writers as sw +from sp_validation.custody import Custody def _nz(seed=0, n=40): @@ -32,9 +33,12 @@ def _theta(n=6): return np.geomspace(1.0, 100.0, n) +MOCK = Custody("mock", "vSYNTH") + + def _roundtrip(s, tmp_path, name): p = tmp_path / f"{name}.sacc" - sio.save(s, str(p), type="mock") + sio.save(s, str(p), custody=MOCK) return sio.load(str(p)) @@ -143,7 +147,7 @@ def get_bandpower_windows(self): part = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell, cl_all, _Workspace()) path = tmp_path / "pseudo_cl.sacc" - sio.save(part, str(path), type="mock") + sio.save(part, str(path), custody=MOCK) data = _paper_pseudo_cl_reader()(path) assert np.array_equal(data["ELL"], ell) @@ -349,7 +353,7 @@ def test_assemble_from_reloaded_parts(tmp_path): parts = _make_parts(nz) reloaded = [] for i, part in enumerate(parts): - sio.save(part, str(tmp_path / f"part{i}.sacc"), type="mock") + sio.save(part, str(tmp_path / f"part{i}.sacc"), custody=MOCK) reloaded.append(sio.load(str(tmp_path / f"part{i}.sacc"))) s = sw.assemble_analysis_sacc(reloaded) assert type(s.covariance).__name__ == "BlockDiagonalCovariance" diff --git a/workflow/CONTRACTS b/workflow/CONTRACTS new file mode 100644 index 00000000..dee2fa88 --- /dev/null +++ b/workflow/CONTRACTS @@ -0,0 +1,4 @@ +@sc host-importable +The host Snakemake imports workflow.common with no sp_validation installed; +common loads the stdlib-only project modules it needs by file path. +only: workflow.common may import stdlib, snakemake, snakemake.* diff --git a/workflow/README.md b/workflow/README.md index e5671848..ea5e3575 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -25,6 +25,50 @@ Runs stay modular, not monolithic: a paper or run composes these rules with Snakemake's `module` directive under its own config and an output `prefix`, so each namespaces cleanly under `results//`. +## Custody: which catalogues are blinded + +Every catalogue in `cosmo_val/cat_config.yaml` declares its custody on its base +entry, `blinding: blinded | unblinded | mock`; an entry that declares none is +blinded. Variants (`_leak_corr`, `_seed`, and entries naming their parent +with `base:`) share their base's custody and blind. A launch prints one +`[custody]` line per catalogue it runs; no `--config` or config file changes a +declaration. + +For a blinded catalogue, every ξ± and pseudo-Cℓ_EE value is shifted by a hidden +cosmology before it is first written. COSEBIs and pure-E/B computed from the +shifted ξ± carry the blind with them; B-modes stay usable. Every SACC file +records the custody it was born under, and assembly refuses parts under any +other. + +A blinded catalogue needs a blind, drawn once, by a person. The launch stops +with this command when none covers it: + +```bash +APPTAINERENV_PYTHONPATH=$PWD/src spv-container exec python -m sp_validation.blinding \ + init --cat-config cosmo_val/cat_config.yaml +``` + +then commit `cosmo_val/blinds//`. No rule draws or rewrites a blind, so +`-F`, `--forcerun` or a fresh output tree reuse it, and every collaborator's run +of the catalogue uses the same one. A re-processing of a catalogue whose blind +is still concealed shares it: `… blinding share +--cat-config …`. + +Revealing re-measures; nothing subtracts a shift: + +1. `… blinding reveal --root --cat-config …` publishes the + seed (`revealed.json`) and moves every file concealed under the blind to + `/revealed//`; +2. one reviewed commit adds `revealed.json` and declares `blinding: unblinded` + on the blind's catalogues; +3. the pipeline re-measures the true products; +4. `… blinding audit --archive /revealed/ --true-root + --cat-config …` checks blinded − true = shift(seed) on every + archived file. + +`… blinding verify --cat-config …` checks a file's stamp against the +registry, without the seed. + ## Running on the cluster — the candide profile `profiles/candide/config.yaml` is the committed SLURM profile: it hands diff --git a/workflow/Snakefile b/workflow/Snakefile index 96a28793..fe05f02b 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -38,9 +38,6 @@ wildcard_constraints: # Compute rules (infrastructure — raw outputs, no evidence.json) include: "rules/twopoint.smk" -# Smokescreen blind-at-birth custody, generic over the blindable parts -# twopoint.smk produces. Included before its consumers in covariance/cosmo_val. -include: "rules/blinding.smk" include: "rules/covariance.smk" include: "rules/inference.smk" include: "rules/masks.smk" diff --git a/workflow/common.py b/workflow/common.py index 7c599714..1054974b 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -9,21 +9,12 @@ import importlib.util import json import os -import re import sys from pathlib import Path -from snakemake.io import temp - -# The running checkout. Anchored on this module's own location, not -# workflow.basedir — under `module` composition basedir reflects the composing -# paper, not the running checkout. +# The checkout this workflow was launched from: workflow/common.py -> . REPO_ROOT = Path(__file__).resolve().parents[1] REPO_SRC = REPO_ROOT / "src" -# For rules that shell out to a script directly rather than through Snakemake's -# `script:` directive. -WORKFLOW_SCRIPTS = str(REPO_ROOT / "workflow" / "scripts") -REPO_SCRIPTS = str(REPO_ROOT / "scripts") def _load_checkout_module(name): @@ -53,10 +44,8 @@ def _load_checkout_module(name): image_runtime = _container.image_runtime resolve_image = _container.resolve_image -# The blinding file-name conventions, shared with ``sp_validation.blinding``. -_blinding_paths = _load_checkout_module("blinding_paths") -init_paths = _blinding_paths.init_paths -part_paths = _blinding_paths.part_paths +# Catalogue custody, resolved as container jobs resolve it. +_custody = _load_checkout_module("custody") # Every job inherits this launch's environment (the slurm executor submits with # --export=ALL), and the Snakemake each job step starts keeps its source cache @@ -78,11 +67,13 @@ def _load_checkout_module(name): ) ) # The catalogue config of the launched checkout: the one file both the host -# (CATALOG_CONFIG, loaded in configure) and every job read catalogues from. +# (CATALOG_CONFIG, loaded in configure) and every job read catalogues from, and +# the blind registry beside it. CAT_CONFIG = str(REPO_ROOT / "cosmo_val" / "cat_config.yaml") -# "blind" is the glass-mock A/B/C realisation convention, NOT Smokescreen -# blinding (a separate axis: the concealed=True SACC stamp). The name is baked -# into on-disk filenames we do not own (e.g. nz_{version}_{A|B|C}.txt). +REGISTRY = _custody.registry_of(CAT_CONFIG) +# "blind" is the n(z) A/B/C realisation convention, not Smokescreen blinding +# (custody, above). The name is baked into on-disk filenames we do not own +# (e.g. nz_{version}_{A|B|C}.txt). BLINDS = ["A", "B", "C"] BLOCK_PAIRS = [("++", "1"), ("--", "2"), ("+-", "3")] @@ -99,10 +90,6 @@ def _load_checkout_module(name): "version": r"SP_v[\d.]+(_w_iv)?(_ecut\d+)?(_leak_corr)?", "blind": r"[ABC]", "nbins": r"\d+", - # Constrained so a producer's ξ± output pattern cannot greedily absorb the - # "_blinded" suffix into npatch (which would make it ambiguous with the - # blind_part rule's {stem}_blinded output). npatch is always an integer. - "npatch": r"\d+", "min_sep": r"[0-9.]+", "max_sep": r"[0-9.]+", "gaussian": r"(g|ng)", @@ -117,11 +104,6 @@ def _load_checkout_module(name): DEFAULT_MASK_SUFFIX = "" CATALOG_CONFIG = None PLANCK18 = None -# Run type gates Smokescreen blind-at-birth (see the blind custody section -# below): "data" blinds the three blindable parts and binds ξ-derived consumers -# to the blinded siblings; "mock" bypasses blinding entirely. Set from -# config["cosmo_val"]["type"] in configure(); "data" is the production default. -RUN_TYPE = "data" def inject_checkout_pythonpath(workflow_config): @@ -207,6 +189,7 @@ def warn_if_image_stale(): def check_host_parity(image): """Stop the launch unless this Snakemake matches the image's Python and Snakemake. + @sc host-image-parity A ``script:`` job appends the host's ``sys.path`` -- standard library included -- to its own, as the fallback that lets it unpickle the host's ``snakemake`` object. So the image and the host Snakemake must share a @@ -244,7 +227,7 @@ def check_host_parity(image): def configure(workflow_config): """Install config-derived values after Snakemake has loaded configfiles.""" - global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, FIDUCIAL, PLANCK18, RUN_TYPE + global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, FIDUCIAL, PLANCK18 from snakemake.common.configfile import load_configfile inject_checkout_pythonpath(workflow_config) @@ -255,11 +238,51 @@ def configure(workflow_config): DEFAULT_MASK_SUFFIX = ( "_masked" if workflow_config["covariance"].get("default_masked", False) else "" ) - RUN_TYPE = workflow_config.get("cosmo_val", {}).get("type", "data") + announce_custody(workflow_config) with open(COSMOLOGY_PARAMS) as f: PLANCK18 = json.load(f) +def announce_custody(workflow_config): + """Print each run catalogue's custody; stop the launch if one cannot run. + + Resolves every version the config names, so a blinded catalogue without a + blind fails here, with the command to run, before any job is scheduled. + """ + from snakemake.exceptions import WorkflowError + + if "type" in workflow_config.get("cosmo_val", {}): + raise WorkflowError( + "cosmo_val.type is not a setting: custody is declared per catalogue " + "in cosmo_val/cat_config.yaml. Remove it from the config." + ) + versions = [ + *workflow_config.get("versions", []), + *(FIDUCIAL.get(k) for k in ("version", "mock_version") if FIDUCIAL.get(k)), + ] + try: + lines = _custody.summary(CATALOG_CONFIG, versions, registry=REGISTRY) + except _custody.CustodyError as err: + raise WorkflowError(str(err)) from None + _print_once(tuple(lines)) + + +@functools.cache +def _print_once(lines): + """Print ``lines`` once per launch, however many Snakefiles configure.""" + for line in lines: + print(line, file=sys.stderr) + + +def custody_token(version): + """The custody token of ``version``: a producer's ``params`` trigger. + + A producer carrying it re-runs when its catalogue's custody changes, and its + job refuses to run under any other custody. + """ + return _custody.custody_of(CATALOG_CONFIG, version, registry=REGISTRY).token + + def fiducial_binning_suffix(fiducial=None): """Return binning suffix for fiducial parameters.""" fiducial = fiducial or FIDUCIAL @@ -355,12 +378,11 @@ def covariance_path( def base_version(version): - """Strip the derived-catalogue suffixes to the base catalogue version. + """The base catalogue of ``version``: its entry, then its ``base:`` links. - The `_leak_corr` / `_ecut{N}` variants share their parent's n(z) and - `cov_th` survey parameters, so lookups keyed on either must strip both. + Variants share their base's n(z) and plotting style. """ - return re.sub(r"_ecut\d+", "", re.sub(r"_leak_corr$", "", version)) + return _custody.base_catalogue(CATALOG_CONFIG, version) def build_redshift_path(version, blind): @@ -445,33 +467,13 @@ def grid_of(grids, binning): def pseudo_cl_tag(config): """Fiducial harmonic-binning tag stamped into pseudo-Cl filenames.""" - return f"blind={config['harmonic']['fiducial']['blind']}_{pseudo_cl_binning_tag(config)}" - - -def pseudo_cl_binning_tag(config): - """The `{binning}_nbins={n}` half of the tag — the binning alone.""" fiducial = config["harmonic"]["fiducial"] - return f"{fiducial['binning']}_nbins={fiducial['nbins']}" - - -def pseudo_cl_analysis_stem(config, version): - """Stem of the analysis pseudo-Cℓ part for a version. - - Its own name (and its own producing rule, twopoint.smk), distinct from the - generic `pseudo_cl` variants the bmodes and mock workflows request: only - this part is blindable, so only it can be temp()'d on a data run. Single - definition shared by the producer, the assembler (cosmo_val.smk) and the - blindable-stem regex (blinding.smk). - - Carries the binning but no `blind=` field: the A/B/C blind is the legacy - n(z) vocabulary (#312), not this part's concealment. - """ - return f"pseudo_cl_analysis_{version}_{pseudo_cl_binning_tag(config)}" + return f"blind={fiducial['blind']}_{fiducial['binning']}_nbins={fiducial['nbins']}" def get_shear_catalog(wildcards): """Resolve shear catalog path from config for a given version.""" - cat_config = CATALOG_CONFIG[wildcards.version.replace("_leak_corr", "")] + cat_config = CATALOG_CONFIG[_custody.entry_of(wildcards.version)] shear_path = cat_config["shear"]["path"] if shear_path.startswith("/"): return shear_path @@ -479,98 +481,6 @@ def get_shear_catalog(wildcards): return str(Path(subdir) / shear_path) -# --------------------------------------------------------------------------- -# Smokescreen blind-at-birth custody -# --------------------------------------------------------------------------- -# Distinct from the glass-mock A/B/C `blind` wildcard above: this is Smokescreen -# concealment (see sp_validation.blinding). RUN_TYPE is the single switch: a -# `data` run binds every ξ-derived consumer to the *_blinded parts, pulling the -# blind_part → blind_init subgraph into the DAG; a `mock` run binds the -# plaintext parts and the subgraph never appears. - - -def run_type(): - """The campaign's run type, ``"data"`` or ``"mock"``. - - Every part writer stamps this as the SACC ``type`` metadata, which is what - ``blinding.assert_consistent_blind`` reads at assembly. A function - rather than the ``RUN_TYPE`` global because ``from common import *`` binds - names before ``configure()`` runs, so only a call reads the configured - value. - """ - return RUN_TYPE - - -def is_data_run(): - """True when blinding is active (production data runs); False for mocks.""" - return run_type() == "data" - - -def blind_root(): - """Root holding one blind-init directory per version, or None on mock runs. - - What `CosmologyValidation(blind_root=...)` takes, so its part writers can - resolve each version's commitment.json themselves. - """ - return str(COSMO_VAL / "blind") if is_data_run() else None - - -def blind_state_dir(version): - """Per-version blind-init custody directory (commitment + encrypted seed).""" - return str(COSMO_VAL / "blind" / version) - - -def blind_state_paths(version): - """The fixed custody-state files blind_init writes for a version.""" - return init_paths(blind_state_dir(version)) - - -def commitment_input(version): - """Input mapping binding a version's commitment.json, on data runs only. - - A part writer stamps its output concealed from that file (sacc_io.save's - `commitment=`), which is what lets a born-blinded or blind-irrelevant part - clear the fail-closed load gate at assembly. - """ - if not is_data_run(): - return {} - return {"commitment": blind_state_paths(version)["commitment"]} - - -def blinded_path(part_path): - """The *_blinded sibling blind_part writes beside a plaintext part.""" - return part_paths(part_path)["blinded"] - - -def version_of(stem): - """Catalogue version embedded in a blindable part's stem. - - blind_part needs it to locate the version's blind state. - """ - m = re.search(WILDCARD_CONSTRAINTS["version"], stem) - if m is None: - raise ValueError(f"no catalogue version found in part stem {stem!r}") - return m.group(0) - - -def blindable_part(part_path): - """On-disk path a run persists for one blindable part. - - Data run -> the blinded sibling (binding it pulls blind_part + blind_init - into the DAG); mock run -> the plaintext part. - """ - return blinded_path(part_path) if is_data_run() else str(part_path) - - -def maybe_temp(part_path): - """temp() a producer's blindable plaintext part on data runs. - - Its only consumer there is blind_part, which escrows the true vector before - Snakemake removes the file, so no plaintext blindable part persists. - """ - return temp(str(part_path)) if is_data_run() else str(part_path) - - # --------------------------------------------------------------------------- # CosmologyValidation diagnostic suite (cosmo_val.py) # --------------------------------------------------------------------------- @@ -647,9 +557,5 @@ def cv_init_params(config): versions=config["versions"], catalog_config=CAT_CONFIG, output_dir=str(COSMO_VAL), - # Custody state the cv's part writers need: the SACC `type` they stamp, - # and the blind whose commitment born-blinded parts are stamped under. - run_type=run_type(), - blind_root=blind_root(), **{key: cv[key] for key in CV_INIT_KEYS}, ) diff --git a/workflow/rules/blinding.smk b/workflow/rules/blinding.smk deleted file mode 100644 index d951ab80..00000000 --- a/workflow/rules/blinding.smk +++ /dev/null @@ -1,61 +0,0 @@ -# Smokescreen blind-at-birth custody rules (sp_validation.blinding). -# -# Dormant unless a consumer binds a *_blinded part through common.blindable_part, -# which only a `data` run does. The terminal assemble_sacc rule (cosmo_val.smk) -# asserts the shared blind across parts. - -import re - -# The blindable stems, derived from the same name-builders the producing rules -# use so part names have one authority: the binning-named ξ± parts (rule xi, one -# per named grid) and the analysis pseudo-Cℓ. None contains "_blinded", so the -# generic blind_part rule can never blind its own output twice. -_VERSION_SLOT = "0VERSION0" # regex-inert placeholder, substituted after escaping -BLINDABLE_STEM = "(?:{})".format( - "|".join( - re.escape(stem) - for stem in ( - [f"{_VERSION_SLOT}_xi_{xi_binning(grid)}" for grid in XI_GRIDS] - + [pseudo_cl_analysis_stem(config, _VERSION_SLOT)] - ) - ).replace(_VERSION_SLOT, WILDCARD_CONSTRAINTS["version"]) -) - - -rule blind_init: - """Draw the seed and publish the commitment + encrypted bundle for a version.""" - output: - commitment=str(COSMO_VAL / "blind" / "{version}" / "commitment.json"), - bundle=str(COSMO_VAL / "blind" / "{version}" / "blind_seed.encrpt"), - key=str(COSMO_VAL / "blind" / "{version}" / "blind_seed.key"), - params: - blind_dir=lambda w: blind_state_dir(w.version), - resources: - runtime=5, - shell: - "python {REPO_SCRIPTS}/blind_data_vector.py" - " blind-init {params.blind_dir}" - - -rule blind_part: - """Conceal one part, escrowing its true vector beside the blinded output.""" - input: - part=str(COSMO_VAL / "{stem}.sacc"), - commitment=lambda w: blind_state_paths(version_of(w.stem))["commitment"], - bundle=lambda w: blind_state_paths(version_of(w.stem))["bundle"], - key=lambda w: blind_state_paths(version_of(w.stem))["key"], - output: - blinded=str(COSMO_VAL / "{stem}_blinded.sacc"), - escrow=str(COSMO_VAL / "{stem}_escrow.encrpt"), - escrow_key=str(COSMO_VAL / "{stem}_escrow.key"), - wildcard_constraints: - stem=BLINDABLE_STEM, - params: - blind_dir=lambda w: blind_state_dir(version_of(w.stem)), - resources: - runtime=10, - # --keep-input: the plaintext part is the producing rule's temp() output, so - # Snakemake removes it once this, its only consumer, finishes. - shell: - "python {REPO_SCRIPTS}/blind_data_vector.py" - " blind-part {input.part} --blind-dir {params.blind_dir} --keep-input" diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index e4fa0dd4..ebb6835e 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -163,8 +163,8 @@ _PSEUDO_CL_TAG = pseudo_cl_tag(config) def cv_pseudo_cl_analysis_sacc(version): - """Analysis pseudo-Cl SACC part: the harmonic block of the analysis file.""" - return str(COSMO_VAL / f"{pseudo_cl_analysis_stem(config, version)}.sacc") + """Tagged pseudo-Cl SACC part: the harmonic block of the analysis file.""" + return str(COSMO_VAL / f"pseudo_cl_{version}_{_PSEUDO_CL_TAG}.sacc") def cv_pseudo_cl_cov(version): @@ -376,9 +376,7 @@ def cv_pseudo_cl_figures(): rule cv_plot_pseudo_cl: """The EE/EB/BB pseudo-Cl figures, from the analysis parts.""" input: - pseudo_cl=[ - blindable_part(cv_pseudo_cl_analysis_sacc(v)) for v in CV_VERSIONS - ], + pseudo_cl=[cv_pseudo_cl_analysis_sacc(v) for v in CV_VERSIONS], pseudo_cl_cov=[cv_pseudo_cl_cov(v) for v in CV_VERSIONS], output: **cv_pseudo_cl_figures(), @@ -398,25 +396,6 @@ rule cv_plot_pseudo_cl: # Pure E/B modes and COSEBIs (per version), then the B-mode summary # --------------------------------------------------------------------------- -# On a data run these re-derive their E-mode vector from the *blinded* ξ± parts, -# so they are born blinded; the commitment binds only there. -def cv_cosebis_inputs(w): - return { - "xi": blindable_part(cv_xi_sacc(w.version, "integration")), - "cov": cv_xi_cov_integration(w.version), - **commitment_input(w.version), - } - - -def cv_pure_eb_inputs(w): - return { - "xi_reporting": blindable_part(cv_xi_sacc(w.version, "reporting")), - "xi_integration": blindable_part(cv_xi_sacc(w.version, "integration")), - "cov_integration": cv_xi_cov_integration(w.version), - **commitment_input(w.version), - } - - rule cv_pure_eb: """Pure E/B-mode decomposition for one version, from its ξ± parts. @@ -424,14 +403,15 @@ rule cv_pure_eb: integration-grid covariance model, so no patched estimator run is involved. """ input: - unpack(cv_pure_eb_inputs), + xi_reporting=lambda w: cv_xi_sacc(w.version, "reporting"), + xi_integration=lambda w: cv_xi_sacc(w.version, "integration"), + cov_integration=lambda w: cv_xi_cov_integration(w.version), output: npz=cv_pure_eb_npz("{version}"), sacc=cv_pure_eb_sacc("{version}"), **cv_pure_eb_figures("{version}"), params: version="{version}", - type=CV.get("type", "data"), min_sep=CV["theta_min"], max_sep=CV["theta_max"], nbins=CV["nbins"], @@ -453,14 +433,14 @@ rule cv_cosebis: on the same grid through the same kernel as the modes. """ input: - unpack(cv_cosebis_inputs), + xi=lambda w: cv_xi_sacc(w.version, "integration"), + cov=lambda w: cv_xi_cov_integration(w.version), output: npz=cv_cosebis_npz("{version}"), sacc=cv_cosebis_sacc("{version}"), **cv_cosebis_figures("{version}"), params: version="{version}", - type=CV.get("type", "data"), min_sep=XI_GRIDS["integration"]["min_sep"], max_sep=XI_GRIDS["integration"]["max_sep"], nbins=XI_GRIDS["integration"]["nbins"], @@ -481,7 +461,7 @@ rule cv_summarize_bmodes: pure_eb=[cv_pure_eb_npz(v) for v in CV_VERSIONS], cosebis=[cv_cosebis_npz(v) for v in CV_VERSIONS], pseudo_cl=( - [blindable_part(cv_pseudo_cl_analysis_sacc(v)) for v in CV_VERSIONS] + [cv_pseudo_cl_analysis_sacc(v) for v in CV_VERSIONS] if CV.get("include_pseudo_cl", False) else [] ), pseudo_cl_cov=( @@ -518,18 +498,15 @@ def cv_assemble_inputs(version): Each part's filename carries enough to bind its producing rule's wildcards. """ - # blindable_part binds the raw-signal parts to their blinded siblings on a - # data run. COSEBIs, pure-E/B and ρ/τ are stamped concealed by their own - # writers, so they bind by name either way. parts = dict( - xi_reporting=blindable_part(cv_xi_sacc(version, "reporting")), + xi_reporting=cv_xi_sacc(version, "reporting"), xi_cov=cv_xi_cov(version), cosebis=cv_cosebis_sacc(version), pure_eb=cv_pure_eb_sacc(version), rho_tau=cv_rho_tau_sacc(version), ) if CV.get("include_pseudo_cl", False): - parts["pseudo_cl"] = blindable_part(cv_pseudo_cl_analysis_sacc(version)) + parts["pseudo_cl"] = cv_pseudo_cl_analysis_sacc(version) parts["pseudo_cl_cov"] = cv_pseudo_cl_cov(version) return parts @@ -542,7 +519,8 @@ rule assemble_sacc: sacc=cv_analysis_sacc("{version}"), params: version="{version}", - type=CV.get("type", "data"), + cat_config=CAT_CONFIG, + custody=lambda w: custody_token(w.version), # The statistics this rule wired, so a typo'd input keyword cannot # silently drop one. expected=lambda w: [ diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 1f61003f..4b3836aa 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -23,8 +23,7 @@ rule xi: catalog=get_shear_catalog, output: txt=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), - # Blindable part: temp() on a data run so only its blinded sibling persists. - sacc=maybe_temp(str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc")), + sacc=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 24 params: ver="{version}", @@ -35,7 +34,7 @@ rule xi: cat_config=CAT_CONFIG, output_dir=str(COSMO_VAL), grid=lambda w: grid_of(XI_GRIDS, w), - type=run_type(), # the part's SACC `type` — custody state at assembly + custody=lambda w: custody_token(w.version), resources: # The fine integration grid needs more memory and wall time than the # ~20-bin reporting one; scale on nbins rather than splitting the rule. @@ -47,10 +46,6 @@ rule xi: rule rho_tau_stats: - # ρ/τ has no blindable input; it binds the commitment only to stamp its part - # concealed pass-through. - input: - unpack(lambda w: commitment_input(w.version)), output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), @@ -65,8 +60,7 @@ rule rho_tau_stats: npatch="{npatch}", cat_config=CAT_CONFIG, output_dir=str(COSMO_VAL), - type=run_type(), - blind_root=blind_root(), + custody=lambda w: custody_token(w.version), resources: mem_mb=30000, disk_mb=20000, @@ -81,22 +75,8 @@ wildcard_constraints: binning="linear|logspace|powspace", -HARMONIC_FIDUCIAL = config["harmonic"]["fiducial"] -PSEUDO_CL_PARAMS = dict( - cat_config=CAT_CONFIG, - nside=1024, - npatch=1, - cosmo_params=PLANCK18, - power=0.5, -) - - rule pseudo_cl: - """Generate pseudo-Cl data vector (born as SACC) with configurable binning. - - The diagnostic variants (bmodes claims, mocks, the fine COSEBIs binning); - the analysis part has its own rule below. - """ + """Generate pseudo-Cl data vector (born as SACC) with configurable binning.""" output: pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.sacc"), wildcard_constraints: @@ -104,34 +84,14 @@ rule pseudo_cl: params: version="{version}", blind="{blind}", + cat_config=CAT_CONFIG, + nside=1024, + npatch=1, + cosmo_params=PLANCK18, binning="{binning}", nbins=lambda w: int(w.nbins), - **PSEUDO_CL_PARAMS, - resources: - mem_mb=32000, - runtime=120, - threads: 12 - script: - "../scripts/generate_pseudo_cl.py" - - -rule pseudo_cl_analysis: - """The analysis pseudo-Cℓ part, at the fiducial harmonic binning. - - Split from the generic `pseudo_cl` rule because this variant alone is - blindable: on a data run the plaintext part is temp(), consumed only by - blind_part, so only the blinded sibling persists. - """ - output: - pseudo_cl=maybe_temp( - str(COSMO_VAL / f"{pseudo_cl_analysis_stem(config, '{version}')}.sacc") - ), - params: - version="{version}", - blind=HARMONIC_FIDUCIAL["blind"], - binning=HARMONIC_FIDUCIAL["binning"], - nbins=int(HARMONIC_FIDUCIAL["nbins"]), - **PSEUDO_CL_PARAMS, + power=0.5, + custody=lambda w: custody_token(w.version), resources: mem_mb=32000, runtime=120, diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 9c6198f4..786ccd9d 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -14,11 +14,14 @@ """ import argparse +from pathlib import Path import numpy as np +import yaml from sp_validation import sacc_io from sp_validation.cosmo_val.sacc_writers import assemble_analysis_sacc +from sp_validation.custody import confirm, custody_of, registry_of # NaMaster iNKA covariance FITS: per-spectrum HDU names, in SACC insertion order. _CL_HDUS = ("COVAR_EE_EE", "COVAR_BB_BB", "COVAR_EB_EB") @@ -86,18 +89,30 @@ def _attach_cov(part, name, xi_cov, pseudo_cl_cov): return part +def declared_custody(cat_config, version): + """The custody ``version`` is declared under in ``cat_config``.""" + catalogues = yaml.safe_load(Path(cat_config).read_text()) + return custody_of(catalogues, version, registry=registry_of(cat_config)) + + def assemble_sacc( version, part_paths, out_path, *, + custody, expected=None, xi_cov=None, pseudo_cl_cov=None, - allow_unblinded=False, ): """Assemble ``{version}.sacc`` from the per-statistic ``part_paths`` mapping. + @sc one-custody-per-assembly + Every part, signal-bearing or not, must carry one stamp, and it must be the + custody ``version`` is declared under: a stale concealed part after a + reveal, a part under another blind or catalogue, a mock in data and + unblinded parts in a blinded file are all refused. + Parameters ---------- version : str @@ -108,10 +123,10 @@ def assemble_sacc( expected : sequence of str, optional Statistics that must be present, from the caller's config toggles. A typo'd input keyword would otherwise silently drop a statistic. + custody : sp_validation.custody.Custody + The custody ``version`` is declared under. xi_cov, pseudo_cl_cov Covariance sourcing — see the module docstring. - allow_unblinded : bool, optional - Passed to :func:`sacc_io.load` for every part; ``True`` only for mocks. """ if expected is not None: unknown = [name for name in expected if name not in CANONICAL] @@ -132,14 +147,12 @@ def assemble_sacc( path = part_paths.get(name) if path is None: continue - part = sacc_io.load(path, allow_unblinded=allow_unblinded) - parts.append(_attach_cov(part, name, xi_cov, pseudo_cl_cov)) + parts.append(_attach_cov(sacc_io.load(path), name, xi_cov, pseudo_cl_cov)) if not parts: raise ValueError(f"no parts found for {version}: {part_paths}") - # Through sacc_io.gather, the one terminal seam: it fails closed unless every - # blindable part shares one blind, and stamps that blind on the result. - s = sacc_io.gather(parts, assemble=assemble_analysis_sacc) - sacc_io.save(s, out_path, type=s.metadata["type"]) + s = sacc_io.save( + assemble_analysis_sacc(parts), out_path, derived_from=parts, custody=custody + ) print(f"Assembled {len(parts)} parts -> {out_path}") return s @@ -156,10 +169,10 @@ def _from_snakemake(smk): version=p["version"], part_paths=part_paths, out_path=str(smk.output[0]), + custody=confirm(declared_custody(p["cat_config"], p["version"]), p["custody"]), expected=list(p["expected"]), xi_cov=getattr(inp, "xi_cov", None), pseudo_cl_cov=getattr(inp, "pseudo_cl_cov", None), - allow_unblinded=(p.get("type", "data") == "mock"), ) @@ -170,11 +183,9 @@ def _from_cli(argv=None): ap.add_argument("--version", required=True, help="Catalogue version") ap.add_argument("--out", required=True, help="Output {version}.sacc path") ap.add_argument( - "--type", - choices=("data", "mock"), - default="data", - help="Run type. 'mock' reads parts freely; 'data' fails closed on " - "unblinded parts (only concealed/blinded parts load).", + "--cat-config", + required=True, + help="cat_config.yaml declaring the catalogue's custody", ) for name in CANONICAL: ap.add_argument( @@ -190,9 +201,9 @@ def _from_cli(argv=None): version=a.version, part_paths=part_paths, out_path=a.out, + custody=declared_custody(a.cat_config, a.version), xi_cov=a.xi_cov, pseudo_cl_cov=a.pseudo_cl_cov, - allow_unblinded=(a.type == "mock"), ) diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index 081d6b75..c0962afc 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -3,10 +3,8 @@ A consumer of the integration-grid ξ± part and the CosmoCov ξ± covariance on the same grid — nothing here touches a catalogue. The covariance goes through the same linear kernel as the modes to give the COSEBIs covariance; it is -analytic, so no Hartlap debiasing applies. - -On a data run the part it reads is the blinded one, so these COSEBIs are born -blinded and the output is stamped under the same commitment. +analytic, so no Hartlap debiasing applies. The COSEBIs part is a derivation of +the ξ± part and carries its custody. """ import numpy as np @@ -65,19 +63,7 @@ save_cosebis_results(results, snakemake.output["npz"], fiducial_scale_cut) -# The part inherits the ξ± part's provenance; `type` and the blind stamp are -# re-applied on save, from the run type and the version's commitment. -metadata = { - k: v - for k, v in part.metadata.items() - if k not in ("type", "concealed", "blind_commitment", "blind_config_digest") -} -s = cosebis_to_sacc({0: sacc_io.get_nz(part, 0)}, metadata, fiducial, fiducial_key) -sacc_io.save( - s, - snakemake.output["sacc"], - type=p["type"], - commitment=snakemake.input.get("commitment", None), -) +s = cosebis_to_sacc({0: sacc_io.get_nz(part, 0)}, part.metadata, fiducial, fiducial_key) +sacc_io.save(s, snakemake.output["sacc"], derived_from=[part]) verify_outputs(snakemake) diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index 56ce49e7..71bffedd 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -6,10 +6,9 @@ drawn from the CosmoCov integration-grid ξ± covariance around a theory mean, so it depends on the covariance model and the grids rather than on the measured vector. A jackknife of the transformed modes would need per-patch realisations, -which are never persisted. - -On a data run the parts it reads are the blinded ones, so these modes are born -blinded and the output is stamped under the same commitment. +which are never persisted. The Monte Carlo draws are seeded, so the +covariance is a function of the covariance model and the grids alone. The +pure-E/B part is a derivation of the two ξ± parts and carries their custody. """ import numpy as np @@ -58,6 +57,7 @@ nz=nz, cosmo=get_cosmo(**p["cosmo_params"]), n_samples=p["n_samples"], + rng=np.random.default_rng(0), ) variances = reporting.covariance.dense.diagonal() @@ -102,25 +102,13 @@ save_pure_eb_results(results, snakemake.output["npz"]) -# The part inherits the ξ± part's provenance; `type` and the blind stamp are -# re-applied on save, from the run type and the version's commitment. -metadata = { - k: v - for k, v in reporting.metadata.items() - if k not in ("type", "concealed", "blind_commitment", "blind_config_digest") -} s = pure_eb_to_sacc( {0: (z, nz)}, - metadata, + reporting.metadata, theta, {key: results[key] for key in sacc_io.PURE_KEYS}, covariance=cov, ) -sacc_io.save( - s, - snakemake.output["sacc"], - type=p["type"], - commitment=snakemake.input.get("commitment", None), -) +sacc_io.save(s, snakemake.output["sacc"], derived_from=[reporting, integration]) verify_outputs(snakemake) diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index e25f6ed9..c94f5640 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -24,8 +24,8 @@ import json import os -from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation +from sp_validation.custody import confirm def generate_pseudo_cl( @@ -39,6 +39,7 @@ def generate_pseudo_cl( binning: str = "linear", nbins: int = None, power: float = 0.5, + custody: str = None, ): """Generate a pseudo-Cl data vector, born as a SACC part at ``out_path``. @@ -48,7 +49,7 @@ def generate_pseudo_cl( Catalog version (e.g., "SP_v1.4.6_leak_corr") out_path : str Exact destination the SACC part is born at — its final (possibly tagged) - name. Skip-if-exists keys on it, so no two rules share a basename. + name. cat_config : str Path to catalog configuration YAML nside : int @@ -66,6 +67,9 @@ def generate_pseudo_cl( Number of ell bins (required) power : float Power for powspace binning (0.5 = sqrt spacing) + custody : str, optional + The custody token the launch resolved for ``version``; the part is not + written under any other. Returns ------- @@ -129,16 +133,15 @@ def generate_pseudo_cl( cv_kwargs["power"] = power cv = CosmologyValidation(**cv_kwargs) + if custody is not None: + confirm(cv.custody(version), custody) # Pseudo-Cls only (no covariance), born directly at the final out_path. cv.calculate_pseudo_cl(out_path=out_path) - if os.path.exists(out_path): - # Readback of the part just written — a legitimate pre-blind consumer. - s = sacc_io.load(out_path, allow_unblinded=True) - ell = sacc_io.get_pseudo_cl(s, (0, 0))[0] - print(f"Generated pseudo-Cl with {len(ell)} ell bins") - print(f"ell range: [{ell.min():.1f}, {ell.max():.1f}]") + ell = cv.pseudo_cls[version]["pseudo_cl"]["ELL"] + print(f"Generated pseudo-Cl with {len(ell)} ell bins") + print(f"ell range: [{ell.min():.1f}, {ell.max():.1f}]") return out_path @@ -155,6 +158,7 @@ def _from_snakemake(smk): binning=p["binning"], nbins=int(p["nbins"]), power=float(p.get("power", 0.5)), + custody=p["custody"], ) diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index fb73712f..6dc074af 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -16,9 +16,10 @@ grids are the same compute with different ``--min-sep/--max-sep/--nbins``. ``CosmologyValidation.calculate_2pcf`` writes the ``.txt`` dump (a raw byproduct); the ξ± data product is born as SACC here, a *part* named by its -binning and tagged with its ``--grid``. The part carries the covariance the -measurement estimated: the dense jackknife covariance when it had patches, the -shot-noise ``varxip``/``varxim`` diagonal when it had none. +binning and tagged with its ``--grid``, sealed under the catalogue's custody. +The part carries the covariance the measurement estimated: the dense jackknife +covariance when it had patches, the shot-noise ``varxip``/``varxim`` diagonal +when it had none. ``output_dir`` is passed explicitly so lc can point each run at its own ``{output}`` tree. @@ -32,6 +33,7 @@ from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation from sp_validation.cosmo_val.sacc_writers import xi_to_sacc +from sp_validation.custody import confirm def run_2pcf( @@ -44,7 +46,7 @@ def run_2pcf( output_dir, sacc_out=None, grid="reporting", - run_type="data", + custody=None, ): """Measure ξ±(θ) for ``ver`` and write its born-as-SACC part. @@ -55,8 +57,9 @@ def run_2pcf( ``cat_config['paths']['output']`` so the ``.txt`` byproduct lands where lc expects. ``sacc_out`` is the exact destination for the SACC part (the Snakemake-declared output); it defaults to a binning-derived name under - the resolved output directory for the CLI path. ``run_type`` (``"data"`` or - ``"mock"``) is stamped as the part's SACC ``type``. + the resolved output directory for the CLI path. ``custody`` is the custody + token the launch resolved for ``ver``; the part is not written under any + other. Returns ------- @@ -70,6 +73,9 @@ def run_2pcf( # so the SACC provenance metadata stamps the npatch actually measured npatch=npatch, ) + declared = cv.custody(ver) + if custody is not None: + confirm(declared, custody) gg = cv.calculate_2pcf( ver=ver, npatch=npatch, @@ -97,7 +103,7 @@ def run_2pcf( output_dir or cv.cc["paths"]["output"], f"{ver}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc", ) - sacc_io.save(s, out_path, type=run_type) + sacc_io.save(s, out_path, custody=declared) print(f"Wrote {grid} ξ± SACC part: {out_path}") return gg @@ -116,7 +122,7 @@ def _from_snakemake(smk): # The SACC part goes exactly where the rule declares it; the .txt # byproduct still lands under the resolved output dir. sacc_out=smk.output["sacc"], - run_type=p["type"], + custody=p["custody"], ) @@ -146,12 +152,6 @@ def _from_cli(argv=None): ap.add_argument( "--grid", default="reporting", help="SACC grid tag for the measured points" ) - ap.add_argument( - "--run-type", - default="data", - choices=("data", "mock"), - help="Campaign run type stamped as the part's SACC `type`", - ) a = ap.parse_args(argv) run_2pcf( ver=a.ver, @@ -162,7 +162,6 @@ def _from_cli(argv=None): cat_config=a.cat_config, output_dir=a.out, grid=a.grid, - run_type=a.run_type, ) diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index 53b2499b..54731d56 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -1,40 +1,19 @@ -# %% -# if interactive -import os -import sys -from pathlib import Path - -from IPython import get_ipython - -ipython = get_ipython() - -# enable autoreload for interactive sessions -if ipython is not None: - ipython.run_line_magic("load_ext", "autoreload") - ipython.run_line_magic("autoreload", "2") -else: - # Force unbuffered stdout and stderr - sys.stdout = os.fdopen(sys.stdout.fileno(), "w", buffering=1) # line-buffered - sys.stderr = os.fdopen(sys.stderr.fileno(), "w", buffering=1) +"""Rule rho_tau_stats: ρ/τ PSF statistics for one version. -from sp_validation.cosmo_val import CosmologyValidation # noqa: E402 +Writes the ρ/τ FITS tables and the version's ρ/τ SACC part, sealed under the +catalogue's custody. +""" -print("Finished imports") - -if ipython is not None: - from snakemake_helpers import snakemake_interactive +from pathlib import Path - snakemake = snakemake_interactive( - "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/output/rho_tau_stats/rho_stats_SP_v1.4.5.fits", - "/home/cdaley/n17data/unions/pure_eb", - ) +from cv_runner import _unbuffer_streams -params = snakemake.params # type: ignore +from sp_validation.cosmo_val import CosmologyValidation +from sp_validation.custody import confirm -# %% -print("Starting CosmologyValidation") +_unbuffer_streams() +params = snakemake.params # noqa: F821 — injected by Snakemake's script: directive -# Use parameters passed from Snakemake rule cv = CosmologyValidation( versions=[params["ver"]], theta_min=float(params["min_sep"]), @@ -43,15 +22,11 @@ npatch=int(params["npatch"]), catalog_config=params["cat_config"], output_dir=params["output_dir"], - run_type=params["type"], - blind_root=params["blind_root"], ) - +confirm(cv.custody(params["ver"]), params["custody"]) cv.calculate_rho_tau_stats() -# Confirm CosmologyValidation produced the requested outputs: the rho/tau FITS -# and the born-as-SACC rho_tau part. -outputs = snakemake.output # type: ignore +outputs = snakemake.output # noqa: F821 for label in ("rho_stats", "tau_stats", "rho_tau"): target = Path(outputs[label]) if not target.exists(): diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py index 29ebf5c4..720e5ae6 100644 --- a/workflow/tests/conftest.py +++ b/workflow/tests/conftest.py @@ -11,15 +11,20 @@ Snakemake is in. The ``candide`` tests need candide itself; CI deselects them. The ``toy`` fixture is a disposable checkout: copies of ``workflow/`` and -``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, a one-catalogue -``cosmo_val/cat_config.yaml``, a touched catalogue file, the processed CosmoCov +``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, a +``cosmo_val/cat_config.yaml`` declaring one catalogue per custody state over a +touched catalogue file, a blind registry of hand-written records (the host +never decrypts, so no record needs a real seed), the processed CosmoCov covariances already in place (their inputs live on candide), and both output roots in tmp. Its runs use a fake image whose Python and Snakemake match the running ones, so the launch-time parity check passes without apptainer. """ +import copy import dataclasses +import hashlib import importlib.util +import json import os import re import shutil @@ -33,8 +38,12 @@ REPO = Path(__file__).resolve().parents[2] -# The toy catalogue and its leakage-corrected variant. +# The toy catalogue and its leakage-corrected variant: blinded under `toy`. VERSIONS = ("SP_v0.1", "SP_v0.1_leak_corr") +# Declared unblinded, and covered by the blind `stale`, which is not revealed. +STALE = "SP_v0.5" +# Declares no custody, and no blind covers it. +UNCOVERED = "SP_v0.4" HOST_PYTHON = ".".join(str(v) for v in sys.version_info[:3]) @@ -121,16 +130,20 @@ class Toy: common: object covariances: dict # (version, "g" | "ng") -> the processed CosmoCov file - def snakemake(self, *args, container=None, cwd=None, env=None, timeout=300): + def snakemake( + self, *args, container=None, config=(), cwd=None, env=None, timeout=300 + ): """Run the host Snakemake in the toy's paper directory, or in ``cwd``. ``container`` overrides the image (default: the matching fake one; - ``False`` leaves the choice to the launch). ``env`` replaces the toy's - environment. + ``False`` leaves the choice to the launch). ``config`` adds + ``KEY=VALUE`` overrides. ``env`` replaces the toy's environment. """ cmd = [sys.executable, "-m", "snakemake", "--cores", "1", *args] if container is not False: - cmd += ["--config", f"container={container or self.image}"] + config = (f"container={container or self.image}", *config) + if config: + cmd += ["--config", *config] return subprocess.run( cmd, cwd=cwd or self.rundir, @@ -160,7 +173,32 @@ def _cat_config(catalogue): "e2_col_corrected": "e2_leak_corrected", }, } - return {VERSIONS[0]: entry, "paths": {"output": "./output"}} + return { + VERSIONS[0]: entry, + "SP_v0.2": dict(copy.deepcopy(entry), blinding="unblinded"), + "SP_v0.3": dict(copy.deepcopy(entry), blinding="mock"), + UNCOVERED: copy.deepcopy(entry), + STALE: dict(copy.deepcopy(entry), blinding="unblinded"), + "paths": {"output": "./output"}, + } + + +def _registry(root): + """Blind records as ``blinding init`` commits them, minus the ciphertext.""" + for name, bases in (("toy", [VERSIONS[0]]), ("stale", [STALE])): + record = root / "cosmo_val" / "blinds" / name + record.mkdir(parents=True) + (record / "commitment.json").write_text( + json.dumps( + { + "blind": name, + "seed_commitment": hashlib.sha256(name.encode()).hexdigest(), + "config_digest": hashlib.sha256(b"config").hexdigest(), + "draw_scheme": 2, + } + ) + ) + (record / "bases").write_text("".join(f"{b}\n" for b in bases)) @pytest.fixture(scope="session") @@ -180,6 +218,7 @@ def toy(tmp_path_factory): (root / "cosmo_val" / "cat_config.yaml").write_text( yaml.safe_dump(_cat_config(catalogue)) ) + _registry(root) rundir = root / "papers" / "cosmo_val" config_path = rundir / "config" / "config.yaml" diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index ec3f7559..34181509 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -7,7 +7,16 @@ import pytest import yaml -from conftest import HOST_PYTHON, REPO, VERSIONS, fake_image, on_candide, parse_jobs +from conftest import ( + HOST_PYTHON, + REPO, + STALE, + UNCOVERED, + VERSIONS, + fake_image, + on_candide, + parse_jobs, +) def test_assemble_resolves(toy): @@ -208,6 +217,75 @@ def test_image_sims_checks_parity_at_launch(toy, tmp_path): assert "uv tool install --force --python 3.13 snakemake==" in result.stdout +def _custody_lines(output): + return [line for line in output.splitlines() if line.startswith("[custody]")] + + +def test_a_catalogue_without_a_blind_stops_the_launch(toy): + """A blinded catalogue with no blind fails at parse, naming the one command.""" + result = toy.snakemake( + "-n", "assemble_sacc_all", config=[f'versions=["{UNCOVERED}"]'] + ) + assert result.returncode != 0, result.stdout + assert "python -m sp_validation.blinding init" in result.stdout + assert "share" in result.stdout + assert "rule assemble_sacc" not in result.stdout + + +def test_no_config_line_unblinds_a_catalogue(toy, tmp_path): + """Custody is read from the checkout's cat_config, never from the merged config.""" + override = tmp_path / "override.yaml" + override.write_text(yaml.safe_dump({VERSIONS[0]: {"blinding": "unblinded"}})) + result = toy.snakemake("-n", "assemble_sacc_all", "--configfile", str(override)) + assert result.returncode == 0, result.stdout + assert f"[custody] {VERSIONS[0]} (+ {VERSIONS[1]}): blinded under toy" in ( + _custody_lines(result.stdout) + ) + + +def test_a_catalogue_and_its_variant_share_one_custody(toy): + result = toy.snakemake("-n", "assemble_sacc_all") + assert result.returncode == 0, result.stdout + assert _custody_lines(result.stdout) == [ + f"[custody] {VERSIONS[0]} (+ {VERSIONS[1]}): blinded under toy" + ] + + +def test_no_rule_draws_or_touches_a_blind(toy): + """Even a forced run schedules nothing that reads or writes the registry.""" + result = toy.snakemake("-F", "-n", "all") + assert result.returncode == 0, result.stdout + jobs = parse_jobs(result.stdout) + registry = (toy.root / "cosmo_val" / "blinds").resolve() + assert jobs + assert not [j.rule for j in jobs if "blind" in j.rule] + touched = [ + f + for j in jobs + for f in j.input + j.output + if (toy.rundir / f).resolve().is_relative_to(registry) + ] + assert not touched, touched + + +def test_the_campaign_type_switch_is_refused(toy, tmp_path): + """A config carrying cosmo_val.type stops the launch with the pointer.""" + config = dict(toy.config, cosmo_val=dict(toy.config["cosmo_val"], type="mock")) + path = tmp_path / "config.yaml" + path.write_text(yaml.safe_dump(config)) + result = toy.snakemake("-n", "assemble_sacc_all", "--configfile", str(path)) + assert result.returncode != 0, result.stdout + assert "custody is declared per catalogue in cosmo_val/cat_config.yaml" in ( + result.stdout + ) + + +def test_unblinding_a_concealed_catalogue_needs_the_reveal(toy): + result = toy.snakemake("-n", "assemble_sacc_all", config=[f'versions=["{STALE}"]']) + assert result.returncode != 0, result.stdout + assert "blinding reveal stale" in result.stdout + + def _real_dry_run(paper, targets): env = {k: v for k, v in os.environ.items() if k != "SNAKEMAKE_PROFILE"} env.update(PYTHONUNBUFFERED="1", PYTHONNOUSERSITE="1") From 5219a5b569edfcaee7872469e7aad245955b269a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 02:13:56 +0200 Subject: [PATCH 080/160] sacc_io: a refused blinded birth names only its derived rows; custody: why the commitment domain is spelt here Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/custody.py | 3 ++- src/sp_validation/sacc_io.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/sp_validation/custody.py b/src/sp_validation/custody.py index 8a3dcb6e..4a80c4a0 100644 --- a/src/sp_validation/custody.py +++ b/src/sp_validation/custody.py @@ -26,7 +26,8 @@ # Top-level keys of the catalogue config that are not catalogues. NOT_CATALOGUES = ("nz", "paths") -# The Smokescreen fork's commitment prefix; test_blinding pins it to the fork's. +# smokescreen.COMMITMENT_DOMAIN, spelt here so this module needs only the +# standard library. COMMITMENT_DOMAIN = b"smokescreen-seed-commitment-v1|" # The stamp every SACC carries: the custody it was born under. diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index b863a9ae..de545f33 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -982,10 +982,11 @@ def seal(s, custody): ) types = {dp.data_type for dp in s.data} signal = {t for t in types if t.startswith(SIGNAL_PREFIX)} - if custody.status == "blinded" and signal - set(SHIFTABLE) - set(UNSHIFTED): + derived = signal - set(SHIFTABLE) - set(UNSHIFTED) + if custody.status == "blinded" and derived: raise ValueError( - f"a blinded catalogue's {sorted(signal - set(SHIFTABLE))} rows are " - "derived statistics: save them with derived_from=[their input parts]" + f"a blinded catalogue's {sorted(derived)} rows are derived " + "statistics: save them with derived_from=[their input parts]" ) if custody.status == "blinded" and signal & set(SHIFTABLE): from . import blinding From e7bb67bde9b5cbdb8518c0d2e1aa3fd9b58f0ded Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 02:28:32 +0200 Subject: [PATCH 081/160] =?UTF-8?q?Blinding:=20B-mode=20leakage=20bounded?= =?UTF-8?q?=20at=20the=20envelope's=20corners;=20the=20audit=20reports=20d?= =?UTF-8?q?erived=20shifts=20in=20=CF=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I4 evaluates the shift at the four (S8, Ωm) corners on the production grids: COSEBIs B-modes move by at most 8e-5σ, pure-mode ξ−_B by up to 1.0e-2σ at θ ≈ 2′ for (+0.075, −0.1), the estimator's own E→B leakage. The audit bounds COSEBIs B at 1e-2σ and pure ξ_B at 1e-1σ, and reports each derived statistic's largest shift. The end-to-end test runs under a fixed seed and a shape-noise covariance at the toy catalogue's density. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/blinding.py | 41 +++++++++++++-------- src/sp_validation/tests/test_blinding.py | 29 +++++++++++---- src/sp_validation/tests/test_custody_e2e.py | 19 ++++++++-- 3 files changed, 62 insertions(+), 27 deletions(-) diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 9cbfafb4..16e4bae9 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -477,6 +477,16 @@ def reveal(name, *, root, cat_config): # A re-measurement repeats a measurement to float noise (TreeCorr's threaded # sums reorder); the audit compares its numbers to this relative tolerance. _REMEASURED = 1e-10 +# How far, in σ, a derived B-mode may move under a blind. COSEBIs B-modes move +# by ~1e-4σ. Pure-mode ξ_B carries the estimator's own E→B leakage of the +# shift: up to 1e-2σ on the production grids at the envelope's edge +# (test_blinding), several times more on coarse grids. A derivation from mixed +# inputs moves B by ~1σ. +_B_SIGMA = { + sacc_io.COSEBI_BB: 1e-2, + sacc_io.PURE_TYPES["xip_B"]: 1e-1, + sacc_io.PURE_TYPES["xim_B"]: 1e-1, +} def _same(x, y): @@ -538,27 +548,25 @@ def _audit_part(blinded, true, fiducial, hidden, tolerance): problems.append(f"blinded − true ≠ shift(seed): residual {residual:.2e}") sigma = None if cov_t is None else np.sqrt(np.diag(cov_t)) - e_shift = {} + derived = (sacc_io.COSEBI_EE, sacc_io.COSEBI_BB, *sacc_io.PURE_TYPES.values()) + moved = {} for i, dp in enumerate(true.data): + kind = dp.data_type if shifted[i]: continue - kind = dp.data_type - b_mode = kind in ( - sacc_io.COSEBI_BB, - *[sacc_io.PURE_TYPES[k] for k in ("xip_B", "xim_B")], - ) - e_mode = kind in (sacc_io.COSEBI_EE, *sacc_io.PURE_TYPES.values()) - if b_mode: - if sigma is None or abs(delta[i]) > 1e-2 * sigma[i]: - problems.append(f"B row {i} ({kind}) moved by {delta[i]:.3e}") - elif e_mode: - if sigma is not None: - e_shift[kind] = max(e_shift.get(kind, 0.0), abs(delta[i]) / sigma[i]) + if kind in derived: + if sigma is None: + problems.append(f"row {i} ({kind}) has no σ to judge its shift by") + elif np.isfinite(delta[i]): + moved[kind] = max(moved.get(kind, 0.0), abs(delta[i]) / sigma[i]) elif abs(delta[i]) > 1e-8 * max(abs(dp.value), 1e-300): problems.append( f"row {i} ({kind}) moved, but the blind leaves it unshifted" ) - return problems, {"residual": residual, "e_shift_over_sigma": e_shift} + for kind, bound in _B_SIGMA.items(): + if moved.get(kind, 0.0) > bound: + problems.append(f"{kind} moved by {moved[kind]:.2e}σ under the blind") + return problems, {"residual": residual, "shift_over_sigma": moved} def audit(name, *, archive, true_root, cat_config, out=None): @@ -571,8 +579,9 @@ def audit(name, *, archive, true_root, cat_config, out=None): noise), and blinded − true must equal the seed's shift on every ξ± and Cℓ_EE block to 1e-6 of the block's largest shift (1e-3 when the theory stack differs from the one the blind was drawn with); COSEBIs and pure-E/B - B rows may move by at most 1e-2 σ, and rows the blind leaves unshifted not - at all. + B rows may move by at most ``_B_SIGMA``, and rows the blind leaves + unshifted not at all. Each part reports the largest shift of each derived + statistic, in σ. """ registry = _custody.registry_of(cat_config) record, commitment = _read_record(registry, name) diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index ac5c6bf6..d1b16242 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -366,15 +366,26 @@ def _shape_noise_variance(left, right): return 2.0 * sigma_e**4 / pairs -def test_shift_leaks_no_b_modes(blinds): - """|ΔBₙ|/σ(Bₙ) and |Δξ_B|/σ stay below 1e-2 on the production grids. - - Single-bin n(z), the 0.08–300′ 1000-bin integration grid and the 1–250′ - 20-bin reporting grid; σ is shape noise at UNIONS depth. The shift is a pure - E-mode theory difference, so the B-modes it induces are numerical only. +@pytest.mark.parametrize("ds8", [0.075, -0.075]) +@pytest.mark.parametrize("dom", [0.1, -0.1]) +def test_shift_leaks_no_b_modes(ds8, dom): + """B-modes a shift at the envelope's edge induces are numerical, and small. + + At each corner of the (S8, Ωm) envelope, on the production grids (the + 0.08–300′ 1000-bin integration grid, the 1–250′ 20-bin reporting grid), + single-bin n(z), σ from shape noise at UNIONS depth: |ΔBₙ|/σ(Bₙ) ≤ 1e-2 on + [12, 83]′ (observed ≤ 1e-4) and |Δξ_B|/σ ≤ 2e-2 (observed ≤ 1.0e-2, pure ξ−_B + at θ ≈ 2′, the estimator's own E→B leakage). """ from sp_validation import b_modes + config = bd.BlindingConfig(theory=TheoryConfig(transfer_function="eisenstein_hu")) + fid = config.theory + hidden = dataclasses.replace( + fid, + S8=fid.S8 + np.sign(ds8) * config.envelope["S8"], + Omega_m=fid.Omega_m + np.sign(dom) * config.envelope["Omega_m"], + ) grids = { "integration": b_modes.log_bin_edges(0.08, 300.0, 1000), "reporting": b_modes.log_bin_edges(1.0, 250.0, 20), @@ -384,7 +395,9 @@ def test_shift_leaks_no_b_modes(blinds): for grid, (left, right) in grids.items(): theta[grid] = np.sqrt(left * right) sio.add_xi(s, (0, 0), theta[grid], *_xi_template(theta[grid]), grid=grid) - shift = np.asarray(sio.seal(s, blinds["TOY"]).mean) - np.asarray(s.mean) + shift = np.zeros(len(s.mean)) + for block, factor in bd.factors(s, fid, hidden): + shift[block.rows] = factor def delta(grid): return tuple( @@ -419,7 +432,7 @@ def delta(grid): for key in ("xip_B", "xim_B"): finite = np.isfinite(modes[key]) assert finite.sum() > 10 - assert np.max(np.abs(modes[key][finite]) / sigma[finite]) <= 1e-2, key + assert np.max(np.abs(modes[key][finite]) / sigma[finite]) <= 2e-2, key # --------------------------------------------------------------------------- # diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py index cd81b083..e8598c08 100644 --- a/src/sp_validation/tests/test_custody_e2e.py +++ b/src/sp_validation/tests/test_custody_e2e.py @@ -31,7 +31,7 @@ GRIDS = { "reporting": {"min_sep": 5.0, "max_sep": 60.0, "nbins": 6, "npatch": 1}, - "integration": {"min_sep": 1.0, "max_sep": 150.0, "nbins": 120, "npatch": 1}, + "integration": {"min_sep": 1.0, "max_sep": 150.0, "nbins": 300, "npatch": 1}, } SCALE_CUT = [12.0, 60.0] PARTS = ("xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau") @@ -112,13 +112,23 @@ def _rho_tau_handlers(): def _covariances(root): - """CosmoCov-format ξ± covariances on both grids, and a NaMaster Cℓ FITS.""" + """ξ± shape-noise covariances on both grids, and a NaMaster Cℓ FITS. + + The ξ± covariances are the diagonal shape noise of the synthetic catalogue + (its density and per-component dispersion), in CosmoCov's text format. + """ from astropy.io import fits + from sp_validation.b_modes import log_bin_edges + + density, sigma_e, area = 20000 / (4.0 * 60) ** 2, 0.05, (4.0 * 60) ** 2 paths = {} for grid, b in GRIDS.items(): + left, right = log_bin_edges(b["min_sep"], b["max_sep"], b["nbins"]) + pairs = np.pi * area * density**2 * (right**2 - left**2) / 2.0 + var = 2.0 * sigma_e**4 / pairs paths[grid] = root / f"cov_{grid}.txt" - np.savetxt(paths[grid], np.diag(np.full(2 * b["nbins"], 1e-10))) + np.savetxt(paths[grid], np.diag(np.concatenate([var, var]))) nbp = len(_namaster_part_inputs()[0]) paths["pseudo_cl"] = root / "pseudo_cl_cov.fits" fits.HDUList( @@ -276,6 +286,8 @@ def toy(tmp_path, monkeypatch): json.dumps({"theory": {"transfer_function": "eisenstein_hu"}}) ) cat_config = Path(params["catalog_config"]) + # A fixed seed, so every run conceals under the same hidden point. + monkeypatch.setattr(bd.secrets, "token_hex", lambda n: "e2e-seed") bd.main( [ "init", @@ -416,6 +428,7 @@ def second_block_fails(*args, **kwargs): assert all(s == true.stamp for s in stamps(out).values()) report = bd.audit("toy", archive=archive, true_root=out, cat_config=toy.cat_config) + print(json.dumps(report, indent=1, default=str)) assert report["ok"], json.dumps(report, indent=1, default=str) assert set(report["parts"]) == set(born) assert ( From 19ddb2cb15bb6460ca98d1b5c8e8c6d60befad01 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 11:31:00 +0200 Subject: [PATCH 082/160] Blinding: a blind opens from its record as stored; one registry reader; the audit's value checks tested MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live blind survives a TheoryConfig schema change. The config digest is the sha256 of the record's stored config, so it no longer depends on the code's dataclasses. The config is rebuilt field for field. A field the record names that the code lacks, or a code field the record lacks with no value in blinding.NEUTRAL, is refused by name; it is never reported as an edited record. A blind committed under tests/data opens under this code and draws the point it was drawn at, so a new field without a neutral value turns CI red before it strands a real blind. custody.records is the one reader of the registry. _read_record and the inline revealed.json reads are gone. audit opens the blind through _open and requires the published seed to be the decrypted one. verify compares the stamp with the declared custody's stamp. The audit's three value checks now have tests that reach them: a mixed file (one ξ± pair left true or shifted twice), an unshifted CL_BB or ρ row that moved, and a derived B row moved by 1σ. Derived statistics are classified by sacc_io.is_derived, the complement rule seal uses, and unshifted rows are compared at the re-measurement tolerance. B_SIGMA is the one derived-B bound, imported by I4. I4 now measures the 20 COSEBIs modes the production part carries: at the envelope's corners they move by up to 5.5e-3σ on [12, 83]′, not the ~1e-4σ that 5 modes give. The bounds are 10× the production measurement: 5e-2σ for COSEBIs B, 1e-1σ for pure ξ_B. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/blinding.py | 267 +++++++++--------- src/sp_validation/sacc_io.py | 16 +- .../tests/data/blinds/committed/bases | 1 + .../data/blinds/committed/commitment.json | 35 +++ .../tests/data/blinds/committed/key | 1 + .../tests/data/blinds/committed/seed.fernet | 1 + src/sp_validation/tests/test_blinding.py | 192 ++++++++++--- 7 files changed, 351 insertions(+), 162 deletions(-) create mode 100644 src/sp_validation/tests/data/blinds/committed/bases create mode 100644 src/sp_validation/tests/data/blinds/committed/commitment.json create mode 100644 src/sp_validation/tests/data/blinds/committed/key create mode 100644 src/sp_validation/tests/data/blinds/committed/seed.fernet diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 16e4bae9..925270a4 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -20,7 +20,8 @@ from reading it by accident. ``share`` adds a catalogue to a blind; ``reveal`` publishes the seed and moves the concealed files aside; ``audit`` proves blinded − true = shift(seed) once the true files are - re-measured; ``verify`` checks a file's stamp against the record, seedless. + re-measured; ``verify`` checks a file's stamp against its catalogue's + custody, seedless. """ import argparse @@ -67,7 +68,7 @@ def record(self): @classmethod def from_record(cls, record): - """The config a record describes; fields it omits take their defaults.""" + """The config a partial record names; fields it omits take their defaults.""" fields = {} if "envelope" in record: fields["envelope"] = {k: float(v) for k, v in record["envelope"].items()} @@ -77,8 +78,13 @@ def from_record(cls, record): def digest(self): """sha256 of the canonical record; int and float literals agree.""" - text = json.dumps(self.record(), sort_keys=True, separators=(",", ":")) - return hashlib.sha256(text.encode("utf-8")).hexdigest() + return record_digest(self.record()) + + +def record_digest(record): + """sha256 of a config record's canonical JSON, whatever the code's schema.""" + text = json.dumps(record, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(text.encode("utf-8")).hexdigest() def _canonical(value): @@ -96,6 +102,38 @@ def _canonical(value): raise TypeError(f"cannot serialise {value!r} into a blind's record") +# A blind's record opens under this code only if it names every TheoryConfig +# field, or omits one listed here with the value under which that field changes +# no shift: a record drawn without it then reproduces its shift. +NEUTRAL = {} + + +def _recorded_config(name, record): + """The config blind ``name`` was drawn under, from its stored record. + + Refuses, naming the fields, a record whose shift this code cannot + reproduce: one naming a field the code lacks, or lacking a code field that + has no value in :data:`NEUTRAL`. + """ + fields = {f.name for f in dataclasses.fields(TheoryConfig)} + theory = {**NEUTRAL, **record.get("theory", {})} + envelope = record.get("envelope", {}) + foreign = sorted(set(record) - {"envelope", "theory"}) + foreign += sorted((set(theory) | set(envelope)) - fields) + unset = sorted(fields - set(theory)) + ["envelope"] * ("envelope" not in record) + if foreign or unset: + raise _custody.CustodyError( + f"blind {name} was drawn under a config this code cannot reproduce: " + f"the record names {foreign}, which this code lacks, and lacks " + f"{unset}, which have no value in blinding.NEUTRAL. Open it with the " + "code it was drawn under, or give each new field its neutral value." + ) + return BlindingConfig( + envelope={k: float(v) for k, v in envelope.items()}, + theory=TheoryConfig(**theory), + ) + + def draw_scheme(): """The installed fork's shift-draw semantics (``smokescreen.DRAW_SCHEME``).""" from smokescreen import DRAW_SCHEME @@ -133,54 +171,47 @@ class Blind: hidden: TheoryConfig -def _read_record(registry, name): - record = Path(registry) / name - try: - commitment = json.loads((record / "commitment.json").read_text()) - except (OSError, ValueError) as err: - raise _custody.CustodyError( - f"blind {name}: no readable record: {err}" - ) from None - return record, commitment - - @functools.cache def _open(registry, name): from cryptography.fernet import Fernet, InvalidToken - record, commitment = _read_record(registry, name) + record = _custody.records(registry).get(name) + if record is None: + raise _custody.CustodyError(f"no blind {name} in {registry}") + c = record.commitment try: - key = (record / "key").read_bytes() - payload = json.loads(Fernet(key).decrypt((record / "seed.fernet").read_bytes())) + key = (registry / name / "key").read_bytes() + ciphertext = (registry / name / "seed.fernet").read_bytes() + payload = json.loads(Fernet(key).decrypt(ciphertext)) except (OSError, ValueError, InvalidToken) as err: raise _custody.CustodyError( - f"blind {name}: the seed does not decrypt with its key ({type(err).__name__})" + f"blind {name}: the seed does not decrypt with its key " + f"({type(err).__name__})" ) from None - config = BlindingConfig.from_record(commitment["config"]) - checks = { - "blind name": (payload["blind"], commitment["blind"], name), + for what, values in { + "blind name": (payload["blind"], c["blind"], name), "config digest": ( payload["config_digest"], - commitment["config_digest"], - config.digest(), - ), - "draw scheme": ( - payload["draw_scheme"], - commitment["draw_scheme"], - draw_scheme(), + c["config_digest"], + record_digest(c["config"]), ), + "draw scheme": (payload["draw_scheme"], c["draw_scheme"]), "seed commitment": ( _custody.seed_commitment(payload["seed"]), - commitment["seed_commitment"], + c["seed_commitment"], ), - } - for what, values in checks.items(): + }.items(): if len(set(values)) != 1: raise _custody.CustodyError( - f"blind {name}: the {what} disagrees between the sealed seed, the " - "record and this install; the record was edited or the install " - "draws differently" + f"blind {name}: the {what} disagrees between the sealed seed and " + "the record; the record was edited" ) + if c["draw_scheme"] != draw_scheme(): + raise _custody.CustodyError( + f"blind {name} was drawn under draw scheme {c['draw_scheme']}; this " + f"install's smokescreen draws under {draw_scheme()}" + ) + config = _recorded_config(name, c["config"]) seed = payload["seed"] return Blind(name, seed, config, hidden_theory(seed, config)) @@ -355,10 +386,8 @@ def _declared_blinded(catalogues, registry, base): def init(name, bases, *, cat_config, config=None): """Draw blind ``name`` for ``bases`` and write its record; return its directory. - @sc blind-drawn-once The one place a seed is drawn. Existing state is refused, never replaced, - so no second draw exists for a catalogue unless a committed record is - deleted. The seed is held in memory and written only as ciphertext. + and the seed is held in memory and written only as ciphertext. """ from cryptography.fernet import Fernet @@ -449,19 +478,18 @@ def reveal(name, *, root, cat_config): mixes shifted and true signal. """ registry = _custody.registry_of(cat_config) - record, commitment = _read_record(registry, name) blind = _open(registry, name) - revealed = record / "revealed.json" - if revealed.exists(): - if json.loads(revealed.read_text())["seed"] != blind.seed: - raise _custody.CustodyError(f"{revealed} records another seed") - else: + record = _custody.records(registry)[name] + revealed = registry / name / "revealed.json" + if record.revealed is None: fd = os.open(revealed, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444) with os.fdopen(fd, "w") as f: json.dump({"seed": blind.seed}, f) + elif record.revealed != blind.seed: + raise _custody.CustodyError(f"{revealed} records another seed") archive = Path(root) / "revealed" / name moved = 0 - for path in _parts_under(root, commitment["seed_commitment"], archive): + for path in _parts_under(root, record.commitment["seed_commitment"], archive): target = archive / path.relative_to(root) target.parent.mkdir(parents=True, exist_ok=True) shutil.move(path, target) @@ -469,7 +497,7 @@ def reveal(name, *, root, cat_config): print(f"[blinding] published the seed of {name}; moved {moved} files to {archive}") print( f"[blinding] commit {revealed} and declare `blinding: unblinded` on " - f"{', '.join(_custody.records(registry)[name].bases)}, then re-run" + f"{', '.join(record.bases)}, then re-run" ) return archive @@ -477,13 +505,13 @@ def reveal(name, *, root, cat_config): # A re-measurement repeats a measurement to float noise (TreeCorr's threaded # sums reorder); the audit compares its numbers to this relative tolerance. _REMEASURED = 1e-10 -# How far, in σ, a derived B-mode may move under a blind. COSEBIs B-modes move -# by ~1e-4σ. Pure-mode ξ_B carries the estimator's own E→B leakage of the -# shift: up to 1e-2σ on the production grids at the envelope's edge -# (test_blinding), several times more on coarse grids. A derivation from mixed -# inputs moves B by ~1σ. -_B_SIGMA = { - sacc_io.COSEBI_BB: 1e-2, +# How far, in σ, a derived B-mode may move under a blind: 10× what a shift at +# the envelope's corners induces on the production grids, with shape-noise σ at +# UNIONS depth. There COSEBIs B (20 modes, [12, 83]′) move by ≤ 6e-3σ, and +# pure-mode ξ_B by ≤ 1e-2σ (ξ−_B at θ ≈ 2′: the estimator's own E→B leakage of +# the shift). A derivation from mixed inputs moves B by ~1σ. +B_SIGMA = { + sacc_io.COSEBI_BB: 5e-2, sacc_io.PURE_TYPES["xip_B"]: 1e-1, sacc_io.PURE_TYPES["xim_B"]: 1e-1, } @@ -525,45 +553,41 @@ def _audit_part(blinded, true, fiducial, hidden, tolerance): stamp, true_stamp = (_custody.read_stamp(x.metadata) for x in (blinded, true)) if true_stamp.token != f"unblinded:{stamp.catalogue}": return [f"the live file is stamped {true_stamp.token}"], None - problems = [] if not _rows_match(blinded, true): - problems.append("rows, tags or tracers differ") - return problems, None + return ["rows, tags or tracers differ"], None + problems = [] if set(blinded.tracers) != set(true.tracers): problems.append("tracers differ") - cov_b, cov_t = _covariance(blinded), _covariance(true) - if not _same_covariance(cov_b, cov_t): + cov = _covariance(true) + if not _same_covariance(_covariance(blinded), cov): problems.append("covariances differ") centres = [x.metadata.get("patch_centers_sha256") for x in (blinded, true)] if centres[0] != centres[1]: problems.append(f"patch centres differ: {centres}") - delta = np.asarray(blinded.mean) - np.asarray(true.mean) - residual, shifted = 0.0, np.zeros(len(delta), bool) + values = np.asarray(true.mean) + delta = np.asarray(blinded.mean) - values + residual, rest = 0.0, np.ones(len(delta), bool) for block, factor in factors(true, fiducial, hidden): scale = np.max(np.abs(factor)) residual = max(residual, np.max(np.abs(delta[block.rows] - factor)) / scale) - shifted[block.rows] = True + rest[block.rows] = False if residual > tolerance: problems.append(f"blinded − true ≠ shift(seed): residual {residual:.2e}") - sigma = None if cov_t is None else np.sqrt(np.diag(cov_t)) - derived = (sacc_io.COSEBI_EE, sacc_io.COSEBI_BB, *sacc_io.PURE_TYPES.values()) + kinds = np.array([dp.data_type for dp in true.data]) moved = {} - for i, dp in enumerate(true.data): - kind = dp.data_type - if shifted[i]: - continue - if kind in derived: - if sigma is None: - problems.append(f"row {i} ({kind}) has no σ to judge its shift by") - elif np.isfinite(delta[i]): - moved[kind] = max(moved.get(kind, 0.0), abs(delta[i]) / sigma[i]) - elif abs(delta[i]) > 1e-8 * max(abs(dp.value), 1e-300): - problems.append( - f"row {i} ({kind}) moved, but the blind leaves it unshifted" - ) - for kind, bound in _B_SIGMA.items(): + for kind in dict.fromkeys(kinds[rest]): + rows = rest & (kinds == kind) + if not sacc_io.is_derived(kind): + if np.max(np.abs(delta[rows])) > _REMEASURED * np.max(np.abs(values[rows])): + problems.append(f"{kind} moved, but the blind leaves it unshifted") + elif cov is None: + problems.append(f"{kind} has no σ to judge its shift by") + else: + ratio = np.abs(delta[rows]) / np.sqrt(np.diag(cov)[rows]) + moved[kind] = float(np.max(ratio[np.isfinite(ratio)], initial=0.0)) + for kind, bound in B_SIGMA.items(): if moved.get(kind, 0.0) > bound: problems.append(f"{kind} moved by {moved[kind]:.2e}σ under the blind") return problems, {"residual": residual, "shift_over_sigma": moved} @@ -572,42 +596,39 @@ def _audit_part(blinded, true, fiducial, hidden, tolerance): def audit(name, *, archive, true_root, cat_config, out=None): """Check every archived file of blind ``name`` against its re-measured twin. - Needs no key: the seed is the published one, checked against the - commitment. For each archived file, the live file at the same relative path - must be stamped unblinded for the same catalogue, with the same rows, tags, - tracers, covariance and patch centres (numbers to a re-measurement's float - noise), and blinded − true must equal the seed's shift on every ξ± and - Cℓ_EE block to 1e-6 of the block's largest shift (1e-3 when the theory - stack differs from the one the blind was drawn with); COSEBIs and pure-E/B - B rows may move by at most ``_B_SIGMA``, and rows the blind leaves - unshifted not at all. Each part reports the largest shift of each derived - statistic, in σ. + The published seed must be the committed one. For each archived file, the + live file at the same relative path must be stamped unblinded for the same + catalogue, with the same rows, tags, tracers, covariance and patch centres + (numbers to a re-measurement's float noise), and blinded − true must equal + the seed's shift on every ξ± and Cℓ_EE block to 1e-6 of the block's + largest shift (1e-3 when the theory stack differs from the one the blind + was drawn with); a derived statistic's B rows may move by at most + :data:`B_SIGMA`, and rows the blind leaves unshifted not at all. Each part + reports the largest shift of each derived statistic, in σ. """ registry = _custody.registry_of(cat_config) - record, commitment = _read_record(registry, name) report = {"blind": name, "ok": False, "problems": [], "parts": {}} - revealed = record / "revealed.json" - seed = json.loads(revealed.read_text())["seed"] if revealed.exists() else None - if seed is None or _custody.seed_commitment(seed) != commitment["seed_commitment"]: - report["problems"].append("the published seed does not match the commitment") + try: + blind = _open(registry, name) + except _custody.CustodyError as err: + report["problems"].append(str(err)) + return _write_report(report, out) + record = _custody.records(registry)[name] + if record.revealed != blind.seed: + report["problems"].append("the published seed is not the committed one") return _write_report(report, out) - config = BlindingConfig.from_record(commitment["config"]) - if config.digest() != commitment["config_digest"]: - report["problems"].append("the recorded config does not match its digest") - if int(commitment["draw_scheme"]) != draw_scheme(): - report["problems"].append("this install draws under another scheme") stack = _theory_stack() tolerance = 1e-6 - if stack != commitment.get("theory_stack"): + if stack != record.commitment.get("theory_stack"): tolerance = 1e-3 report["theory_stack"] = { - "record": commitment.get("theory_stack"), + "record": record.commitment.get("theory_stack"), "now": stack, } - hidden = hidden_theory(seed, config) + fiducial, hidden = blind.config.theory, blind.hidden report["shift"] = { - key: getattr(hidden, key) - getattr(config.theory, key) - for key in config.envelope + key: getattr(hidden, key) - getattr(fiducial, key) + for key in blind.config.envelope } archive, true_root = Path(archive), Path(true_root) for path in sorted(archive.rglob("*.sacc")): @@ -618,11 +639,11 @@ def audit(name, *, archive, true_root, cat_config, out=None): continue blinded = sacc_io.load(path) stamp = _custody.read_stamp(blinded.metadata) - if stamp.commitment != commitment["seed_commitment"]: + if stamp.commitment != record.commitment["seed_commitment"]: report["parts"][relative] = {"problems": ["not concealed under this blind"]} continue problems, numbers = _audit_part( - blinded, sacc_io.load(live), config.theory, hidden, tolerance + blinded, sacc_io.load(live), fiducial, hidden, tolerance ) report["parts"][relative] = {"problems": problems, **(numbers or {})} report["ok"] = ( @@ -640,34 +661,26 @@ def _write_report(report, out): def verify(path, *, cat_config): - """Problems with a file's stamp against the registry and declaration (seedless).""" + """Problems with a file's stamp against its catalogue's custody (seedless).""" stamp = _custody.read_stamp(sacc_io.load(path).metadata) - registry = _custody.registry_of(cat_config) - problems = [] - if stamp.status == "blinded": - record = _custody.records(registry).get(stamp.blind) - if record is None: - return [f"no blind {stamp.blind} in {registry}"] - c = record.commitment - for what, stamped, recorded in ( - ("seed commitment", stamp.commitment, c["seed_commitment"]), - ("config digest", stamp.config_digest, c["config_digest"]), - ("draw scheme", stamp.draw_scheme, int(c["draw_scheme"])), - ): - if stamped != recorded: - problems.append(f"the {what} differs from the record") - if stamp.draw_scheme != draw_scheme(): - problems.append("this install draws under another scheme") - if stamp.catalogue not in record.bases: - problems.append(f"blind {record.name} does not cover {stamp.catalogue}") try: declared = _custody.custody_of( - _catalogues(cat_config), stamp.catalogue, registry=registry + _catalogues(cat_config), + stamp.catalogue, + registry=_custody.registry_of(cat_config), ) - if declared.stamp != stamp.stamp: - problems.append(f"{stamp.catalogue} is declared {declared.token}") except _custody.CustodyError as err: - problems.append(str(err)) + return [str(err)] + problems = [] + keys = {**stamp.stamp, **declared.stamp} + differ = sorted(k for k in keys if stamp.stamp.get(k) != declared.stamp.get(k)) + if differ: + problems.append( + f"the stamp's {differ} differ from {stamp.catalogue}'s custody, " + f"{declared.token}" + ) + if stamp.status == "blinded" and stamp.draw_scheme != draw_scheme(): + problems.append("this install draws under another scheme") return problems diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index de545f33..bc8ce30d 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -954,6 +954,17 @@ def update_statistic(s, sub): UNSHIFTED = (CL_BB, CL_EB) +def is_derived(data_type): + """Whether ``data_type`` is a derived statistic (COSEBIs, pure-E/B, …). + + Signal the blind neither conceals nor leaves unchanged: it moves only + through the shiftable rows it is computed from. + """ + return data_type.startswith(SIGNAL_PREFIX) and data_type not in ( + SHIFTABLE + UNSHIFTED + ) + + def _stamped(s): return any(key in s.metadata for key in _custody.STAMP_KEYS) @@ -981,14 +992,13 @@ def seal(s, custody): "only as a derivation (save(..., derived_from=[...]))" ) types = {dp.data_type for dp in s.data} - signal = {t for t in types if t.startswith(SIGNAL_PREFIX)} - derived = signal - set(SHIFTABLE) - set(UNSHIFTED) + derived = {t for t in types if is_derived(t)} if custody.status == "blinded" and derived: raise ValueError( f"a blinded catalogue's {sorted(derived)} rows are derived " "statistics: save them with derived_from=[their input parts]" ) - if custody.status == "blinded" and signal & set(SHIFTABLE): + if custody.status == "blinded" and types & set(SHIFTABLE): from . import blinding out = blinding.conceal(s, blinding.open_blind(custody)) diff --git a/src/sp_validation/tests/data/blinds/committed/bases b/src/sp_validation/tests/data/blinds/committed/bases new file mode 100644 index 00000000..e20be6bb --- /dev/null +++ b/src/sp_validation/tests/data/blinds/committed/bases @@ -0,0 +1 @@ +COMMITTED diff --git a/src/sp_validation/tests/data/blinds/committed/commitment.json b/src/sp_validation/tests/data/blinds/committed/commitment.json new file mode 100644 index 00000000..35b1f7ad --- /dev/null +++ b/src/sp_validation/tests/data/blinds/committed/commitment.json @@ -0,0 +1,35 @@ +{ + "blind": "committed", + "seed_commitment": "810515ec5aa95a3a2445c949f8084f1280e25a21579ed7071562682fd87b8068", + "config": { + "envelope": { + "S8": 0.075, + "Omega_m": 0.1 + }, + "theory": { + "S8": 0.8, + "Omega_m": 0.3, + "Omega_b": 0.0469, + "h": 0.7, + "n_s": 0.96, + "m_nu": 0.06, + "w0": -1.0, + "wa": 0.0, + "mass_split": "normal", + "transfer_function": "boltzmann_camb", + "halofit_version": "mead2020_feedback", + "hmcode_logT_AGN": 7.5, + "ia_bias": 0.0, + "ia_z_piv": 0.62, + "ia_alphaz": 0.0 + } + }, + "config_digest": "9c0de8766e8947da70d4bf0a936436acae3bdb4c6b5ee304cd3373d1a7a214de", + "draw_scheme": 2, + "theory_stack": { + "pyccl": "3.3.4", + "camb": "1.6.6", + "smokescreen": "1.5.6" + }, + "created": "2026-09-26T09:16:50+00:00" +} diff --git a/src/sp_validation/tests/data/blinds/committed/key b/src/sp_validation/tests/data/blinds/committed/key new file mode 100644 index 00000000..43fc5df6 --- /dev/null +++ b/src/sp_validation/tests/data/blinds/committed/key @@ -0,0 +1 @@ +C7iIarmGreIIF8_Xs2kxQ9ofySIhSQn2UfYUI30KvII= \ No newline at end of file diff --git a/src/sp_validation/tests/data/blinds/committed/seed.fernet b/src/sp_validation/tests/data/blinds/committed/seed.fernet new file mode 100644 index 00000000..81f5403b --- /dev/null +++ b/src/sp_validation/tests/data/blinds/committed/seed.fernet @@ -0,0 +1 @@ +gAAAAABqt42C0vXv4_tLg74hQ_3vQRitvL_ZzRGOZ_SxTJBbJk7PfIH10iKeNTocfyIblDU162039lrm-m_Tcv0vLvOpjhRFMq7lyc-OnAM1S4BaeRjv7M84r3DZJqvpO_Cwk85CjLlqxtXqjOormHMfGX7tZS93eH00yKgLZ-8S5pGjre9dJ8hN5AlQHAiyAmB_bDzaMQVRBR0DCK-G8WYSizlg-QZk7pgKNfxVyWy6OZk8K9xn_dxAXkSPQK_ROyJgZtDr3zf65rx-Xt3O-3z9kpAXzPSXEdUeYEotmqOxXYKiSanACAk= \ No newline at end of file diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index d1b16242..0f1b71e3 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -11,6 +11,7 @@ import json import os import stat +from pathlib import Path import numpy as np import pytest @@ -22,6 +23,9 @@ from sp_validation.blinding_theory import TheoryConfig FAST = {"theory": {"transfer_function": "eisenstein_hu"}} +DATA = Path(__file__).parent / "data" +# The hidden (S8, Ωm) of the blind committed under DATA/blinds/committed. +COMMITTED_POINT = (0.8374952413635888, 0.32730573347638175) # --------------------------------------------------------------------------- # @@ -369,13 +373,12 @@ def _shape_noise_variance(left, right): @pytest.mark.parametrize("ds8", [0.075, -0.075]) @pytest.mark.parametrize("dom", [0.1, -0.1]) def test_shift_leaks_no_b_modes(ds8, dom): - """B-modes a shift at the envelope's edge induces are numerical, and small. + """B-modes a shift at the envelope's edge induces stay within the audit's bound. At each corner of the (S8, Ωm) envelope, on the production grids (the 0.08–300′ 1000-bin integration grid, the 1–250′ 20-bin reporting grid), - single-bin n(z), σ from shape noise at UNIONS depth: |ΔBₙ|/σ(Bₙ) ≤ 1e-2 on - [12, 83]′ (observed ≤ 1e-4) and |Δξ_B|/σ ≤ 2e-2 (observed ≤ 1.0e-2, pure ξ−_B - at θ ≈ 2′, the estimator's own E→B leakage). + single-bin n(z), σ from shape noise at UNIONS depth: the 20 COSEBIs B-modes + on [12, 83]′ and pure-mode ξ_B move by at most ``blinding.B_SIGMA``. """ from sp_validation import b_modes @@ -412,12 +415,12 @@ def delta(grid): np.diag(np.concatenate([var, var])), left, right, - nmodes=5, + nmodes=20, scale_cuts=[(12.0, 83.0)], ).values() - sigma_b = np.sqrt(np.diag(result["cov"])[5:]) + sigma_b = np.sqrt(np.diag(result["cov"])[20:]) assert np.max(np.abs(result["En"])) > 1e2 * np.max(np.abs(result["Bn"])) - assert np.max(np.abs(result["Bn"]) / sigma_b) <= 1e-2 + assert np.max(np.abs(result["Bn"]) / sigma_b) <= bd.B_SIGMA[sio.COSEBI_BB] rep_left, rep_right = grids["reporting"] modes = b_modes.pure_eb_from_xi( @@ -432,7 +435,8 @@ def delta(grid): for key in ("xip_B", "xim_B"): finite = np.isfinite(modes[key]) assert finite.sum() > 10 - assert np.max(np.abs(modes[key][finite]) / sigma[finite]) <= 2e-2, key + bound = bd.B_SIGMA[sio.PURE_TYPES[key]] + assert np.max(np.abs(modes[key][finite]) / sigma[finite]) <= bound, key # --------------------------------------------------------------------------- # @@ -730,7 +734,7 @@ def test_open_blind_refuses_a_tampered_record(fresh, tamper): def edit(r): r["config"]["envelope"]["S8"] = 0.3 - r["config_digest"] = bd.BlindingConfig.from_record(r["config"]).digest() + r["config_digest"] = bd.record_digest(r["config"]) _edit(commitment, edit) elif tamper == "scheme": @@ -746,28 +750,94 @@ def edit(r): bd.open_blind(_custody(fresh, "TOY")) +def _init_storing(root, edit): + """Draw blind toy for TOY (seed ``toy-seed``), its config stored as ``edit`` + makes it: the record a code with another TheoryConfig schema writes.""" + record = bd.BlindingConfig.record + with pytest.MonkeyPatch.context() as m: + m.setattr(bd.BlindingConfig, "record", lambda self: edit(record(self))) + m.setattr(bd.secrets, "token_hex", lambda n: "toy-seed") + _init(root, "toy", "TOY") + + +def test_a_record_lacking_a_field_opens_to_the_shift_it_was_drawn_with( + tmp_path, blinds, monkeypatch +): + """A record that predates a TheoryConfig field opens once the field's + neutral value is declared, and conceals exactly as the blind that names it; + until then it is refused by the field's name, never as an edited record.""" + _cat_config(tmp_path) + + def without_alphaz(record): + del record["theory"]["ia_alphaz"] + return record + + _init_storing(tmp_path, without_alphaz) + custody = _custody(tmp_path, "TOY") + with pytest.raises(cu.CustodyError, match="ia_alphaz") as refused: + bd.open_blind(custody) + assert "edited" not in str(refused.value) + + monkeypatch.setattr(bd, "NEUTRAL", {"ia_alphaz": 0.0}) + s = two_bin_sacc(cl=False) + assert np.array_equal( + np.asarray(sio.seal(s, custody).mean), + np.asarray(sio.seal(s, blinds["TOY"]).mean), + ) + + +def test_a_record_naming_a_field_this_code_lacks_is_refused_by_name(tmp_path): + _cat_config(tmp_path) + _init_storing( + tmp_path, lambda r: {**r, "theory": {**r["theory"], "baryon_boost": 0.0}} + ) + with pytest.raises(cu.CustodyError, match="baryon_boost") as refused: + bd.open_blind(_custody(tmp_path, "TOY")) + assert "edited" not in str(refused.value) + + +def test_the_committed_blind_opens_under_this_code(): + """``tests/data/blinds/committed`` opens to the point it was drawn at. + + A TheoryConfig field added without its value in ``blinding.NEUTRAL`` would + strand every live blind; this record turns that red. + """ + blind = bd._open(DATA / "blinds", "committed") + assert blind.config == bd.BlindingConfig() + assert (blind.hidden.S8, blind.hidden.Omega_m) == pytest.approx( + COMMITTED_POINT, rel=1e-12 + ) + + # --------------------------------------------------------------------------- # # I11: the audit proves blinded − true = shift(seed) # --------------------------------------------------------------------------- # -def _reveal_pair(root, *, patch_centers=("a", "a")): - """A concealed part in an archive and its true twin at the same path.""" +def _concealed(root, s): + """``s`` sealed under blind toy, before its seed is published.""" + return sio.seal(s, _custody(root, "TOY")) + + +def _write(path, s, stamp): + """``s`` on disk under ``stamp`` as given, as the door would never write it.""" + s = s.copy() + s.metadata.update(stamp) + path.parent.mkdir(parents=True, exist_ok=True) + s.save_fits(str(path), overwrite=True) + + +def _audit(root, archived, live, *, centres=("a", "a"), seed=None): + """Publish toy's seed (or ``seed``), archive ``archived`` under toy's stamp + with ``live`` as its re-measured twin, and audit the pair.""" blinded = _custody(root, "TOY") - seed = bd.open_blind(blinded).seed + seed = seed or bd.open_blind(blinded).seed (root / "blinds" / "toy" / "revealed.json").write_text(json.dumps({"seed": seed})) - true = cu.Custody("unblinded", "TOY") - s = two_bin_sacc() - for tree, custody, centers in ( - ("archive", blinded, patch_centers[0]), - ("live", true, patch_centers[1]), + for tree, s, stamp, centre in ( + ("archive", archived, blinded.stamp, centres[0]), + ("live", live, cu.Custody("unblinded", "TOY").stamp, centres[1]), ): - part = s.copy() - part.metadata["patch_centers_sha256"] = centers - (root / tree / "sub").mkdir(parents=True) - sio.save(part, root / tree / "sub" / "part.sacc", custody=custody) - return seed - - -def _audit(root): + s = s.copy() + s.metadata["patch_centers_sha256"] = centre + _write(root / tree / "sub" / "part.sacc", s, stamp) return bd.audit( "toy", archive=root / "archive", @@ -776,9 +846,15 @@ def _audit(root): ) +def _problems(report): + return report["problems"] + [ + problem for part in report["parts"].values() for problem in part["problems"] + ] + + def test_the_audit_passes_on_a_true_reveal(fresh): - _reveal_pair(fresh) - report = _audit(fresh) + s = two_bin_sacc(rho=True) + report = _audit(fresh, _concealed(fresh, s), s) assert report["ok"], report ((path, part),) = report["parts"].items() assert path == "sub/part.sacc" and part["residual"] <= 1e-6 @@ -788,11 +864,63 @@ def test_the_audit_passes_on_a_true_reveal(fresh): def test_the_audit_fails_on_a_wrong_seed(fresh): - _reveal_pair(fresh) - (fresh / "blinds" / "toy" / "revealed.json").write_text(json.dumps({"seed": "x"})) - assert not _audit(fresh)["ok"] + s = two_bin_sacc() + report = _audit(fresh, _concealed(fresh, s), s, seed="not-the-seed") + assert not report["ok"] and "published seed" in _problems(report)[0] def test_the_audit_fails_on_other_patch_centres(fresh): - _reveal_pair(fresh, patch_centers=("a", "b")) - assert not _audit(fresh)["ok"] + s = two_bin_sacc() + report = _audit(fresh, _concealed(fresh, s), s, centres=("a", "b")) + assert not report["ok"] and "patch centres" in _problems(report)[0] + + +@pytest.mark.parametrize("times", [0, 2], ids=["left_true", "shifted_twice"]) +def test_the_audit_fails_on_a_mixed_file(fresh, times): + """One ξ± pair under the blinded stamp carries 0× or 2× the shift.""" + s = two_bin_sacc(rho=True) + archived = _concealed(fresh, s) + for i in _rows(s, sio.XI_PLUS, sio.XI_MINUS): + if s.data[i].tracers == sio._pair((1, 1)): + shift = archived.data[i].value - s.data[i].value + archived.data[i].value = s.data[i].value + times * shift + report = _audit(fresh, archived, s) + assert not report["ok"] + assert any("≠ shift(seed)" in problem for problem in _problems(report)) + + +@pytest.mark.parametrize("kind", [sio.CL_BB, sio.RHO_PLUS.format(k=0)]) +def test_the_audit_fails_when_an_unshifted_row_moved(fresh, kind): + s = two_bin_sacc(rho=True) + archived = _concealed(fresh, s) + i = _rows(s, kind)[-1] + archived.data[i].value += 1e-6 * abs(s.data[i].value) + report = _audit(fresh, archived, s) + assert not report["ok"] + assert _problems(report) == [f"{kind} moved, but the blind leaves it unshifted"] + + +@pytest.mark.parametrize( + "e_kind, b_kind", + [ + (sio.COSEBI_EE, sio.COSEBI_BB), + (sio.PURE_TYPES["xim_E"], sio.PURE_TYPES["xim_B"]), + ], + ids=["cosebis", "pure_eb"], +) +@pytest.mark.parametrize("b_sigma", [0.0, 1.0]) +def test_the_audit_bounds_derived_b_modes(fresh, e_kind, b_kind, b_sigma): + """A derived part's E rows move with the blind; its B rows by ≤ ``B_SIGMA``.""" + true = cosebis_sacc() if e_kind == sio.COSEBI_EE else pure_eb_sacc() + true.add_covariance(np.full(len(true.mean), 0.01)) + archived = true.copy() + for i in _rows(true, e_kind): + archived.data[i].value += 0.3 + for i in _rows(true, b_kind): + archived.data[i].value += b_sigma * 0.1 + report = _audit(fresh, archived, true) + assert report["ok"] == (b_sigma == 0.0), report + (part,) = report["parts"].values() + assert part["shift_over_sigma"][e_kind] == pytest.approx(3.0) + if b_sigma: + assert _problems(report) == [f"{b_kind} moved by 1.00e+00σ under the blind"] From 344ec7ecad6a97c19457bcf562a0f4602b8f91cd Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 11:31:10 +0200 Subject: [PATCH 083/160] E2E: enough MC draws for a full-rank pure-E/B covariance; an interrupted seal inside run_2pcf leaves no part MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With 10 draws, the 12 × 12 combined ξ±_B covariance that calculate_eb_statistics solves had rank at most 9. np.linalg.solve raised "Singular matrix" in about 1 run in 5, depending on the float noise of the measured θ. 40 draws make it full rank. The interrupted-birth check makes blinding.conceal raise during run_2pcf. That is after the part is built and before it is written, so a writer that persisted the part before sealing it would fail the check. cv_pure_eb saves its part, and so runs the one-stamp check on its two inputs, before it draws a figure or writes the npz. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/tests/test_custody_e2e.py | 13 ++++++------- workflow/scripts/cv_pure_eb.py | 16 +++++++--------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py index e8598c08..bdb08f08 100644 --- a/src/sp_validation/tests/test_custody_e2e.py +++ b/src/sp_validation/tests/test_custody_e2e.py @@ -227,7 +227,8 @@ def run_chain(cat_config, version, out, cov, *, grid_parts=None): "min_sep": rep["min_sep"], "max_sep": rep["max_sep"], "nbins": rep["nbins"], - "n_samples": 10, + # Enough draws for a full-rank 12 × 12 combined ξ±_B covariance. + "n_samples": 40, "cosmo_params": {"transfer_function": "eisenstein_hu"}, "fiducial_scale_cut": SCALE_CUT, }, @@ -369,14 +370,12 @@ def test_a_blinded_catalogue_from_birth_to_audit(toy, monkeypatch): assert not (out / "laundered.sacc").exists() # --- an interrupted birth leaves no part --------------------------------- - from sp_validation.cosmo_val import sacc_writers - - def after_compute(*args, **kwargs): - raise RuntimeError("interrupted after the measurement") + def interrupted(*args, **kwargs): + raise RuntimeError("interrupted while sealing") with monkeypatch.context() as m: - m.setattr(sacc_writers, "xi_to_sacc", after_compute) - with pytest.raises(RuntimeError): + m.setattr(bd, "conceal", interrupted) + with pytest.raises(RuntimeError, match="while sealing"): run_chain( toy.cat_config, "TOY", diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index 71bffedd..7e16c2b4 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -59,6 +59,13 @@ n_samples=p["n_samples"], rng=np.random.default_rng(0), ) +# Written first, so parts under two custodies are refused before any product +# of them exists. +sacc_io.save( + pure_eb_to_sacc({0: (z, nz)}, reporting.metadata, theta, modes, covariance=cov), + snakemake.output["sacc"], + derived_from=[reporting, integration], +) variances = reporting.covariance.dense.diagonal() results = { @@ -102,13 +109,4 @@ save_pure_eb_results(results, snakemake.output["npz"]) -s = pure_eb_to_sacc( - {0: (z, nz)}, - reporting.metadata, - theta, - {key: results[key] for key in sacc_io.PURE_KEYS}, - covariance=cov, -) -sacc_io.save(s, snakemake.output["sacc"], derived_from=[reporting, integration]) - verify_outputs(snakemake) From a6bbe44058948e7c08a7fa722ca3dc45ec0ea6ba Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 11:31:21 +0200 Subject: [PATCH 084/160] =?UTF-8?q?Docs=20say=20what=20the=20code=20does;?= =?UTF-8?q?=20the=20extract=E2=86=92conceal=E2=86=92merge=20helper=20goes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - papers/bmodes pseudo_cl_io: sacc_io.load refuses a part without a custody stamp. It does not refuse unblinded parts. - sacc_io.update_statistic and its three tests are deleted. It was the documented merge-back half of the extract → conceal → merge flow that sealing at birth replaces, a value overwrite outside the door, and nothing called it. merge's docstring makes no custody claim. - custody.py states its rule once, in the @sc custody-is-declared text. A catalogue that no blind covers is told to git pull first, since its blind may already be committed upstream, and that only the custodian draws one. - blinding_theory no longer names the tests that exercise it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- papers/bmodes/scripts/pseudo_cl_io.py | 4 +-- src/sp_validation/blinding_theory.py | 7 ++--- src/sp_validation/custody.py | 30 +++++++++--------- src/sp_validation/sacc_io.py | 42 ++----------------------- src/sp_validation/tests/test_sacc_io.py | 42 +------------------------ 5 files changed, 23 insertions(+), 102 deletions(-) diff --git a/papers/bmodes/scripts/pseudo_cl_io.py b/papers/bmodes/scripts/pseudo_cl_io.py index b7b1c796..27c7df3a 100644 --- a/papers/bmodes/scripts/pseudo_cl_io.py +++ b/papers/bmodes/scripts/pseudo_cl_io.py @@ -7,8 +7,8 @@ def load_pseudo_cl_data(path): """Return a pseudo-Cℓ part's spectra as ``{"ELL", "EE", "EB", "BB"}`` arrays. The bandpower covariance is a separate FITS product (``pseudo_cl_cov``). - Loading goes through ``sacc_io.load``, so an unblinded real-data part is - refused. + Loading goes through ``sacc_io.load``, which refuses a part without a + custody stamp. """ ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(sacc_io.load(str(path)), (0, 0)) missing = [name for name, values in (("BB", bb), ("EB", eb)) if values is None] diff --git a/src/sp_validation/blinding_theory.py b/src/sp_validation/blinding_theory.py index bb46e4c8..360cd1a5 100644 --- a/src/sp_validation/blinding_theory.py +++ b/src/sp_validation/blinding_theory.py @@ -6,8 +6,7 @@ recipe; :func:`xi_ccl` and :func:`cl_ee` are the tomographic shear ξ± and Cℓ_EE between two bins' n(z). CCL builds the nonlinear P(k) through its Boltzmann-CAMB HMCode2020 route and projects with its own Limber - (``angular_cl``) and FFTLog (``correlation``). ``test_camb_ccl_crosscheck`` - compares this path with a direct CAMB run. + (``angular_cl``) and FFTLog (``correlation``). The generic cosmology machinery here is destined for ``cs_util.cosmo`` (cs_util#80). @@ -21,8 +20,8 @@ import numpy as np -# Fixed constants of the fiducial, passed explicitly to CCL (and to the CAMB -# oracle in the tests) rather than left to either stack's default. +# Fixed constants of the fiducial, passed explicitly to CCL rather than left to +# its default. NEFF = 3.046 T_CMB = 2.7255 diff --git a/src/sp_validation/custody.py b/src/sp_validation/custody.py index 4a80c4a0..8b5445ce 100644 --- a/src/sp_validation/custody.py +++ b/src/sp_validation/custody.py @@ -1,15 +1,10 @@ """Custody: whether a catalogue's signal is blinded, unblinded or a mock. -Custody is declared once per *base* catalogue, as ``blinding:`` on its entry in -``cosmo_val/cat_config.yaml``; an entry that declares nothing is blinded, so a -new catalogue fails closed. Variants share their base's custody and blind: -``_leak_corr`` and ``_seed`` versions, and entries that name their parent -with ``base:``. - -A blinded base must be covered by exactly one blind in the registry beside the +The custody of every catalogue version (:func:`custody_of`), the stamp a SACC +born under it carries, and the reader of the blind registry beside the catalogue config (``cosmo_val/blinds//``: ``commitment.json``, the -``bases`` it covers, and ``revealed.json`` once its seed is published). The -registry is written only by ``python -m sp_validation.blinding``. +``bases`` it covers, and ``revealed.json`` once its seed is published), which +only ``python -m sp_validation.blinding`` writes. Standard library only: the host Snakemake loads this file by path, and container jobs import it, so both resolve custody from the same declaration. @@ -180,12 +175,14 @@ def _no_blind(version, base, declared, registry): repo = registry.parent.parent cat_config = registry.parent / "cat_config.yaml" return ( - f"{version} is blinded ({why}) and no blind covers it.\n" - "Draw one, once:\n" + f"{version} is blinded ({why}) and no blind covers it in this checkout.\n" + "git pull first: its blind may already be drawn and committed. Never " + "draw a second one.\n" + "Only the custodian draws a blind, once:\n" f" APPTAINERENV_PYTHONPATH={repo}/src spv-container exec " f"python -m sp_validation.blinding init {base} " f"--cat-config {cat_config}\n" - "then commit cosmo_val/blinds//.\n" + "then commits cosmo_val/blinds//.\n" "Re-processing a catalogue whose blind is still concealed? Share it " f"instead: python -m sp_validation.blinding share {base} " f"--cat-config {cat_config}\n" @@ -198,10 +195,13 @@ def custody_of(catalogues, version, *, registry): """The custody of ``version``, from its base's declaration and the registry. @sc custody-is-declared - Custody is read only here, from the parsed catalogue config and the blind + Custody is declared once per *base* catalogue, as ``blinding:`` on its + entry in ``cosmo_val/cat_config.yaml``, and read only here, with the blind registry; no workflow config, environment variable or call argument can - change it. An entry declaring nothing is blinded, and variants share their - base's custody and blind. + change it. An entry declaring nothing is blinded, so a new catalogue fails + closed. Variants share their base's custody and blind: ``_leak_corr`` and + ``_seed`` versions, and entries naming their parent with ``base:``. A + blinded base is covered by exactly one blind. Raises ------ diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index bc8ce30d..c7abfc2c 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -762,9 +762,8 @@ def merge(saccs): block-diagonal is out of scope here; see ``assemble_covariance``). Metadata must be consistent: keys present in several inputs must carry - equal values (files under two custody stamps cannot merge), and the union - lands on the result. This deliberately replaces the - library's clash behaviour, which mangles clashing keys by appending + equal values, and the union lands on the result. This deliberately replaces + the library's clash behaviour, which mangles clashing keys by appending labels. Grid consistency follows tagging semantics: the ``grid`` tag declares @@ -906,43 +905,6 @@ def _check_grid_consistency(s, angle): ) -def update_statistic(s, sub): - """Overwrite the values of ``s``'s points that match ``sub``'s, in place. - - The merge-back half of the extract → conceal → merge blinding flow - (PRD #241 §4): each point of ``sub`` is matched to exactly one point of - ``s`` by ``(data_type, tracers, tags)``, and that point's *value* is - replaced. Nothing else changes — insertion order, tags, windows and the - covariance are untouched (blinding shifts the mean only), so ``sub``'s - own covariance (e.g. the sub-block ``extract`` attaches) is deliberately - not consulted. A ``sub`` point with no match, or with several, raises - ``ValueError``. - - Parameters - ---------- - s : sacc.Sacc - Target, mutated in place. - sub : sacc.Sacc - The replacement block, e.g. ``extract(s, ...)`` after concealment. - """ - claimed = set() - for point in sub.data: - idx = s.indices(point.data_type, point.tracers, **point.tags) - if len(idx) != 1: - raise ValueError( - f"update_statistic: {len(idx)} points in the target match " - f"({point.data_type}, {point.tracers}, {point.tags}) — need " - "exactly one" - ) - if idx[0] in claimed: - raise ValueError( - f"update_statistic: two sub points match the same target " - f"point ({point.data_type}, {point.tracers}, {point.tags})" - ) - claimed.add(idx[0]) - s.data[idx[0]].value = point.value - - # --------------------------------------------------------------------------- # # The file door: every SACC is born sealed, derived with one stamp, or refused # --------------------------------------------------------------------------- # diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 381aeefb..54f2ed40 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -619,7 +619,7 @@ def test_tomographic_xi_covariance_one_contiguous_block(): # --------------------------------------------------------------------------- # # 13. merge(): per-statistic files combine into one; shared tracers stored # once; covariance block-diagonal (all-or-none); metadata union with -# loud conflicts. update_statistic(): value-only merge-back. +# loud conflicts. # --------------------------------------------------------------------------- # def _xi_sacc(metadata=None): s = sio.new_sacc({0: _nz(0)}, metadata=metadata) @@ -708,46 +708,6 @@ def test_merge_conflicting_metadata_fails(): sio.merge([s_xi, s_co]) -def test_update_statistic_values_only(): - s = _multi_statistic_sacc() - tr = ("source_0", "source_0") - xi, cl, co = _xi_block(s, tr), _cl_block(s, tr), _cosebi_block(s, tr) - cov = [(xi, _spd(len(xi), 1)), (cl, _spd(len(cl), 2)), (co, _spd(len(co), 3))] - sio.assemble_covariance(s, cov) - dense_before = s.covariance.dense.copy() - mean_before = s.mean.copy() - - # extract -> shift (a stand-in for conceal) -> merge back - sub = sio.extract(s, data_type=sio.XI_PLUS, tracers=tr) - for point in sub.data: - point.value += 1e-4 - sio.update_statistic(s, sub) - - idx_p = s.indices(sio.XI_PLUS, tr) - assert np.allclose(s.mean[idx_p], mean_before[idx_p] + 1e-4) - untouched = np.setdiff1d(np.arange(len(s.mean)), idx_p) - assert np.array_equal(s.mean[untouched], mean_before[untouched]) - # covariance and insertion order untouched - assert np.array_equal(s.covariance.dense, dense_before) - - -def test_update_statistic_requires_unique_match(): - s = _xi_sacc() - sub = sio.extract(s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0")) - missing = sub.copy() - missing.data[0].tags["theta"] = 999.0 # matches nothing in s - with pytest.raises(ValueError, match="0 points"): - sio.update_statistic(s, missing) - - -def test_update_statistic_rejects_duplicate_sub_points(): - s = _xi_sacc() - sub = sio.extract(s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0")) - sub.data[1].tags = dict(sub.data[0].tags) # two sub points -> one target - with pytest.raises(ValueError, match="same target"): - sio.update_statistic(s, sub) - - def test_merge_rejects_divergent_shared_tracer(): s_xi, s_co = _xi_sacc(), _cosebi_sacc() s_co.tracers["source_0"].nz = s_co.tracers["source_0"].nz * 2.0 From 54c7b2470ed3ef5426e44bd77f334436fc90d14a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 11:31:22 +0200 Subject: [PATCH 085/160] workflow: a checkout other than production's names its output tree Custody, meaning the declaration and the blind registry, comes from the launched checkout, but COSMO_VAL defaults to the production checkout's tree. A scratch checkout that re-declares a production catalogue could therefore write its concealed parts over production's. configure() now refuses a launch from any other checkout unless COSMO_VAL is set, and the message names both choices. The production checkout keeps its default and its path strings. The candide dry-runs (P4) pass the production tree explicitly. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- workflow/README.md | 4 +++- workflow/common.py | 34 +++++++++++++++++++++++++--------- workflow/tests/test_dag.py | 12 ++++++++++++ 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index ea5e3575..39cfcd5c 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -145,7 +145,9 @@ halves of one commit, split. The catalogue config is the launched checkout's too: `cosmo_val/cat_config.yaml`, read by the host and handed to every job. The `papers/cosmo_val` suite writes only under `COSMO_VAL` or `COSMO_INFERENCE` (environment variables, defaulting -to the shared trees on candide) or the run directory's `results/`. Other rules +to the production checkout's trees on candide) or the run directory's +`results/`. A launch from any other checkout must set `COSMO_VAL`, so its +products never land in production's tree by default. Other rules write elsewhere: masks under the run directory's `output/masks/`, `papers/bmodes`' figures and macros under its run directory's `docs/`, the image sims under their `grids_base`. diff --git a/workflow/common.py b/workflow/common.py index 1054974b..d54564bc 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -55,16 +55,12 @@ def _load_checkout_module(name): # Output roots are env-overridable so a reproduction run can write into a -# fresh tree without clobbering (or silently reusing) prior products. -COSMO_VAL = Path( - os.environ.get( - "COSMO_VAL", "/n17data/cdaley/unions/code/sp_validation/cosmo_val/output" - ) -) +# fresh tree without clobbering (or silently reusing) prior products. They +# default to the production checkout's trees on candide. +PRODUCTION = Path("/n17data/cdaley/unions/code/sp_validation") +COSMO_VAL = Path(os.environ.get("COSMO_VAL", PRODUCTION / "cosmo_val" / "output")) COSMO_INFERENCE = Path( - os.environ.get( - "COSMO_INFERENCE", "/n17data/cdaley/unions/code/sp_validation/cosmo_inference" - ) + os.environ.get("COSMO_INFERENCE", PRODUCTION / "cosmo_inference") ) # The catalogue config of the launched checkout: the one file both the host # (CATALOG_CONFIG, loaded in configure) and every job read catalogues from, and @@ -225,11 +221,31 @@ def check_host_parity(image): ) +def check_output_root(): + """Stop another checkout's launch from writing into the production outputs. + + Custody is the launched checkout's, so a checkout other than + :data:`PRODUCTION` names its ``COSMO_VAL``: parts concealed under its + blinds, or re-measured under its declarations, never land in production's + tree unless asked to. + """ + from snakemake.exceptions import WorkflowError + + if "COSMO_VAL" in os.environ or PRODUCTION.resolve() == REPO_ROOT: + return + raise WorkflowError( + f"this checkout ({REPO_ROOT}) is not {PRODUCTION}, whose output tree " + f"COSMO_VAL defaults to. Name the tree to write into: COSMO_VAL= " + f"snakemake … (COSMO_VAL={COSMO_VAL} to write into production's)." + ) + + def configure(workflow_config): """Install config-derived values after Snakemake has loaded configfiles.""" global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, FIDUCIAL, PLANCK18 from snakemake.common.configfile import load_configfile + check_output_root() inject_checkout_pythonpath(workflow_config) warn_if_image_stale() check_host_parity(resolve_container(workflow_config.get("container"))) diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 34181509..61354a69 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -280,6 +280,14 @@ def test_the_campaign_type_switch_is_refused(toy, tmp_path): ) +def test_another_checkout_names_its_output_tree(toy): + """A checkout other than production's must set COSMO_VAL to launch.""" + env = {k: v for k, v in toy.env.items() if k != "COSMO_VAL"} + result = toy.snakemake("-n", "assemble_sacc_all", env=env) + assert result.returncode != 0, result.stdout + assert "COSMO_VAL=" in result.stdout + + def test_unblinding_a_concealed_catalogue_needs_the_reveal(toy): result = toy.snakemake("-n", "assemble_sacc_all", config=[f'versions=["{STALE}"]']) assert result.returncode != 0, result.stdout @@ -288,6 +296,10 @@ def test_unblinding_a_concealed_catalogue_needs_the_reveal(toy): def _real_dry_run(paper, targets): env = {k: v for k, v in os.environ.items() if k != "SNAKEMAKE_PROFILE"} + # Against the production output tree, whichever checkout runs the tests. + env.setdefault( + "COSMO_VAL", "/n17data/cdaley/unions/code/sp_validation/cosmo_val/output" + ) env.update(PYTHONUNBUFFERED="1", PYTHONNOUSERSITE="1") return subprocess.run( [ From 2280070a95324e1d11e3264064fb3c931f6c1dd3 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 11:34:31 +0200 Subject: [PATCH 086/160] E2E: pure-E/B from parts under two custodies writes nothing A cv_pure_eb run on a true reporting part and a concealed integration part must fail on the one-stamp check before any npz, figure or part exists. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/tests/test_custody_e2e.py | 44 ++++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py index bdb08f08..fd935d3d 100644 --- a/src/sp_validation/tests/test_custody_e2e.py +++ b/src/sp_validation/tests/test_custody_e2e.py @@ -206,6 +206,24 @@ def run_chain(cat_config, version, out, cov, *, grid_parts=None): "fiducial_scale_cut": SCALE_CUT, }, ) + pure_eb(version, pure_eb_outputs(version, out), xi, cov) + assemble(cat_config, version, out, paths, cov, token) + return paths + + +def pure_eb_outputs(version, out): + return { + "npz": str(out / f"{version}_pure_eb.npz"), + "sacc": str(out / f"{version}_pure_eb.sacc"), + "figure_integration_vs_reporting": str(out / f"{version}_eb_ivr.png"), + "figure_xis": str(out / f"{version}_eb_xis.png"), + "figure_ptes": str(out / f"{version}_eb_ptes.png"), + "figure_covariance": str(out / f"{version}_eb_cov.png"), + } + + +def pure_eb(version, output, xi, cov): + """Rule cv_pure_eb for ``version`` from the ξ± parts ``xi`` (by grid).""" rep = GRIDS["reporting"] run_rule( "cv_pure_eb.py", @@ -214,14 +232,7 @@ def run_chain(cat_config, version, out, cov, *, grid_parts=None): "xi_integration": str(xi["integration"]), "cov_integration": str(cov["integration"]), }, - output={ - "npz": str(out / f"{version}_pure_eb.npz"), - "sacc": str(paths["pure_eb"]), - "figure_integration_vs_reporting": str(out / f"{version}_eb_ivr.png"), - "figure_xis": str(out / f"{version}_eb_xis.png"), - "figure_ptes": str(out / f"{version}_eb_ptes.png"), - "figure_covariance": str(out / f"{version}_eb_cov.png"), - }, + output=output, params={ "version": version, "min_sep": rep["min_sep"], @@ -233,8 +244,6 @@ def run_chain(cat_config, version, out, cov, *, grid_parts=None): "fiducial_scale_cut": SCALE_CUT, }, ) - assemble(cat_config, version, out, paths, cov, token) - return paths def assemble(cat_config, version, out, paths, cov, token): @@ -358,6 +367,21 @@ def test_a_blinded_catalogue_from_birth_to_audit(toy, monkeypatch): toy.cov, blinded.token, ) + # Pure-E/B from parts under two custodies writes nothing at all. + mixed = pure_eb_outputs("TOY", toy.root / "mixed") + (toy.root / "mixed").mkdir() + with pytest.raises(ValueError, match="stamps"): + pure_eb( + "TOY", + mixed, + { + "reporting": open_parts["reporting"], + "integration": out / "TOY_xi_integration.sacc", + }, + toy.cov, + ) + assert not list((toy.root / "mixed").iterdir()) + plain = sio.load(open_parts["reporting"]) for key in cu.STAMP_KEYS: plain.metadata.pop(key, None) From 606a1bb9f9f811d8dfc9704e7638c6961c65e6bf Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 11:52:39 +0200 Subject: [PATCH 087/160] Blinding: a schema refusal names both field lists plainly; P6 pins "git pull first" Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/blinding.py | 9 +++++---- workflow/tests/test_dag.py | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 925270a4..0ecc5cbc 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -123,10 +123,11 @@ def _recorded_config(name, record): unset = sorted(fields - set(theory)) + ["envelope"] * ("envelope" not in record) if foreign or unset: raise _custody.CustodyError( - f"blind {name} was drawn under a config this code cannot reproduce: " - f"the record names {foreign}, which this code lacks, and lacks " - f"{unset}, which have no value in blinding.NEUTRAL. Open it with the " - "code it was drawn under, or give each new field its neutral value." + f"blind {name} was drawn under a config this code cannot reproduce. " + f"Fields the record names and this code lacks: {foreign or 'none'}. " + "Fields this code has and the record lacks, with no value in " + f"blinding.NEUTRAL: {unset or 'none'}. Open the blind with the code " + "it was drawn under, or give each new field its neutral value." ) return BlindingConfig( envelope={k: float(v) for k, v in envelope.items()}, diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 61354a69..30053aee 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -222,11 +222,12 @@ def _custody_lines(output): def test_a_catalogue_without_a_blind_stops_the_launch(toy): - """A blinded catalogue with no blind fails at parse, naming the one command.""" + """A blinded catalogue with no blind fails at parse: pull, then draw or share.""" result = toy.snakemake( "-n", "assemble_sacc_all", config=[f'versions=["{UNCOVERED}"]'] ) assert result.returncode != 0, result.stdout + assert "git pull first" in result.stdout assert "python -m sp_validation.blinding init" in result.stdout assert "share" in result.stdout assert "rule assemble_sacc" not in result.stdout From 4c95ffc55f60e01f7f26aafe4503fda054e83e70 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 13:01:01 +0200 Subject: [PATCH 088/160] Blinding: reveal refuses an empty root; the audit compares signal; init opens its record first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - reveal collects the concealed parts before publishing the seed, and refuses a root holding none (mistyped, or unbound in the container) unless its archive already holds parts, so an interrupted reveal re-runs. audit names an empty archive. - The audit compares only signal rows (and their covariance): ρ/τ carries no signal and a re-run does not reproduce it, so a ρ/τ part is judged by its stamp. Cℓ_BB/EB still may not move. Derived E rows are reported, not proven; the docstring and README say so. - init opens its record through the path _open runs and conceals one ξ± row before writing anything, so a config no job can conceal under (an unknown envelope or theory key, an unbuildable theory) never becomes a blind. BlindingConfig.from_record goes: init --config merges over the defaults and reads through _recorded_config, the one reader of the record format. - One declared-custody reader, blinding.declared_custody, for verify, assemble_sacc and the tests; the audit compares the live stamp's status and catalogue instead of spelling a token. - Tests drive verify, share, the job-side token check and the installed draw-scheme check to failure; the committed-blind test no longer ties the fixture to today's defaults. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/blinding.py | 175 ++++++++++++------- src/sp_validation/sacc_io.py | 9 +- src/sp_validation/tests/test_blinding.py | 181 +++++++++++++++++++- src/sp_validation/tests/test_custody_e2e.py | 47 +++-- workflow/README.md | 6 +- workflow/scripts/assemble_sacc.py | 11 +- 6 files changed, 329 insertions(+), 100 deletions(-) diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 0ecc5cbc..bae37e9e 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -11,17 +11,12 @@ concealed ξ± are concealed with it; the shift is pure E-mode, so B-mode null tests stay valid. - A blind is drawn once, by a person, with ``python -m sp_validation.blinding - init``, into the registry beside the catalogue config - (``cosmo_val/blinds//``): the public ``commitment.json`` (the seed's - commitment and the full :class:`BlindingConfig`), the seed as Fernet - ciphertext (``seed.fernet``), its ``key``, and the ``bases`` it covers. The - seed exists on disk only inside the ciphertext; the encryption keeps anyone - from reading it by accident. ``share`` adds a catalogue to a blind; - ``reveal`` publishes the seed and moves the concealed files aside; - ``audit`` proves blinded − true = shift(seed) once the true files are - re-measured; ``verify`` checks a file's stamp against its catalogue's - custody, seedless. + The commands of ``python -m sp_validation.blinding`` are the only writers + of the blind registry (``cosmo_val/blinds/CONTRACTS``): ``init`` draws a + blind, once, by a person; ``share`` adds a catalogue to it; ``reveal`` + publishes its seed and moves the concealed files aside. ``audit`` checks + the re-measured true files against that archive, and ``verify`` checks a + file's stamp against its catalogue's custody, seedless. """ import argparse @@ -66,16 +61,6 @@ def record(self): """The config as plain data: every field, numbers as floats.""" return _canonical(self) - @classmethod - def from_record(cls, record): - """The config a partial record names; fields it omits take their defaults.""" - fields = {} - if "envelope" in record: - fields["envelope"] = {k: float(v) for k, v in record["envelope"].items()} - if "theory" in record: - fields["theory"] = TheoryConfig(**record["theory"]) - return cls(**fields) - def digest(self): """sha256 of the canonical record; int and float literals agree.""" return record_digest(self.record()) @@ -109,11 +94,11 @@ def _canonical(value): def _recorded_config(name, record): - """The config blind ``name`` was drawn under, from its stored record. + """The :class:`BlindingConfig` of blind ``name``, from its config record. - Refuses, naming the fields, a record whose shift this code cannot - reproduce: one naming a field the code lacks, or lacking a code field that - has no value in :data:`NEUTRAL`. + The one reader of the record format. Refuses, naming the fields, a record + whose shift this code cannot reproduce: one naming a field the code lacks, + or lacking a code field that has no value in :data:`NEUTRAL`. """ fields = {f.name for f in dataclasses.fields(TheoryConfig)} theory = {**NEUTRAL, **record.get("theory", {})} @@ -123,11 +108,11 @@ def _recorded_config(name, record): unset = sorted(fields - set(theory)) + ["envelope"] * ("envelope" not in record) if foreign or unset: raise _custody.CustodyError( - f"blind {name} was drawn under a config this code cannot reproduce. " - f"Fields the record names and this code lacks: {foreign or 'none'}. " - "Fields this code has and the record lacks, with no value in " - f"blinding.NEUTRAL: {unset or 'none'}. Open the blind with the code " - "it was drawn under, or give each new field its neutral value." + f"blind {name}'s config does not fit this code. Fields the record " + f"names and this code lacks: {foreign or 'none'}. Fields this code " + "has and the record lacks, with no value in blinding.NEUTRAL: " + f"{unset or 'none'}. A drawn blind opens under the code it was " + "drawn with, or once each new field has its neutral value." ) return BlindingConfig( envelope={k: float(v) for k, v in envelope.items()}, @@ -172,6 +157,12 @@ class Blind: hidden: TheoryConfig +def _blind(name, seed, record): + """Blind ``name`` from its seed and config record, as it is opened.""" + config = _recorded_config(name, record) + return Blind(name, seed, config, hidden_theory(seed, config)) + + @functools.cache def _open(registry, name): from cryptography.fernet import Fernet, InvalidToken @@ -212,9 +203,7 @@ def _open(registry, name): f"blind {name} was drawn under draw scheme {c['draw_scheme']}; this " f"install's smokescreen draws under {draw_scheme()}" ) - config = _recorded_config(name, c["config"]) - seed = payload["seed"] - return Blind(name, seed, config, hidden_theory(seed, config)) + return _blind(name, payload["seed"], c["config"]) def open_blind(custody): @@ -355,6 +344,14 @@ def _catalogues(cat_config): return yaml.safe_load(Path(cat_config).read_text()) +def declared_custody(cat_config, version): + """The custody ``version`` is declared under in the catalogue config at + ``cat_config``, with the blind registry beside it.""" + return _custody.custody_of( + _catalogues(cat_config), version, registry=_custody.registry_of(cat_config) + ) + + def _theory_stack(): from importlib.metadata import PackageNotFoundError, version @@ -384,11 +381,28 @@ def _declared_blinded(catalogues, registry, base): raise _custody.CustodyError(f"{base} is already covered by blind {covering[0]}") +def _conceal_one_row(name, seed, record): + """Open blind ``name`` from its seed and config record and conceal one ξ± + row under it, as every blinded birth does; raise if it cannot.""" + blind = _blind(name, seed, record) + z = np.linspace(0.0, 2.0, 101) + s = sacc_io.new_sacc({0: (z, np.exp(-0.5 * ((z - 0.7) / 0.2) ** 2))}) + sacc_io.add_xi(s, (0, 0), [10.0], [0.0], [0.0], grid="probe") + try: + factors(s, blind.config.theory, blind.hidden) + except Exception as err: + raise _custody.CustodyError( + f"no row conceals under blind {name}'s config: {type(err).__name__}: {err}" + ) from err + + def init(name, bases, *, cat_config, config=None): """Draw blind ``name`` for ``bases`` and write its record; return its directory. The one place a seed is drawn. Existing state is refused, never replaced, - and the seed is held in memory and written only as ciphertext. + and the seed is held in memory and written only as ciphertext. The record + is opened and conceals a row before anything is written, so every blind + on disk is one its jobs can conceal under. """ from cryptography.fernet import Fernet @@ -427,6 +441,7 @@ def init(name, bases, *, cat_config, config=None): "theory_stack": _theory_stack(), "created": datetime.now(timezone.utc).isoformat(timespec="seconds"), } + _conceal_one_row(name, seed, commitment["config"]) staging.mkdir(parents=True) (staging / "commitment.json").write_text(json.dumps(commitment, indent=2) + "\n") @@ -482,20 +497,30 @@ def reveal(name, *, root, cat_config): blind = _open(registry, name) record = _custody.records(registry)[name] revealed = registry / name / "revealed.json" + if record.revealed is not None and record.revealed != blind.seed: + raise _custody.CustodyError(f"{revealed} records another seed") + archive = Path(root) / "revealed" / name + concealed = list(_parts_under(root, record.commitment["seed_commitment"], archive)) + # Publishing cannot be undone, and a reveal that archived nothing leaves + # the concealed products unauditable: an empty (mistyped, or unbound in + # the container) root is refused first. A re-run finds its archive. + if not concealed and not any(archive.rglob("*.sacc")): + raise _custody.CustodyError( + f"nothing concealed under blind {name} in {root}; check --root (and " + "that the container binds it). The seed stays unpublished." + ) if record.revealed is None: fd = os.open(revealed, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444) with os.fdopen(fd, "w") as f: json.dump({"seed": blind.seed}, f) - elif record.revealed != blind.seed: - raise _custody.CustodyError(f"{revealed} records another seed") - archive = Path(root) / "revealed" / name - moved = 0 - for path in _parts_under(root, record.commitment["seed_commitment"], archive): + for path in concealed: target = archive / path.relative_to(root) target.parent.mkdir(parents=True, exist_ok=True) shutil.move(path, target) - moved += 1 - print(f"[blinding] published the seed of {name}; moved {moved} files to {archive}") + print( + f"[blinding] published the seed of {name}; moved {len(concealed)} files " + f"to {archive}" + ) print( f"[blinding] commit {revealed} and declare `blinding: unblinded` on " f"{', '.join(record.bases)}, then re-run" @@ -549,12 +574,30 @@ def _same_covariance(a, b): return a.shape == b.shape and np.allclose(a, b, rtol=0.0, atol=_REMEASURED * scale) +def _signal(s): + """``s``'s signal rows and their covariance, or None if it has none.""" + keep = np.array([sacc_io.is_signal(dp.data_type) for dp in s.data], bool) + if not keep.any(): + return None + out = s.copy() + out.keep_indices(keep) + return out + + def _audit_part(blinded, true, fiducial, hidden, tolerance): - """Problems with one archived part against its re-measured twin.""" + """Problems with one archived part against its re-measured twin. + + Only signal rows are compared: ρ/τ carries none, and a re-run does not + reproduce it (TreeCorr draws its jackknife patches afresh), so a part + without signal is judged by its stamp alone. + """ stamp, true_stamp = (_custody.read_stamp(x.metadata) for x in (blinded, true)) - if true_stamp.token != f"unblinded:{stamp.catalogue}": + if (true_stamp.status, true_stamp.catalogue) != ("unblinded", stamp.catalogue): return [f"the live file is stamped {true_stamp.token}"], None - if not _rows_match(blinded, true): + blinded, true = _signal(blinded), _signal(true) + if blinded is None and true is None: + return [], {"signal_rows": 0} + if blinded is None or true is None or not _rows_match(blinded, true): return ["rows, tags or tracers differ"], None problems = [] if set(blinded.tracers) != set(true.tracers): @@ -599,13 +642,14 @@ def audit(name, *, archive, true_root, cat_config, out=None): The published seed must be the committed one. For each archived file, the live file at the same relative path must be stamped unblinded for the same - catalogue, with the same rows, tags, tracers, covariance and patch centres - (numbers to a re-measurement's float noise), and blinded − true must equal - the seed's shift on every ξ± and Cℓ_EE block to 1e-6 of the block's - largest shift (1e-3 when the theory stack differs from the one the blind - was drawn with); a derived statistic's B rows may move by at most - :data:`B_SIGMA`, and rows the blind leaves unshifted not at all. Each part - reports the largest shift of each derived statistic, in σ. + catalogue. Its signal rows must match the archived ones in tags, tracers, + covariance and patch centres (numbers to a re-measurement's float noise), + and blinded − true must equal the seed's shift on every ξ± and Cℓ_EE block + to 1e-6 of the block's largest shift (1e-3 when the theory stack differs + from the one the blind was drawn with); Cℓ_BB and Cℓ_EB may not move, and + a derived statistic's B rows by at most :data:`B_SIGMA`. A derived + statistic's E rows are reported, in σ, not proven: the archive does not + record which ξ± parts, through which kernel, they came from. """ registry = _custody.registry_of(cat_config) report = {"blind": name, "ok": False, "problems": [], "parts": {}} @@ -647,10 +691,10 @@ def audit(name, *, archive, true_root, cat_config, out=None): blinded, sacc_io.load(live), fiducial, hidden, tolerance ) report["parts"][relative] = {"problems": problems, **(numbers or {})} - report["ok"] = ( - bool(report["parts"]) - and not report["problems"] - and not any(part["problems"] for part in report["parts"].values()) + if not report["parts"]: + report["problems"].append("the archive holds no parts") + report["ok"] = not report["problems"] and not any( + part["problems"] for part in report["parts"].values() ) return _write_report(report, out) @@ -665,11 +709,7 @@ def verify(path, *, cat_config): """Problems with a file's stamp against its catalogue's custody (seedless).""" stamp = _custody.read_stamp(sacc_io.load(path).metadata) try: - declared = _custody.custody_of( - _catalogues(cat_config), - stamp.catalogue, - registry=_custody.registry_of(cat_config), - ) + declared = declared_custody(cat_config, stamp.catalogue) except _custody.CustodyError as err: return [str(err)] problems = [] @@ -703,7 +743,9 @@ def main(argv=None): sub.add_argument(name, nargs="+" if plus else None) sub.add_argument("--cat-config", required=True) commands.choices["init"].add_argument( - "--config", help="JSON record of BlindingConfig fields (defaults otherwise)" + "--config", + help="JSON of BlindingConfig fields over the defaults: an `envelope` " + "replaces the default one; `theory` fields replace theirs", ) commands.choices["reveal"].add_argument("--root", required=True) commands.choices["audit"].add_argument("--archive", required=True) @@ -712,11 +754,12 @@ def main(argv=None): a = parser.parse_args(argv) if a.command == "init": - config = ( - BlindingConfig.from_record(json.loads(Path(a.config).read_text())) - if a.config - else None - ) + config = None + if a.config: + given = json.loads(Path(a.config).read_text()) + default = BlindingConfig().record() + theory = {**default["theory"], **given.get("theory", {})} + config = _recorded_config(a.blind, {**default, **given, "theory": theory}) init(a.blind, a.bases, cat_config=a.cat_config, config=config) elif a.command == "share": share(a.blind, a.base, cat_config=a.cat_config) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index c7abfc2c..5ec9d6f8 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -916,15 +916,18 @@ def _check_grid_consistency(s, angle): UNSHIFTED = (CL_BB, CL_EB) +def is_signal(data_type): + """Whether ``data_type`` carries cosmological signal.""" + return data_type.startswith(SIGNAL_PREFIX) + + def is_derived(data_type): """Whether ``data_type`` is a derived statistic (COSEBIs, pure-E/B, …). Signal the blind neither conceals nor leaves unchanged: it moves only through the shiftable rows it is computed from. """ - return data_type.startswith(SIGNAL_PREFIX) and data_type not in ( - SHIFTABLE + UNSHIFTED - ) + return is_signal(data_type) and data_type not in SHIFTABLE + UNSHIFTED def _stamped(s): diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 0f1b71e3..6be7a15e 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -61,10 +61,15 @@ def _init(root, blind, *bases): def _custody(root, version): + return bd.declared_custody(root / "cat_config.yaml", version) + + +def _declare(root, **entries): + """Set catalogue entries of ``root``'s catalogue config.""" path = root / "cat_config.yaml" - return cu.custody_of( - yaml.safe_load(path.read_text()), version, registry=cu.registry_of(path) - ) + config = yaml.safe_load(path.read_text()) + config.update(entries) + path.write_text(yaml.safe_dump(config)) @pytest.fixture(scope="module") @@ -617,7 +622,7 @@ def test_the_digest_ignores_int_versus_float(): theory=TheoryConfig(w0=-1, wa=0, ia_bias=0), ) assert a.digest() == b.digest() - assert bd.BlindingConfig.from_record(json.loads(json.dumps(a.record()))) == a + assert bd._recorded_config("toy", json.loads(json.dumps(a.record()))) == a def test_the_commitment_is_the_forks(): @@ -701,6 +706,59 @@ def test_the_record_is_read_only(fresh): ) +@pytest.mark.parametrize( + "config, named", + [ + ({"envelope": {"S8": 0.075, "Omega_M": 0.1}}, "Omega_M"), + ({"theory": {"transfer_function": "eisenstein_hu", "Omega_M": 0.3}}, "Omega_M"), + ({"theory": {"transfer_function": "eisenstein-hu"}}, "eisenstein-hu"), + ], + ids=["envelope_key", "theory_key", "unbuildable"], +) +def test_init_refuses_a_config_no_blind_conceals_under(tmp_path, config, named): + """A config whose blind could not conceal a row is refused before any + record exists, so the base stays free for a blind drawn under a good one.""" + _cat_config(tmp_path) + (tmp_path / "bad.json").write_text(json.dumps(config)) + args = ["init", "toy", "TOY", "--cat-config", str(tmp_path / "cat_config.yaml")] + with pytest.raises(cu.CustodyError, match=named): + bd.main([*args, "--config", str(tmp_path / "bad.json")]) + assert not (tmp_path / "blinds").exists() or not any( + (tmp_path / "blinds").iterdir() + ) + _init(tmp_path, "toy", "TOY") + + +def test_share_adds_a_base_to_a_concealed_blind(fresh): + _declare(fresh, TOY_V={"base": "TOY"}, LATER={}) + cat_config = str(fresh / "cat_config.yaml") + for base, refusal in { + "TOY": "already covered", + "TOY_V": "variant", + "TOY_leak_corr": "variant", + "TOY_OPEN": "blinded first", + "TOY_MOCK": "blinded first", + }.items(): + with pytest.raises(cu.CustodyError, match=refusal): + bd.share("toy", base, cat_config=cat_config) + with pytest.raises(cu.CustodyError, match="no blind"): + _custody(fresh, "OTHER") + + assert bd.main(["share", "toy", "OTHER", "--cat-config", cat_config]) == 0 + toy, other = _custody(fresh, "TOY"), _custody(fresh, "OTHER") + assert (other.status, other.blind, other.commitment) == ( + "blinded", + "toy", + toy.commitment, + ) + + (fresh / "blinds" / "toy" / "revealed.json").write_text( + json.dumps({"seed": bd.open_blind(toy).seed}) + ) + with pytest.raises(cu.CustodyError, match="revealed"): + bd.share("toy", "LATER", cat_config=cat_config) + + # --------------------------------------------------------------------------- # # I10: an opened blind is the one committed # --------------------------------------------------------------------------- # @@ -754,10 +812,13 @@ def _init_storing(root, edit): """Draw blind toy for TOY (seed ``toy-seed``), its config stored as ``edit`` makes it: the record a code with another TheoryConfig schema writes.""" record = bd.BlindingConfig.record + config = bd.BlindingConfig(theory=TheoryConfig(**FAST["theory"])) with pytest.MonkeyPatch.context() as m: m.setattr(bd.BlindingConfig, "record", lambda self: edit(record(self))) m.setattr(bd.secrets, "token_hex", lambda n: "toy-seed") - _init(root, "toy", "TOY") + # That code's init opened the record under its own schema. + m.setattr(bd, "_conceal_one_row", lambda name, seed, record: None) + bd.init("toy", ["TOY"], cat_config=root / "cat_config.yaml", config=config) def test_a_record_lacking_a_field_opens_to_the_shift_it_was_drawn_with( @@ -803,12 +864,59 @@ def test_the_committed_blind_opens_under_this_code(): strand every live blind; this record turns that red. """ blind = bd._open(DATA / "blinds", "committed") - assert blind.config == bd.BlindingConfig() assert (blind.hidden.S8, blind.hidden.Omega_m) == pytest.approx( COMMITTED_POINT, rel=1e-12 ) +def test_a_blind_drawn_under_another_draw_scheme_is_refused(tmp_path, monkeypatch): + """Its seed and record agree, but this install's fork draws differently: + opening it is refused, and a file stamped under it fails ``verify``.""" + _cat_config(tmp_path) + installed = bd.draw_scheme() + with monkeypatch.context() as m: + m.setattr(bd, "draw_scheme", lambda: installed + 1) + _init(tmp_path, "toy", "TOY") + custody = _custody(tmp_path, "TOY") + with pytest.raises(cu.CustodyError, match="draw scheme"): + bd.open_blind(custody) + part = tmp_path / "rho_tau.sacc" + sio.save(rho_sacc(), part, custody=custody) # no signal: the blind stays shut + assert bd.verify(part, cat_config=tmp_path / "cat_config.yaml") == [ + "this install draws under another scheme" + ] + + +# --------------------------------------------------------------------------- # +# verify: a file's stamp against its catalogue's custody, seedless +# --------------------------------------------------------------------------- # +def _verify(root, path, capsys): + code = bd.main(["verify", str(path), "--cat-config", str(root / "cat_config.yaml")]) + return code, capsys.readouterr().out + + +def test_verify_names_what_disagrees_with_the_declaration(fresh, capsys): + part = fresh / "part.sacc" + sio.save(two_bin_sacc(cl=False), part, custody=_custody(fresh, "TOY")) + assert _verify(fresh, part, capsys)[0] == 0 + + forged = fresh / "forged.sacc" + _write( + forged, + two_bin_sacc(cl=False), + {**_custody(fresh, "TOY").stamp, "blinding_commitment": "0" * 64}, + ) + code, out = _verify(fresh, forged, capsys) + assert code == 1 and "['blinding_commitment']" in out + + (fresh / "blinds" / "toy" / "revealed.json").write_text( + json.dumps({"seed": bd.open_blind(_custody(fresh, "TOY")).seed}) + ) + _declare(fresh, TOY={"blinding": "unblinded"}) + code, out = _verify(fresh, part, capsys) + assert code == 1 and "'blinding'" in out and "unblinded:TOY" in out + + # --------------------------------------------------------------------------- # # I11: the audit proves blinded − true = shift(seed) # --------------------------------------------------------------------------- # @@ -852,6 +960,42 @@ def _problems(report): ] +def test_reveal_refuses_a_root_holding_nothing_concealed(fresh): + """A mistyped or unbound ``--root`` is refused before the seed is published; + a reveal interrupted after its first move runs again.""" + root = fresh / "output" + part = root / "sub" / "part.sacc" + part.parent.mkdir(parents=True) + sio.save(two_bin_sacc(cl=False), part, custody=_custody(fresh, "TOY")) + revealed = fresh / "blinds" / "toy" / "revealed.json" + cat_config = fresh / "cat_config.yaml" + (fresh / "empty").mkdir() + for wrong in (fresh / "outptu", fresh / "empty"): + with pytest.raises(cu.CustodyError, match="nothing concealed"): + bd.reveal("toy", root=wrong, cat_config=cat_config) + assert not revealed.exists() + + archive = bd.reveal("toy", root=root, cat_config=cat_config) + assert revealed.exists() and (archive / "sub" / "part.sacc").exists() + assert bd.reveal("toy", root=root, cat_config=cat_config) == archive + with pytest.raises(cu.CustodyError, match="nothing concealed"): + bd.reveal("toy", root=fresh / "empty", cat_config=cat_config) + + +def test_the_audit_of_an_empty_archive_says_so(fresh): + (fresh / "blinds" / "toy" / "revealed.json").write_text( + json.dumps({"seed": bd.open_blind(_custody(fresh, "TOY")).seed}) + ) + (fresh / "archive").mkdir() + report = bd.audit( + "toy", + archive=fresh / "archive", + true_root=fresh, + cat_config=fresh / "cat_config.yaml", + ) + assert not report["ok"] and report["problems"] == ["the archive holds no parts"] + + def test_the_audit_passes_on_a_true_reveal(fresh): s = two_bin_sacc(rho=True) report = _audit(fresh, _concealed(fresh, s), s) @@ -889,7 +1033,30 @@ def test_the_audit_fails_on_a_mixed_file(fresh, times): assert any("≠ shift(seed)" in problem for problem in _problems(report)) -@pytest.mark.parametrize("kind", [sio.CL_BB, sio.RHO_PLUS.format(k=0)]) +@pytest.mark.parametrize( + "make", [rho_sacc, two_bin_sacc], ids=["alone", "beside_signal"] +) +def test_the_audit_passes_rho_tau_remeasured_at_run_noise(fresh, make): + """ρ/τ carries no signal, and a re-run does not reproduce it (TreeCorr's + k-means patches move θ by ~1e-3 and the values by O(1)): its part is + judged by its stamp, and beside signal rows only the signal is compared.""" + true = make(rho=True) if make is two_bin_sacc else make() + if true.covariance is None: + true.add_covariance(np.full(len(true.mean), 1e-14)) + archived = _concealed(fresh, true) + rerun = true.copy() + rho = [i for i, dp in enumerate(true.data) if not sio.is_signal(dp.data_type)] + cov = np.array(true.covariance.dense) + for i in rho: + rerun.data[i].value = 1.5 * true.data[i].value + 1e-7 + rerun.data[i].tags["theta"] *= 1 + 1.7e-3 + cov[i, i] *= 1.036 + rerun.add_covariance(cov, overwrite=True) + report = _audit(fresh, archived, rerun) + assert report["ok"], report + + +@pytest.mark.parametrize("kind", [sio.CL_BB, sio.CL_EB]) def test_the_audit_fails_when_an_unshifted_row_moved(fresh, kind): s = two_bin_sacc(rho=True) archived = _concealed(fresh, s) diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py index fd935d3d..66a89750 100644 --- a/src/sp_validation/tests/test_custody_e2e.py +++ b/src/sp_validation/tests/test_custody_e2e.py @@ -71,15 +71,6 @@ def run_rule(script, *, input=None, output=None, params=None): ) -def declared(cat_config, version): - """The custody a host Snakemake resolves for ``version``.""" - return cu.custody_of( - yaml.safe_load(Path(cat_config).read_text()), - version, - registry=cu.registry_of(cat_config), - ) - - def _namaster_part_inputs(): """``(ell_eff, cl_all, workspace)`` on an nside-32 full-sky workspace.""" import healpy as hp @@ -144,7 +135,7 @@ def _covariances(root): def run_chain(cat_config, version, out, cov, *, grid_parts=None): """The cosmo_val chain for one version, into ``out``; returns the part paths.""" out.mkdir(parents=True, exist_ok=True) - token = declared(cat_config, version).token + token = bd.declared_custody(cat_config, version).token xi = {} for grid, b in GRIDS.items(): xi[grid] = out / f"{version}_xi_{grid}.sacc" @@ -316,7 +307,7 @@ def toy(tmp_path, monkeypatch): def test_a_blinded_catalogue_from_birth_to_audit(toy, monkeypatch): out = toy.root / "cosmo_val" - blinded = declared(toy.cat_config, "TOY") + blinded = bd.declared_custody(toy.cat_config, "TOY") assert blinded.status == "blinded" # --- a blinded run ------------------------------------------------------- @@ -443,7 +434,7 @@ def second_block_fails(*args, **kwargs): config = yaml.safe_load(toy.cat_config.read_text()) config["TOY"]["blinding"] = "unblinded" toy.cat_config.write_text(yaml.safe_dump(config, sort_keys=False)) - true = declared(toy.cat_config, "TOY") + true = bd.declared_custody(toy.cat_config, "TOY") assert true.token == "unblinded:TOY" for version in ("TOY", "TOY_leak_corr"): @@ -475,7 +466,37 @@ def refuse(custody): monkeypatch.setattr(bd, "open_blind", refuse) out = toy.root / "mock" run_chain(toy.cat_config, "TOY_MOCK", out, toy.cov) - mock = declared(toy.cat_config, "TOY_MOCK") + mock = bd.declared_custody(toy.cat_config, "TOY_MOCK") found = stamps(out) assert len(found) == 2 + len(PARTS) assert all(s == mock.stamp for s in found.values()), found + + +@pytest.mark.parametrize( + "script", ["run_2pcf.py", "run_rho_tau.py", "assemble_sacc.py"] +) +def test_a_job_refuses_a_custody_changed_since_the_launch(toy, script): + """TOY's launch resolved it unblinded; declared blinded by the time the job + runs, the job refuses before it measures or writes anything.""" + out = toy.root / "stale" + outputs = { + "sacc": out / "part.sacc", + "rho_stats": out / "rho.fits", + "tau_stats": out / "tau.fits", + "rho_tau": out / "rho_tau.sacc", + } + out.mkdir() + with pytest.raises(cu.CustodyError, match="changed since the launch"): + run_rule( + script, + output={k: str(v) for k, v in outputs.items()}, + params={ + "ver": "TOY", + "version": "TOY", + **GRIDS["reporting"], + "cat_config": str(toy.cat_config), + "output_dir": str(out), + "custody": "unblinded:TOY", + }, + ) + assert not list(out.iterdir()) diff --git a/workflow/README.md b/workflow/README.md index 39cfcd5c..b8f61c37 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -63,8 +63,10 @@ Revealing re-measures; nothing subtracts a shift: on the blind's catalogues; 3. the pipeline re-measures the true products; 4. `… blinding audit --archive /revealed/ --true-root - --cat-config …` checks blinded − true = shift(seed) on every - archived file. + --cat-config …` checks blinded − true = shift(seed) on every ξ± + and Cℓ_EE row of every archived file, and bounds the derived B-modes (their + E-mode shifts are reported). ρ/τ is judged by its stamp: it carries no + signal, and a re-run does not reproduce it. `… blinding verify --cat-config …` checks a file's stamp against the registry, without the seed. diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 786ccd9d..947ea27b 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -14,14 +14,13 @@ """ import argparse -from pathlib import Path import numpy as np -import yaml from sp_validation import sacc_io +from sp_validation.blinding import declared_custody from sp_validation.cosmo_val.sacc_writers import assemble_analysis_sacc -from sp_validation.custody import confirm, custody_of, registry_of +from sp_validation.custody import confirm # NaMaster iNKA covariance FITS: per-spectrum HDU names, in SACC insertion order. _CL_HDUS = ("COVAR_EE_EE", "COVAR_BB_BB", "COVAR_EB_EB") @@ -89,12 +88,6 @@ def _attach_cov(part, name, xi_cov, pseudo_cl_cov): return part -def declared_custody(cat_config, version): - """The custody ``version`` is declared under in ``cat_config``.""" - catalogues = yaml.safe_load(Path(cat_config).read_text()) - return custody_of(catalogues, version, registry=registry_of(cat_config)) - - def assemble_sacc( version, part_paths, From 57a59f5963896aadc84ed0b1ef98c1abe9177c83 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 13:01:02 +0200 Subject: [PATCH 089/160] =?UTF-8?q?Docs=20say=20each=20fact=20once;=20the?= =?UTF-8?q?=20=CF=81/=CF=84=20script=20checks=20its=20outputs=20with=20ver?= =?UTF-8?q?ify=5Foutputs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The registry layout lives in cosmo_val/blinds/CONTRACTS alone (custody.py and blinding.py point there); custody.py's stdlib paragraph goes to its @sc contract; TheoryConfig no longer names its test; cv_pure_eb states its covariance's independence once. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- cosmo_val/blinds/CONTRACTS | 12 +++++++----- src/sp_validation/blinding_theory.py | 5 ++--- src/sp_validation/custody.py | 7 +------ workflow/scripts/cv_pure_eb.py | 11 +++++------ workflow/scripts/run_rho_tau.py | 13 ++----------- 5 files changed, 17 insertions(+), 31 deletions(-) diff --git a/cosmo_val/blinds/CONTRACTS b/cosmo_val/blinds/CONTRACTS index f1055209..082b6335 100644 --- a/cosmo_val/blinds/CONTRACTS +++ b/cosmo_val/blinds/CONTRACTS @@ -1,8 +1,10 @@ @sc blind-drawn-once The blind registry: one directory per blind, written only by `python -m sp_validation.blinding`. `init` creates / with -commitment.json (the public record), seed.fernet (the seed, encrypted) and -key, all read-only, and `bases`; `share` appends a base catalogue to `bases`; -`reveal` writes revealed.json once. The record is committed to git. No -Snakemake rule reads, writes or draws anything here, so no run can re-draw a -blind, and every checkout resolves the same one. +commitment.json (the public record: the seed's commitment, the full +BlindingConfig as data, its digest, the draw scheme and the theory stack), +seed.fernet (the seed, encrypted) and key, all read-only, and `bases`, the +base catalogues it covers; `share` appends a base catalogue to `bases`; +`reveal` writes revealed.json (the published seed) once. The record is +committed to git. No Snakemake rule reads, writes or draws anything here, so +no run can re-draw a blind, and every checkout resolves the same one. diff --git a/src/sp_validation/blinding_theory.py b/src/sp_validation/blinding_theory.py index 360cd1a5..24be52c8 100644 --- a/src/sp_validation/blinding_theory.py +++ b/src/sp_validation/blinding_theory.py @@ -36,9 +36,8 @@ class TheoryConfig: Every field is a deliberate, configurable choice. The defaults mirror the ``cosmo_inference`` CosmoSIS fiducial (the ``SP_v1.4.6.3_A_cell`` pipeline + ``values_ia.ini`` central values), so the CCL theory computed here and - the CAMB theory CosmoSIS computes agree to the level the CAMB↔CCL - cross-check test asserts. Adopting a different named group fiducial is a - change to these *values*, not to any code. + the CAMB theory CosmoSIS computes agree. Adopting a different named group + fiducial is a change to these *values*, not to any code. Cosmology is parametrised by the blind axes ``S8`` and ``Omega_m`` and converted to CCL's native ``sigma8``/``Omega_c`` by :meth:`sigma8` / diff --git a/src/sp_validation/custody.py b/src/sp_validation/custody.py index 8b5445ce..c498fbd1 100644 --- a/src/sp_validation/custody.py +++ b/src/sp_validation/custody.py @@ -2,12 +2,7 @@ The custody of every catalogue version (:func:`custody_of`), the stamp a SACC born under it carries, and the reader of the blind registry beside the -catalogue config (``cosmo_val/blinds//``: ``commitment.json``, the -``bases`` it covers, and ``revealed.json`` once its seed is published), which -only ``python -m sp_validation.blinding`` writes. - -Standard library only: the host Snakemake loads this file by path, and -container jobs import it, so both resolve custody from the same declaration. +catalogue config (``cosmo_val/blinds/CONTRACTS``). """ import hashlib diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index 7e16c2b4..61a5282d 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -3,12 +3,11 @@ A consumer of the two ξ± parts plus one covariance file — nothing here touches a catalogue. The modes come from the reporting and integration parts through the pipeline kernel; the covariance is Monte Carlo through that same kernel, -drawn from the CosmoCov integration-grid ξ± covariance around a theory mean, so -it depends on the covariance model and the grids rather than on the measured -vector. A jackknife of the transformed modes would need per-patch realisations, -which are never persisted. The Monte Carlo draws are seeded, so the -covariance is a function of the covariance model and the grids alone. The -pure-E/B part is a derivation of the two ξ± parts and carries their custody. +seeded draws from the CosmoCov integration-grid ξ± covariance around a theory +mean, so it is a function of the covariance model and the grids alone, never +of the measured vector. A jackknife of the transformed modes would need +per-patch realisations, which are never persisted. The pure-E/B part is a +derivation of the two ξ± parts and carries their custody. """ import numpy as np diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index 54731d56..4b16fd59 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -4,9 +4,7 @@ catalogue's custody. """ -from pathlib import Path - -from cv_runner import _unbuffer_streams +from cv_runner import _unbuffer_streams, verify_outputs from sp_validation.cosmo_val import CosmologyValidation from sp_validation.custody import confirm @@ -25,11 +23,4 @@ ) confirm(cv.custody(params["ver"]), params["custody"]) cv.calculate_rho_tau_stats() - -outputs = snakemake.output # noqa: F821 -for label in ("rho_stats", "tau_stats", "rho_tau"): - target = Path(outputs[label]) - if not target.exists(): - raise FileNotFoundError( - f"Expected {label} file not found after CosmologyValidation run: {target}" - ) +verify_outputs(snakemake) # noqa: F821 From 12ed1820c3ec8acf1a9ea6836c50e47d9e841a49 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 19:45:06 +0200 Subject: [PATCH 090/160] workflow: resolve_container checks the image it returns The host/image parity check had a home beside the resolver, so configure() resolved the image once only to check it, and every Snakefile that picks its own image (image_sims.smk, the smoke Snakefile) had to remember a second line. resolve_container now runs check_host_parity on the image it returns: the image checked is the image that runs, and the extra call sites go. check_host_parity stays cached, since composed Snakefiles evaluate container: more than once. Removing the check from resolve_container turns 5 host tests red (both parity cases, the unreadable-image line, the image under ~/.cache, image sims). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- workflow/common.py | 11 +++++------ workflow/rules/image_sims.smk | 1 - workflow/tests/data/container_smoke/Snakefile | 11 ++++------- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/workflow/common.py b/workflow/common.py index 474cdf39..2dbb2776 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -122,16 +122,16 @@ def inject_checkout_pythonpath(workflow_config): def resolve_container(override=None): - """Return the image every rule should run in. + """Return the image every rule should run in, stopping the launch on a mismatch. ``override`` wins if set (a ``docker://`` tag, a ``.sif`` path or a sandbox directory -- Snakemake's ``container:`` accepts all three); otherwise ``resolve_image()``, so jobs run what interactive ``spv-container`` work - runs. + runs. The image returned has passed ``check_host_parity``. """ - if override: - return str(override) - return resolve_image()[0] + image = str(override) if override else resolve_image()[0] + check_host_parity(image) + return image def warn_if_image_stale(): @@ -215,7 +215,6 @@ def configure(workflow_config): inject_checkout_pythonpath(workflow_config) warn_if_image_stale() - check_host_parity(resolve_container(workflow_config.get("container"))) CATALOG_CONFIG = load_configfile(CAT_CONFIG) FIDUCIAL = workflow_config["fiducial"] DEFAULT_MASK_SUFFIX = ( diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index 56097e6a..cc992391 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -98,7 +98,6 @@ if _missing_structural: # stack). Binds come from the driving profile's ``apptainer-args``. A null # ``sif`` resolves to the workflow's one image (see workflow/image_sims/config.yaml). SIF = common.resolve_container(IMSIM["sif"]) -common.check_host_parity(SIF) # --- repositories (bound into the image; branch code overrides) ----------- SHAPEPIPE_REPO = IMSIM["shapepipe_repo"] diff --git a/workflow/tests/data/container_smoke/Snakefile b/workflow/tests/data/container_smoke/Snakefile index 3dce83fd..1e9be8b7 100644 --- a/workflow/tests/data/container_smoke/Snakefile +++ b/workflow/tests/data/container_smoke/Snakefile @@ -1,9 +1,9 @@ # Standalone workflow exercised by workflow/tests/test_container_smoke.py. # # It composes workflow/ as every entry Snakefile does: common's launch-time -# setup, the image a launch resolves, the host/image parity check. The executor, -# the apptainer deployment and the binds come from the driving profile -# (workflow/profiles/candide). See container_smoke.py for what the job checks. +# setup, the image a launch resolves. The executor, the apptainer deployment and +# the binds come from the driving profile (workflow/profiles/candide). See +# container_smoke.py for what the job checks. import os import sys @@ -11,11 +11,8 @@ import sys sys.path.insert(0, os.path.realpath(os.path.join(str(workflow.basedir), "../../.."))) import common -IMAGE = common.resolve_container(config.get("container")) -common.check_host_parity(IMAGE) - -container: IMAGE +container: common.resolve_container(config.get("container")) rule container_smoke: From e9637f888002ade713c3f287d4258cb89adc93c4 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 19:45:14 +0200 Subject: [PATCH 091/160] workflow/tests: P5 checks that the job imports the launched checkout's src The smoke Snakefile imported common but never put the checkout's src/ on the job's PYTHONPATH, so its job imported the image's baked sp_validation, and P5's assertion (any editable src/ layout) passed on it. The smoke Snakefile now calls inject_checkout_pythonpath as configure() does for the entry Snakefiles, and P5 asserts the job's sp_validation is this checkout's src/sp_validation/__init__.py. Run on n08 through the default profile (apptainer, no SLURM): the job reports /src/sp_validation/__init__.py; with the injection removed it reports /sp_validation/src/sp_validation/__init__.py, which the new assertion rejects and the old one accepted. P5 itself needs a SLURM submit host and has not been run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- workflow/tests/data/container_smoke/Snakefile | 9 ++++++--- .../tests/data/container_smoke/container_smoke.py | 11 +++++------ workflow/tests/test_container_smoke.py | 10 +++------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/workflow/tests/data/container_smoke/Snakefile b/workflow/tests/data/container_smoke/Snakefile index 1e9be8b7..62f4754b 100644 --- a/workflow/tests/data/container_smoke/Snakefile +++ b/workflow/tests/data/container_smoke/Snakefile @@ -1,9 +1,10 @@ # Standalone workflow exercised by workflow/tests/test_container_smoke.py. # # It composes workflow/ as every entry Snakefile does: common's launch-time -# setup, the image a launch resolves. The executor, the apptainer deployment and -# the binds come from the driving profile (workflow/profiles/candide). See -# container_smoke.py for what the job checks. +# setup, this checkout's src/ on the job's PYTHONPATH, the image a launch +# resolves. The executor, the apptainer deployment and the binds come from the +# driving profile (workflow/profiles/candide). See container_smoke.py for what +# the job checks. import os import sys @@ -11,6 +12,8 @@ import sys sys.path.insert(0, os.path.realpath(os.path.join(str(workflow.basedir), "../../.."))) import common +common.inject_checkout_pythonpath(config) + container: common.resolve_container(config.get("container")) diff --git a/workflow/tests/data/container_smoke/container_smoke.py b/workflow/tests/data/container_smoke/container_smoke.py index c01eac72..effc0b0b 100644 --- a/workflow/tests/data/container_smoke/container_smoke.py +++ b/workflow/tests/data/container_smoke/container_smoke.py @@ -7,8 +7,8 @@ * the job really ran inside the image (``APPTAINER_CONTAINER``, set by apptainer itself -- without it the rest could all pass on the bare host); - * the editable ``sp_validation`` install resolves on the container's - PYTHONPATH (import provenance: file + version, not just import success); + * which ``sp_validation`` the job imports (file + version, not just import + success); * the numeric stack works (numpy eigh on a small fixed matrix). ``OMP_NUM_THREADS`` is recorded but not asserted -- see the assertions; * which commit of this checkout is running (git rev-parse from inside the @@ -30,7 +30,7 @@ "apptainer_container": os.environ.get("APPTAINER_CONTAINER", "unset"), } -# --- editable install resolves inside the container ------------------------ +# --- the sp_validation the job imports ------------------------------------- import sp_validation # noqa: E402 sp_validation_info = { @@ -51,9 +51,8 @@ } # --- provenance: what commit is actually running in the container --------- -# workflow/tests/data/container_smoke/ -> repo root, four levels up. -# (This is the checkout the Snakefile came from, which is what we want to -# report; the editable install may well resolve to a *different* checkout.) +# workflow/tests/data/container_smoke/ -> repo root, four levels up: the +# checkout the Snakefile came from. repo_dir = os.path.abspath( os.path.join(os.path.dirname(os.path.abspath(__file__)), *([os.pardir] * 4)) ) diff --git a/workflow/tests/test_container_smoke.py b/workflow/tests/test_container_smoke.py index 1482c0dc..8045103d 100644 --- a/workflow/tests/test_container_smoke.py +++ b/workflow/tests/test_container_smoke.py @@ -77,15 +77,11 @@ def test_container_smoke(): # something: apptainer sets APPTAINER_CONTAINER in every process it starts. assert report["container"]["apptainer_container"] != "unset", report["container"] - # The install must resolve to an editable src/ checkout, not a site-packages - # copy. It need not be *this* checkout: the container's editable install - # points at the shared /n17data working tree, while the Snakefile under test - # is read from wherever the test runs. - module_file = Path(report["sp_validation"]["file"]) - assert module_file.parts[-3:] == ("src", "sp_validation", "__init__.py"), ( + # The job imports the launched checkout's sp_validation, not the image's. + module_file = Path(report["sp_validation"]["file"]).resolve() + assert module_file == (REPO / "src/sp_validation/__init__.py").resolve(), ( module_file ) - assert "site-packages" not in module_file.parts, module_file # The numeric stack agrees with the same computation run here. np.testing.assert_allclose( From bda60dae8c7f55db3e68467127301eee9d621962 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 19:47:13 +0200 Subject: [PATCH 092/160] workflow: COSMO_VAL defaults to the launched checkout's own output tree A launch from any checkout that named no COSMO_VAL read that checkout's catalogue config and code but wrote every cosmo_val product into the production tree, silently. COSMO_VAL now defaults to the launched checkout's (gitignored) cosmo_val/output, so production is written only from the production checkout or when a launch names it; COSMO_INFERENCE keeps its shared default. README says so. P2 gains an unnamed case: with COSMO_VAL unset, every declared output lies under the toy checkout's cosmo_val/output, the inference root or results/. Restoring the production default turns it red (outputs under /n17data/cdaley/unions/code/sp_validation/cosmo_val/output). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- workflow/README.md | 9 ++++++--- workflow/common.py | 11 +++++------ workflow/tests/test_dag.py | 16 +++++++++++----- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index e5671848..d60ad6b3 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -100,9 +100,12 @@ halves of one commit, split. The catalogue config is the launched checkout's too: `cosmo_val/cat_config.yaml`, read by the host and handed to every job. The `papers/cosmo_val` suite writes -only under `COSMO_VAL` or `COSMO_INFERENCE` (environment variables, defaulting -to the shared trees on candide) or the run directory's `results/`. Other rules -write elsewhere: masks under the run directory's `output/masks/`, +only under `COSMO_VAL` or `COSMO_INFERENCE` or the run directory's `results/`. +Both are environment variables: `COSMO_VAL` defaults to the launched +checkout's own `cosmo_val/output`, so writing into another checkout's products +means naming its tree (`COSMO_VAL=/cosmo_val/output`); +`COSMO_INFERENCE` defaults to the shared tree on candide. Other rules write +elsewhere: masks under the run directory's `output/masks/`, `papers/bmodes`' figures and macros under its run directory's `docs/`, the image sims under their `grids_base`. diff --git a/workflow/common.py b/workflow/common.py index 2dbb2776..b4844efe 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -41,12 +41,11 @@ # Output roots are env-overridable so a reproduction run can write into a -# fresh tree without clobbering (or silently reusing) prior products. -COSMO_VAL = Path( - os.environ.get( - "COSMO_VAL", "/n17data/cdaley/unions/code/sp_validation/cosmo_val/output" - ) -) +# fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL +# defaults to the launched checkout's own (gitignored) cosmo_val/output, so a +# launch writes into another checkout's products only when it names that tree; +# COSMO_INFERENCE defaults to the shared tree on candide. +COSMO_VAL = Path(os.environ.get("COSMO_VAL", REPO_ROOT / "cosmo_val" / "output")) COSMO_INFERENCE = Path( os.environ.get( "COSMO_INFERENCE", "/n17data/cdaley/unions/code/sp_validation/cosmo_inference" diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index ec3f7559..bbd21622 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -69,13 +69,19 @@ def test_one_integration_grid(toy): assert {part, covariance} <= by_rule["cv_pure_eb"], by_rule["cv_pure_eb"] -def test_outputs_stay_in_the_output_roots(toy): - """Nothing the suite declares lands outside the configured output roots.""" - result = toy.snakemake("-n", "all") +@pytest.mark.parametrize("named", [True, False], ids=["named", "unnamed"]) +def test_outputs_stay_in_the_output_roots(toy, named): + """Nothing the suite declares lands outside the configured output roots. + + A launch that names no COSMO_VAL writes into its own checkout's + cosmo_val/output. + """ + env = toy.env if named else {k: v for k, v in toy.env.items() if k != "COSMO_VAL"} + cosmo_val = toy.cosmo_val if named else toy.root / "cosmo_val" / "output" + result = toy.snakemake("-n", "all", env=env) assert result.returncode == 0, result.stdout roots = [ - r.resolve() - for r in (toy.cosmo_val, toy.cosmo_inference, toy.rundir / "results") + r.resolve() for r in (cosmo_val, toy.cosmo_inference, toy.rundir / "results") ] outputs = [Path(o) for j in parse_jobs(result.stdout) for o in j.output] assert outputs, result.stdout From 7867fe0ba495d409691b9832ac6d1261cfe1e0ec Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 19:49:02 +0200 Subject: [PATCH 093/160] =?UTF-8?q?calculate=5F2pcf:=20the=20=CE=BE=C2=B1?= =?UTF-8?q?=20text=20dump=20carries=20columns=20only,=20so=20it=20reads=20?= =?UTF-8?q?back?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With patches, calculate_2pcf wrote its .txt with write_cov=True and no per-patch results. TreeCorr 5.1.4 writes num_rows only alongside patch results, so its reader ran on into the cov block and raised ("got 12 columns instead of 11"). The two ξ± figure rules re-enter calculate_2pcf on the reporting grid (npatch=100) and hit exactly that read. The dump now carries the columns only; the covariance matrix lives in the SACC part, and the figure readers use the columns. test_a_patched_xi_dump_reads_back measures at npatch=4, then re-enters calculate_2pcf from a fresh CosmologyValidation and compares the columns; under write_cov=True it fails with the ValueError above. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/cosmo_val/real_space.py | 12 +++++------- src/sp_validation/tests/test_cosmo_val.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 1521ecc7..89dad2a9 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -94,13 +94,11 @@ def calculate_2pcf(self, ver, npatch=None, **treecorr_config): # Process the catalog & write the correlation functions gg.process(cat_gal) - # Never write_patch_results: a per-patch ξ± realisation is an - # unblinded data vector, and nothing downstream reads one — the - # covariance a consumer needs is the matrix, which the SACC part - # carries. The .txt keeps the matrix only where there are patches to - # estimate it from; at npatch=1 var_method is "shot" and it would add - # nothing over the varxip/varxim columns. - gg.write(out_fname, write_patch_results=False, write_cov=int(npatch) > 1) + # Columns only. The covariance matrix lives in the SACC part; a + # per-patch ξ± realisation is an unblinded data vector nothing reads; + # and TreeCorr cannot read back a text file carrying the matrix + # without the per-patch results. + gg.write(out_fname, write_patch_results=False, write_cov=False) # Add correlation object to class if not hasattr(self, "cat_ggs"): diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 5e248797..813cf747 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -482,6 +482,25 @@ def test_xi_part_carries_the_covariance_the_measurement_estimated( cov.diag, np.concatenate([gg.varxip, gg.varxim]) ) + def test_a_patched_xi_dump_reads_back(self, tmp_path): + """calculate_2pcf reads back the text dump a patched measurement wrote. + + The ξ± figure rules re-enter calculate_2pcf on the reporting grid, which + has patches, and are handed the dump rule xi wrote (to its precision). + """ + params, version = self._write_synthetic_catalogs(tmp_path) + binning = dict(npatch=4, min_sep=5.0, max_sep=100.0, nbins=6) + measured = CosmologyValidation(versions=[version], **params).calculate_2pcf( + version, **binning + ) + read = CosmologyValidation(versions=[version], **params).calculate_2pcf( + version, **binning + ) + for column in ("meanr", "npairs", "xip", "xim", "varxip", "varxim"): + np.testing.assert_allclose( + getattr(read, column), getattr(measured, column), rtol=1e-4 + ) + def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): """calculate_2pcf's ξ± is the same on 4 and on 48 TreeCorr threads. From 5a1047ba6686e156bb7c6e4e0c58c7acdcb4466e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 20:57:53 +0200 Subject: [PATCH 094/160] Blinding tests: each audit, reveal, init and open_blind guard is red without it Eight guards survived their removal with the whole blinding suite and the custody E2E green: the audit's covariance, live-stamp, row and archive- commitment checks, its strict tolerance, reveal's refusal of a record publishing another seed, init's refusal of a staging directory, and open_blind's binding of the blind to the custody's commitment. One case each now names the problem the guard reports; each turns red when its guard is removed (run on n08 against the merged head). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/tests/test_blinding.py | 69 ++++++++++++++++++++---- 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 6be7a15e..c7f7e44a 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -696,6 +696,9 @@ def test_init_refuses_existing_state(fresh): _init(fresh, "again", "TOY") with pytest.raises(cu.CustodyError, match="blinded first"): _init(fresh, "open", "TOY_OPEN") + (fresh / "blinds" / ".later.tmp").mkdir() + with pytest.raises(cu.CustodyError, match=r"\.later\.tmp exists"): + _init(fresh, "later", "OTHER") def test_the_record_is_read_only(fresh): @@ -776,7 +779,7 @@ def test_open_blind_verifies_the_record(fresh): @pytest.mark.parametrize( "tamper", - ["wrong_key", "config", "config_and_digest", "scheme", "name"], + ["wrong_key", "config", "config_and_digest", "scheme", "name", "custody"], ) def test_open_blind_refuses_a_tampered_record(fresh, tamper): from cryptography.fernet import Fernet @@ -804,8 +807,11 @@ def edit(r): lambda r: r.update(blind="toy2"), ) (fresh / "blinds" / "toy2" / "bases").write_text("TOY\n") + custody = _custody(fresh, "TOY") + if tamper == "custody": + custody = dataclasses.replace(custody, commitment="0" * 64) with pytest.raises(cu.CustodyError): - bd.open_blind(_custody(fresh, "TOY")) + bd.open_blind(custody) def _init_storing(root, edit): @@ -933,15 +939,17 @@ def _write(path, s, stamp): s.save_fits(str(path), overwrite=True) -def _audit(root, archived, live, *, centres=("a", "a"), seed=None): - """Publish toy's seed (or ``seed``), archive ``archived`` under toy's stamp - with ``live`` as its re-measured twin, and audit the pair.""" +def _audit(root, archived, live, *, centres=("a", "a"), seed=None, stamps=None): + """Publish toy's seed (or ``seed``), archive ``archived`` with ``live`` as its + re-measured twin, stamped as ``stamps`` (toy's and TOY unblinded), and audit + the pair.""" blinded = _custody(root, "TOY") seed = seed or bd.open_blind(blinded).seed (root / "blinds" / "toy" / "revealed.json").write_text(json.dumps({"seed": seed})) + stamps = stamps or (blinded.stamp, cu.Custody("unblinded", "TOY").stamp) for tree, s, stamp, centre in ( - ("archive", archived, blinded.stamp, centres[0]), - ("live", live, cu.Custody("unblinded", "TOY").stamp, centres[1]), + ("archive", archived, stamps[0], centres[0]), + ("live", live, stamps[1], centres[1]), ): s = s.copy() s.metadata["patch_centers_sha256"] = centre @@ -982,6 +990,18 @@ def test_reveal_refuses_a_root_holding_nothing_concealed(fresh): bd.reveal("toy", root=fresh / "empty", cat_config=cat_config) +def test_reveal_refuses_a_record_publishing_another_seed(fresh): + root = fresh / "output" + part = root / "part.sacc" + part.parent.mkdir() + sio.save(two_bin_sacc(cl=False), part, custody=_custody(fresh, "TOY")) + revealed = fresh / "blinds" / "toy" / "revealed.json" + revealed.write_text(json.dumps({"seed": "0" * 32})) + with pytest.raises(cu.CustodyError, match="another seed"): + bd.reveal("toy", root=root, cat_config=fresh / "cat_config.yaml") + assert part.exists() + + def test_the_audit_of_an_empty_archive_says_so(fresh): (fresh / "blinds" / "toy" / "revealed.json").write_text( json.dumps({"seed": bd.open_blind(_custody(fresh, "TOY")).seed}) @@ -1019,9 +1039,40 @@ def test_the_audit_fails_on_other_patch_centres(fresh): assert not report["ok"] and "patch centres" in _problems(report)[0] -@pytest.mark.parametrize("times", [0, 2], ids=["left_true", "shifted_twice"]) +SPOILT_PAIRS = { + "covariance": "covariances differ", + "rows": "rows, tags or tracers differ", + "live_mock": "the live file is stamped mock:TOY", + "live_other_catalogue": "the live file is stamped unblinded:OTHER", + "archive_other_blind": "not concealed under this blind", +} + + +@pytest.mark.parametrize("spoilt", SPOILT_PAIRS) +def test_the_audit_names_a_pair_that_is_not_concealed_and_true(fresh, spoilt): + """An archived part is audited only under this blind, against the same + catalogue re-measured unblinded with the same rows and covariance.""" + s = two_bin_sacc() + live = two_bin_sacc(cl=False) if spoilt == "rows" else s.copy() + if spoilt == "covariance": + live.add_covariance(1.01 * np.asarray(s.covariance.dense), overwrite=True) + archived_stamp = _custody(fresh, "TOY").stamp + if spoilt == "archive_other_blind": + archived_stamp = {**archived_stamp, "blinding_commitment": "0" * 64} + live_custody = { + "live_mock": cu.Custody("mock", "TOY"), + "live_other_catalogue": cu.Custody("unblinded", "OTHER"), + }.get(spoilt, cu.Custody("unblinded", "TOY")) + stamps = (archived_stamp, live_custody.stamp) + report = _audit(fresh, _concealed(fresh, s), live, stamps=stamps) + assert not report["ok"] and _problems(report) == [SPOILT_PAIRS[spoilt]] + + +@pytest.mark.parametrize( + "times", [0, 2, 1 + 1e-4], ids=["left_true", "shifted_twice", "off_by_1e-4"] +) def test_the_audit_fails_on_a_mixed_file(fresh, times): - """One ξ± pair under the blinded stamp carries 0× or 2× the shift.""" + """One ξ± pair under the blinded stamp carries 0×, 2× or 1.0001× the shift.""" s = two_bin_sacc(rho=True) archived = _concealed(fresh, s) for i in _rows(s, sio.XI_PLUS, sio.XI_MINUS): From aed84f5a5548b478b16d9d2b1048d7172fa7e0d0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 21:29:27 +0200 Subject: [PATCH 095/160] Blinding tests: inputs under two blinds or two catalogues are refused The two-stamp test mixed two statuses only, so a derivation guard that compared statuses alone stayed green while it accepted parts concealed under two blinds, or from two unblinded catalogues. The test now mixes both, and also as an assembly declared under the first part's custody. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/tests/test_blinding.py | 25 +++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index c7f7e44a..9dd46df9 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -37,6 +37,7 @@ def _cat_config(root): "TOY": {}, "OTHER": {}, "TOY_OPEN": {"blinding": "unblinded"}, + "OTHER_OPEN": {"blinding": "unblinded"}, "TOY_MOCK": {"blinding": "mock"}, "paths": {"output": str(root / "output")}, } @@ -74,7 +75,7 @@ def _declare(root, **entries): @pytest.fixture(scope="module") def blinds(tmp_path_factory): - """Blinds `toy` (TOY) and `other` (OTHER), and the four custodies. + """Blinds `toy` (TOY) and `other` (OTHER), and the five custodies. Their seeds are fixed, so every run tests the same hidden points. """ @@ -84,7 +85,8 @@ def blinds(tmp_path_factory): for blind, base in (("toy", "TOY"), ("other", "OTHER")): m.setattr(bd.secrets, "token_hex", lambda n, b=blind: f"{b}-seed") _init(root, blind, base) - return {v: _custody(root, v) for v in ("TOY", "OTHER", "TOY_OPEN", "TOY_MOCK")} + versions = ("TOY", "OTHER", "TOY_OPEN", "OTHER_OPEN", "TOY_MOCK") + return {v: _custody(root, v) for v in versions} @pytest.fixture @@ -540,11 +542,24 @@ def test_a_derivation_inherits_its_inputs_stamp(blinds, tmp_path): assert cu.read_stamp(written.metadata).stamp == blinds["TOY"].stamp -def test_inputs_under_two_stamps_are_refused(blinds, tmp_path): - a = sio.seal(two_bin_sacc(cl=False), blinds["TOY_OPEN"]) - b = sio.seal(two_bin_sacc(cl=False), blinds["TOY_MOCK"]) +@pytest.mark.parametrize( + "first, second", + [ + ("TOY_OPEN", "TOY_MOCK"), # two statuses + ("TOY", "OTHER"), # two blinds + ("TOY_OPEN", "OTHER_OPEN"), # two unblinded catalogues + ], +) +def test_inputs_under_two_stamps_are_refused(blinds, tmp_path, first, second): + a = sio.seal(two_bin_sacc(cl=False), blinds[first]) + b = sio.seal(two_bin_sacc(cl=False), blinds[second]) with pytest.raises(ValueError, match="stamps"): sio.save(cosebis_sacc(), tmp_path / "c.sacc", derived_from=[a, b]) + with pytest.raises(ValueError, match="stamps"): + sio.save( + a.copy(), tmp_path / "a.sacc", derived_from=[a, b], custody=blinds[first] + ) + assert not list(tmp_path.iterdir()) def test_plaintext_cannot_be_laundered_under_a_concealed_stamp(blinds, tmp_path): From 56d08dc5eb5d573cc66bc5e43aaef4ed21e2693f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 21:29:27 +0200 Subject: [PATCH 096/160] custody.confirm: the job agrees with its Snakemake, not with the launch Under the slurm executor the job step re-parses the workflow, so a job's params.custody is resolved at job start, not at launch. The check compares that with the container's resolution; its message, docstring and test now say so. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/custody.py | 12 +++++++++--- src/sp_validation/tests/test_custody_e2e.py | 8 ++++---- workflow/scripts/generate_pseudo_cl.py | 4 ++-- workflow/scripts/run_2pcf.py | 4 ++-- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/sp_validation/custody.py b/src/sp_validation/custody.py index c498fbd1..af409875 100644 --- a/src/sp_validation/custody.py +++ b/src/sp_validation/custody.py @@ -311,10 +311,16 @@ def read_stamp(metadata): def confirm(custody, token): - """``custody``, if its token is the one the launch resolved; else raise.""" + """``custody``, if its token is the job's ``params.custody``; else raise. + + ``token`` is resolved by the Snakemake process that runs the job: the + launch under a local executor, the job step at job start under slurm. The + job's own resolution, in the container, must agree with it. + """ if custody.token != token: raise CustodyError( - f"custody of {custody.catalogue} changed since the launch: the job " - f"resolves {custody.token}, the launch resolved {token}" + f"custody of {custody.catalogue} differs between the job and its " + f"Snakemake: the job resolves {custody.token}, Snakemake resolved " + f"{token}; launch again" ) return custody diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py index 66a89750..656434aa 100644 --- a/src/sp_validation/tests/test_custody_e2e.py +++ b/src/sp_validation/tests/test_custody_e2e.py @@ -475,9 +475,9 @@ def refuse(custody): @pytest.mark.parametrize( "script", ["run_2pcf.py", "run_rho_tau.py", "assemble_sacc.py"] ) -def test_a_job_refuses_a_custody_changed_since_the_launch(toy, script): - """TOY's launch resolved it unblinded; declared blinded by the time the job - runs, the job refuses before it measures or writes anything.""" +def test_a_job_refuses_a_custody_other_than_its_params(toy, script): + """Snakemake resolved TOY unblinded; declared blinded when the job resolves + it, the job refuses before it measures or writes anything.""" out = toy.root / "stale" outputs = { "sacc": out / "part.sacc", @@ -486,7 +486,7 @@ def test_a_job_refuses_a_custody_changed_since_the_launch(toy, script): "rho_tau": out / "rho_tau.sacc", } out.mkdir() - with pytest.raises(cu.CustodyError, match="changed since the launch"): + with pytest.raises(cu.CustodyError, match="differs between the job"): run_rule( script, output={k: str(v) for k, v in outputs.items()}, diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index c94f5640..a190695a 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -68,8 +68,8 @@ def generate_pseudo_cl( power : float Power for powspace binning (0.5 = sqrt spacing) custody : str, optional - The custody token the launch resolved for ``version``; the part is not - written under any other. + The custody token Snakemake resolved for ``version`` (the rule's + ``params.custody``); the part is not written under any other. Returns ------- diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 6dc074af..0fa59c46 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -58,8 +58,8 @@ def run_2pcf( expects. ``sacc_out`` is the exact destination for the SACC part (the Snakemake-declared output); it defaults to a binning-derived name under the resolved output directory for the CLI path. ``custody`` is the custody - token the launch resolved for ``ver``; the part is not written under any - other. + token Snakemake resolved for ``ver`` (the rule's ``params.custody``); the + part is not written under any other. Returns ------- From eee0c3deb5a44a43abc45a6f62ec45c8b1e29e5d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 21:29:28 +0200 Subject: [PATCH 097/160] workflow: output roots are resolved; P4 dry-runs name no output tree Snakemake keys its persistence records by path string, so /n17data/... and /automnt/n17data/... spellings of one tree kept separate records and a switch between them silenced the params and code triggers. COSMO_VAL and COSMO_INFERENCE are resolved once. The candide dry-runs resolve against the checkout's own tree, as a launch naming none does. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- workflow/common.py | 11 ++++++++--- workflow/tests/test_dag.py | 4 ---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/workflow/common.py b/workflow/common.py index e8de34e6..65bffded 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -58,13 +58,18 @@ def _load_checkout_module(name): # fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL # defaults to the launched checkout's own (gitignored) cosmo_val/output, so a # launch writes into another checkout's products only when it names that tree; -# COSMO_INFERENCE defaults to the shared tree on candide. -COSMO_VAL = Path(os.environ.get("COSMO_VAL", REPO_ROOT / "cosmo_val" / "output")) +# COSMO_INFERENCE defaults to the shared tree on candide. Both are resolved: +# Snakemake keys its persistence records (the params and code triggers) by path +# string, so every spelling of one tree (/n17data -> /automnt/n17data) must +# declare the same paths. +COSMO_VAL = Path( + os.environ.get("COSMO_VAL", REPO_ROOT / "cosmo_val" / "output") +).resolve() COSMO_INFERENCE = Path( os.environ.get( "COSMO_INFERENCE", "/n17data/cdaley/unions/code/sp_validation/cosmo_inference" ) -) +).resolve() # The catalogue config of the launched checkout: the one file both the host # (CATALOG_CONFIG, loaded in configure) and every job read catalogues from, and # the blind registry beside it. diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 7aef6bb4..b30bcf16 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -295,10 +295,6 @@ def test_unblinding_a_concealed_catalogue_needs_the_reveal(toy): def _real_dry_run(paper, targets): env = {k: v for k, v in os.environ.items() if k != "SNAKEMAKE_PROFILE"} - # Against the production output tree, whichever checkout runs the tests. - env.setdefault( - "COSMO_VAL", "/n17data/cdaley/unions/code/sp_validation/cosmo_val/output" - ) env.update(PYTHONUNBUFFERED="1", PYTHONNOUSERSITE="1") return subprocess.run( [ From 2523f8c97b69d5c17061cf3d0114e9abea0cfc41 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 21:29:28 +0200 Subject: [PATCH 098/160] Blinding: drop the audit's tracer-name check and xi_ccl's unused ell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _rows_match already requires the same tracers on every signal row, and a different n(z) under one name shows in the residual. The ξ± ℓ grid is a module constant, XI_ELL. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/blinding.py | 2 -- src/sp_validation/blinding_theory.py | 31 ++++++++++++---------------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index bae37e9e..7a8dce02 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -600,8 +600,6 @@ def _audit_part(blinded, true, fiducial, hidden, tolerance): if blinded is None or true is None or not _rows_match(blinded, true): return ["rows, tags or tracers differ"], None problems = [] - if set(blinded.tracers) != set(true.tracers): - problems.append("tracers differ") cov = _covariance(true) if not _same_covariance(_covariance(blinded), cov): problems.append("covariances differ") diff --git a/src/sp_validation/blinding_theory.py b/src/sp_validation/blinding_theory.py index 24be52c8..3c768035 100644 --- a/src/sp_validation/blinding_theory.py +++ b/src/sp_validation/blinding_theory.py @@ -169,17 +169,13 @@ def ccl_cosmology(params, config): return _COSMO_CACHE[key] -def xi_ell_grid(): - """The ℓ grid the ξ± Hankel projection integrates over. - - Integers 2…49, then 200 log-spaced multipoles up to 6·10⁴ — dense enough - at low ℓ (where ξ± at large θ lives) and wide enough for the small-θ - tail. ``ccl.correlation`` interpolates C(ℓ) internally, so this fixes the - resolution of every ξ± this module produces. - """ - return np.unique( - np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)]).astype(float) - ) +# The ℓ grid the ξ± Hankel projection integrates over: integers 2…49, then 200 +# log-spaced multipoles up to 6·10⁴ — dense enough at low ℓ (where ξ± at large θ +# lives) and wide enough for the small-θ tail. ``ccl.correlation`` interpolates +# C(ℓ) internally, so this fixes the resolution of every ξ± this module produces. +XI_ELL = np.unique( + np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)]).astype(float) +) # Tracers are rebuilt for every pair of every block at both cosmologies of a @@ -238,11 +234,11 @@ def cl_ee(params, config, nz_i, nz_j, ell): return ccl.angular_cl(cosmo, tracer_i, tracer_j, np.asarray(ell, dtype=float)) -def xi_ccl(params, config, nz_i, nz_j, theta_arcmin, ell=None): +def xi_ccl(params, config, nz_i, nz_j, theta_arcmin): """ξ± at ``theta_arcmin`` for one bin pair. - Cross Cℓ_EE on :func:`xi_ell_grid` (or ``ell``), then ``ccl.correlation`` - (FFTLog Hankel transform) at θ in degrees, ``type="GG+"`` / ``"GG-"``. + Cross Cℓ_EE on :data:`XI_ELL`, then ``ccl.correlation`` (FFTLog Hankel + transform) at θ in degrees, ``type="GG+"`` / ``"GG-"``. Returns ------- @@ -251,10 +247,9 @@ def xi_ccl(params, config, nz_i, nz_j, theta_arcmin, ell=None): """ import pyccl as ccl - ell = xi_ell_grid() if ell is None else np.asarray(ell, dtype=float) cosmo = ccl_cosmology(params, config) - cl = cl_ee(params, config, nz_i, nz_j, ell) + cl = cl_ee(params, config, nz_i, nz_j, XI_ELL) theta_deg = np.asarray(theta_arcmin) / 60.0 - xip = ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta_deg, type="GG+") - xim = ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta_deg, type="GG-") + xip = ccl.correlation(cosmo, ell=XI_ELL, C_ell=cl, theta=theta_deg, type="GG+") + xim = ccl.correlation(cosmo, ell=XI_ELL, C_ell=cl, theta=theta_deg, type="GG-") return xip, xim From 25229a7fa0b98d4db1e01e12c739054102b99c16 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 21:31:32 +0200 Subject: [PATCH 099/160] workflow/tests: a symlinked COSMO_VAL declares the resolved paths Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- workflow/tests/test_dag.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index b30bcf16..c02ef7c3 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -102,6 +102,18 @@ def test_outputs_stay_in_the_output_roots(toy, named): assert not strays, strays +def test_every_spelling_of_an_output_root_declares_the_same_paths(toy, tmp_path): + """Snakemake keys its params and code triggers by path string, so a + symlinked spelling of COSMO_VAL declares the tree's resolved paths.""" + link = tmp_path / "cosmo_val_link" + link.symlink_to(toy.cosmo_val, target_is_directory=True) + result = toy.snakemake("-n", "all", env={**toy.env, "COSMO_VAL": str(link)}) + assert result.returncode == 0, result.stdout + outputs = [o for j in parse_jobs(result.stdout) for o in j.output] + assert any(o.startswith(str(toy.cosmo_val.resolve())) for o in outputs), outputs + assert not [o for o in outputs if o.startswith(str(link))], outputs + + @pytest.mark.parametrize( "python, snakemake_version", [("3.13.1", None), (None, "9.0.0")], From cad4786e9e7b9bc819f23badd9e0d80ad917c442 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 22:07:13 +0200 Subject: [PATCH 100/160] workflow: the checkout and output roots take the plain /nXXdataN spelling Resolving COSMO_VAL and COSMO_INFERENCE spelled them /automnt/nXXdataN, so a file target named in the plain form people type (as the E2E run plan names them) found no rule, and the /automnt paths handed to jobs do not exist on the node that owns the disk. common._plain resolves a path and spells it back under /; REPO_ROOT and both roots go through it, so the host and every node declare one path per file. P2 checks both roots under a symlinked spelling, and on candide that /automnt-spelled roots declare plain paths a target can name. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- workflow/README.md | 7 ++++-- workflow/common.py | 36 ++++++++++++++++++++-------- workflow/tests/test_dag.py | 48 +++++++++++++++++++++++++++++++------- 3 files changed, 70 insertions(+), 21 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index 35bfa834..820d2128 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -180,8 +180,11 @@ directory. `/automnt/nXXdataN` works only from a node that does *not* own that disk. On the owning node the disk is mounted directly at `/nXXdataN` and there is no `/automnt/nXXdataN` entry at all, so a job that lands there dies about one second after the allocation starts, before any log file is written. This is why -`n17` is in the profile's exclude list. Every canonical path in `common.py` -already uses the plain form; keep new paths the same. +`n17` is in the profile's exclude list. `common.py` spells the launched +checkout, `COSMO_VAL` and `COSMO_INFERENCE` in the plain form whatever spelling +it is given (a symlink, a relative path, `/automnt`), and Snakemake matches a +target by its path string, so name file targets in the plain form too; keep new +paths the same. ### Run Snakemake from the host, never from inside the container diff --git a/workflow/common.py b/workflow/common.py index 65bffded..a6879362 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -12,8 +12,26 @@ import sys from pathlib import Path + +def _plain(path): + """``path`` resolved, in the spelling every candide node can reach. + + A data disk is mounted at ``/nXXdataN`` on the node that owns it and + reached from every other node through ``/nXXdataN -> /automnt/nXXdataN``; + the owning node has no ``/automnt/nXXdataN``. A resolved path under + ``/automnt/`` is therefore spelled back under ``/`` when that + link exists here. + """ + path = Path(path).resolve() + if len(path.parts) > 2 and path.parts[1] == "automnt": + disk = Path("/", path.parts[2]) + if disk.resolve() == Path(*path.parts[:3]): + return disk.joinpath(*path.parts[3:]) + return path + + # The checkout this workflow was launched from: workflow/common.py -> . -REPO_ROOT = Path(__file__).resolve().parents[1] +REPO_ROOT = _plain(__file__).parents[1] REPO_SRC = REPO_ROOT / "src" @@ -58,18 +76,16 @@ def _load_checkout_module(name): # fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL # defaults to the launched checkout's own (gitignored) cosmo_val/output, so a # launch writes into another checkout's products only when it names that tree; -# COSMO_INFERENCE defaults to the shared tree on candide. Both are resolved: -# Snakemake keys its persistence records (the params and code triggers) by path -# string, so every spelling of one tree (/n17data -> /automnt/n17data) must -# declare the same paths. -COSMO_VAL = Path( - os.environ.get("COSMO_VAL", REPO_ROOT / "cosmo_val" / "output") -).resolve() -COSMO_INFERENCE = Path( +# COSMO_INFERENCE defaults to the shared tree on candide. Snakemake keys its +# persistence records (params, input, code) and matches targets by path string, +# so both roots are spelled plain whatever spelling a launch gives, and a file +# target is named in that plain form. +COSMO_VAL = _plain(os.environ.get("COSMO_VAL", REPO_ROOT / "cosmo_val" / "output")) +COSMO_INFERENCE = _plain( os.environ.get( "COSMO_INFERENCE", "/n17data/cdaley/unions/code/sp_validation/cosmo_inference" ) -).resolve() +) # The catalogue config of the launched checkout: the one file both the host # (CATALOG_CONFIG, loaded in configure) and every job read catalogues from, and # the blind registry beside it. diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index c02ef7c3..4c7bb524 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -13,6 +13,7 @@ STALE, UNCOVERED, VERSIONS, + _load_module, fake_image, on_candide, parse_jobs, @@ -102,16 +103,45 @@ def test_outputs_stay_in_the_output_roots(toy, named): assert not strays, strays -def test_every_spelling_of_an_output_root_declares_the_same_paths(toy, tmp_path): - """Snakemake keys its params and code triggers by path string, so a - symlinked spelling of COSMO_VAL declares the tree's resolved paths.""" - link = tmp_path / "cosmo_val_link" - link.symlink_to(toy.cosmo_val, target_is_directory=True) - result = toy.snakemake("-n", "all", env={**toy.env, "COSMO_VAL": str(link)}) +@pytest.mark.parametrize("root", ["COSMO_VAL", "COSMO_INFERENCE"]) +def test_every_spelling_of_an_output_root_declares_the_same_paths(toy, tmp_path, root): + """Snakemake keys its persistence records by path string, so a symlinked + spelling of an output root declares the tree's resolved paths.""" + tree = Path(toy.env[root]).resolve() + link = tmp_path / "link" + link.symlink_to(tree, target_is_directory=True) + result = toy.snakemake("-n", "all", env={**toy.env, root: str(link)}) assert result.returncode == 0, result.stdout - outputs = [o for j in parse_jobs(result.stdout) for o in j.output] - assert any(o.startswith(str(toy.cosmo_val.resolve())) for o in outputs), outputs - assert not [o for o in outputs if o.startswith(str(link))], outputs + declared = [f for j in parse_jobs(result.stdout) for f in j.input + j.output] + assert any(f.startswith(f"{tree}/") for f in declared), declared + assert not [f for f in declared if f.startswith(f"{link}/")], declared + + +@pytest.mark.candide +@on_candide +def test_output_roots_take_the_plain_spelling(toy): + """A root on a candide disk is declared under /nXXdataN, the one spelling + every node has, however the launch spells it: a file target named there + resolves, and no declared path lies under /automnt.""" + tree = Path("/n17data/cdaley/unions/.spv-dag-toy") # a dry-run creates nothing + env = { + **toy.env, + "COSMO_VAL": f"/automnt{tree}/val", + "COSMO_INFERENCE": f"/automnt{tree}/inference", + } + common = _load_module(toy.root / "workflow" / "common.py", "plain_common", env) + assert (common.COSMO_VAL, common.COSMO_INFERENCE) == ( + tree / "val", + tree / "inference", + ) + grids = toy.common.xi_grids(toy.config, toy.config["fiducial"]) + reporting = toy.common.grid_binning(grids["reporting"]) + target = tree / "val" / f"{VERSIONS[0]}_xi_{reporting}.sacc" + result = toy.snakemake("-n", str(target), env=env) + assert result.returncode == 0, result.stdout + declared = [f for j in parse_jobs(result.stdout) for f in j.input + j.output] + assert str(target) in declared, declared + assert not [f for f in declared if f.startswith("/automnt/")], declared @pytest.mark.parametrize( From 75bac1186b1e1c8f69c4971643cd3a209580a2ba Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 22:14:19 +0200 Subject: [PATCH 101/160] Blinding: init drops its seed round-trip check Fernet decrypts with the key that encrypted, and the payload holds only str and int values, so the check could never fire. The seed's only on-disk form is ciphertext (test_init_never_writes_the_seed), and open_blind authenticates the payload whenever a blind is opened. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/blinding.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 7a8dce02..fe80e756 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -430,8 +430,6 @@ def init(name, bases, *, cat_config, config=None): "draw_scheme": draw_scheme(), } ciphertext = Fernet(key).encrypt(json.dumps(payload).encode("utf-8")) - if json.loads(Fernet(key).decrypt(ciphertext)) != payload: - raise _custody.CustodyError("the sealed seed does not round-trip") commitment = { "blind": name, "seed_commitment": _custody.seed_commitment(seed), From ee22798f37712ede21e03d223d9bf56688506a2f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 22:46:04 +0200 Subject: [PATCH 102/160] workflow: roots take the plain spelling on every host _plain spelled /automnt/ back as / only where / links to /automnt/, which holds on every node but the one owning the disk. A job step re-derives its paths from the launch's environment on its own node, so on the owning node it declared /automnt paths that node lacks. The mapping is now the same on every host. The plain-spelling test stands in a disk no host has for the owning node, so it runs everywhere, CI included. The README names the disks a job sees under their plain spelling (the ones the profile binds by name), where the checkout and both output roots have to sit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- workflow/README.md | 9 ++++++--- workflow/common.py | 9 ++++----- workflow/tests/test_dag.py | 13 +++++++------ 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index 820d2128..a17b17bc 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -165,8 +165,7 @@ To reproduce a run from the image alone, opt out: snakemake --profile workflow/profiles/candide --config checkout_pythonpath=false ``` -Either way the checkout has to sit under one of the profile's bind mounts to be -visible inside the job. +Either way the checkout has to sit on a disk the jobs see (next section). Most of the time this default is all you need. Reach for a different *image* only when the dependency stack changed — a new package, a lockfile bump — not @@ -184,7 +183,11 @@ second after the allocation starts, before any log file is written. This is why checkout, `COSMO_VAL` and `COSMO_INFERENCE` in the plain form whatever spelling it is given (a symlink, a relative path, `/automnt`), and Snakemake matches a target by its path string, so name file targets in the plain form too; keep new -paths the same. +paths the same. A job sees a plain path only on a disk the profile binds by +name — `/home`, `/n17data`, `/n23data1`, `/n09data` (the `/automnt` bind does +not serve it) — so the checkout and both output roots sit on one of those. To +work on another disk, add it to both bind lists: the candide profile's +`apptainer-args` and `container.DEFAULT_BINDS`. ### Run Snakemake from the host, never from inside the container diff --git a/workflow/common.py b/workflow/common.py index a6879362..c7a9aaba 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -19,14 +19,13 @@ def _plain(path): A data disk is mounted at ``/nXXdataN`` on the node that owns it and reached from every other node through ``/nXXdataN -> /automnt/nXXdataN``; the owning node has no ``/automnt/nXXdataN``. A resolved path under - ``/automnt/`` is therefore spelled back under ``/`` when that - link exists here. + ``/automnt/`` is therefore spelled back under ``/``, whatever + the host: a job step re-derives its paths on its own node, and must + derive the launch's. """ path = Path(path).resolve() if len(path.parts) > 2 and path.parts[1] == "automnt": - disk = Path("/", path.parts[2]) - if disk.resolve() == Path(*path.parts[:3]): - return disk.joinpath(*path.parts[3:]) + return Path("/", *path.parts[2:]) return path diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 4c7bb524..4406816e 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -117,13 +117,14 @@ def test_every_spelling_of_an_output_root_declares_the_same_paths(toy, tmp_path, assert not [f for f in declared if f.startswith(f"{link}/")], declared -@pytest.mark.candide -@on_candide def test_output_roots_take_the_plain_spelling(toy): - """A root on a candide disk is declared under /nXXdataN, the one spelling - every node has, however the launch spells it: a file target named there - resolves, and no declared path lies under /automnt.""" - tree = Path("/n17data/cdaley/unions/.spv-dag-toy") # a dry-run creates nothing + """A root given as /automnt//... is declared as //..., the one + spelling every node has, on any host: a job step re-derives the launch's + paths on its own node, and the node that owns a disk has neither + /automnt/ nor a / link to it, like the disk no host has here. + A file target named in the plain spelling resolves, and no declared path + lies under /automnt.""" + tree = Path("/n00data0/spv-dag-toy") # a dry-run creates nothing env = { **toy.env, "COSMO_VAL": f"/automnt{tree}/val", From efb32fa6d0af8e8625d118528e70c7f9969b622d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 22:46:04 +0200 Subject: [PATCH 103/160] workflow/README: a collaborator's launch names its own COSMO_INFERENCE COSMO_INFERENCE defaults to the shared tree, which only its owner can write, so anyone else's launch cannot write the CosmoCov chain there. The README says how to name a private tree: link the shared tree's data/mask/, the one input the covariance rules take from it, and the chain runs in the private tree. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- workflow/README.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index a17b17bc..08732e2d 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -150,8 +150,19 @@ only under `COSMO_VAL` or `COSMO_INFERENCE` or the run directory's `results/`. Both are environment variables: `COSMO_VAL` defaults to the launched checkout's own `cosmo_val/output`, so writing into another checkout's products means naming its tree (`COSMO_VAL=/cosmo_val/output`); -`COSMO_INFERENCE` defaults to the shared tree on candide. Other rules write -elsewhere: masks under the run directory's `output/masks/`, +`COSMO_INFERENCE` defaults to the shared tree on candide, +`/n17data/cdaley/unions/code/sp_validation/cosmo_inference`, which holds the +CosmoCov covariances and only its owner can write. Anyone else launches with +`COSMO_INFERENCE=` of their own, holding a link to the shared tree's +`data/mask/` (the one input the covariance rules take from it); the CosmoCov +chain then runs there: + +```bash +mkdir -p /data +ln -s /n17data/cdaley/unions/code/sp_validation/cosmo_inference/data/mask /data/ +``` + +Other rules write elsewhere: masks under the run directory's `output/masks/`, `papers/bmodes`' figures and macros under its run directory's `docs/`, the image sims under their `grids_base`. From 1dc8acef10ec347081da7190898a2f104555dc67 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 27 Sep 2026 22:46:05 +0200 Subject: [PATCH 104/160] custody: the registry is spelled as its catalogue config is registry_of resolved the config path, so on candide the host registry, and the init/share commands the launch prints from it, were spelled /automnt/nXXdataN, which the node owning the disk lacks. It keeps the given spelling now: the plain one on the host. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/custody.py | 4 ++-- src/sp_validation/tests/test_custody.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/custody.py b/src/sp_validation/custody.py index af409875..0bcc6369 100644 --- a/src/sp_validation/custody.py +++ b/src/sp_validation/custody.py @@ -80,8 +80,8 @@ def seed_commitment(seed): def registry_of(cat_config): - """The blind registry beside a catalogue config.""" - return Path(cat_config).resolve().parent / "blinds" + """The blind registry beside a catalogue config, spelled as the config is.""" + return Path(cat_config).absolute().parent / "blinds" def entry_of(version): diff --git a/src/sp_validation/tests/test_custody.py b/src/sp_validation/tests/test_custody.py index ec10a27a..8e1eb4c9 100644 --- a/src/sp_validation/tests/test_custody.py +++ b/src/sp_validation/tests/test_custody.py @@ -71,6 +71,21 @@ def test_undeclared_catalogue_without_a_blind_fails_with_the_command(tmp_path): assert "share" in message +def test_the_printed_commands_name_the_config_as_it_was_given(tmp_path): + """The operator pastes these commands, so they keep the launch's spelling + of the checkout: on candide the plain /nXXdataN, never the /automnt path it + resolves to.""" + (tmp_path / "real" / "cosmo_val").mkdir(parents=True) + checkout = tmp_path / "checkout" + checkout.symlink_to(tmp_path / "real", target_is_directory=True) + registry = cu.registry_of(checkout / "cosmo_val" / "cat_config.yaml") + with pytest.raises(cu.CustodyError) as err: + cu.custody_of(_catalogues(SP_v9=None), "SP_v9", registry=registry) + message = str(err.value) + assert f"APPTAINERENV_PYTHONPATH={checkout}/src " in message + assert f"--cat-config {checkout}/cosmo_val/cat_config.yaml" in message + + def test_declared_blinded_without_a_blind_fails(tmp_path): cats = _catalogues(SP_v9="blinded") with pytest.raises(cu.CustodyError, match="no blind covers it"): From 70e6397d293400c82f5fe27fb3c60457fb72c54a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 00:02:33 +0200 Subject: [PATCH 105/160] =?UTF-8?q?No=20plaintext=20=CE=BE=C2=B1:=20Cosmol?= =?UTF-8?q?ogyValidation=20returns=20sealed=20parts,=20no=20text=20dumps,?= =?UTF-8?q?=20patch=20centres=20as=20inputs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit calculate_2pcf returns the ξ± part sealed under the catalogue's custody (concealed when blinded) and keeps it in cv.xi_parts; it writes no text dump and never reads one back. The ξ± figures, the notebook COSEBIs, the Paper II scripts, the catalogue-paper ratio figure and the lc sweep read parts through sacc_io.xi_correlation, a GGCorrelation-shaped view. The jackknife calculate_pure_eb and the aperture mass work from TreeCorr's own measurement (_measure_xi) and refuse a blinded catalogue. Jackknife patch centres are drawn once per base catalogue by the new rule xi_patches into COSMO_VAL/patches/ and are an input of every patched xi job; a catalogue and its variants split alike, and each part names its centres by sha256. TreeCorr takes its thread count from treecorr_config (the CPUs the process may use), so a caller's num_threads still reaches it. Workflow jobs read workflow/matplotlibrc (MATPLOTLIBRC, set by common.py), so a user's own matplotlibrc -- a LaTeX preamble the image lacks -- no longer stops every figure rule; the image gains sfmath for sans-serif usetex figures. Tests: I14 on a blinded synthetic catalogue; I13 extended to the patch-centre path; the E2E runs xi_patches, a patched reporting grid and the ξ± figure rule, and finds no true ξ± value in any byte of the blinded run or its figures; P12, P13 and P14 (a real xi + figure run through apptainer before and after a blind is drawn, under a matplotlibrc asking for a missing LaTeX package). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- Dockerfile | 3 + papers/bmodes/rules/claims.smk | 10 +- .../bb_covariance_blind_independence.py | 9 +- .../scripts/compute_cosebis_pte_single.py | 13 +- .../scripts/cosebis_binning_comparison.py | 48 +-- papers/bmodes/scripts/cosebis_data_vector.py | 16 +- .../scripts/cosebis_version_comparison.py | 18 +- .../bmodes/scripts/gather_pure_eb_chunks.py | 18 +- .../harmonic_config_cosebis_comparison.py | 22 +- .../scripts/precompute_pure_eb_chunk.py | 19 +- .../bmodes/scripts/run_cosebis_ptes_sweep.py | 4 +- papers/bmodes/scripts/run_xi_sweep.py | 4 +- papers/catalog/2025_10_02_xi_sys_rho_tau.py | 28 +- src/sp_validation/cosmo_val/core.py | 29 +- src/sp_validation/cosmo_val/cosebis.py | 30 +- src/sp_validation/cosmo_val/pure_eb.py | 32 +- src/sp_validation/cosmo_val/real_space.py | 308 ++++++++++-------- src/sp_validation/sacc_io.py | 67 ++++ src/sp_validation/tests/_synthetic.py | 3 + src/sp_validation/tests/test_cosmo_val.py | 172 +++++++--- src/sp_validation/tests/test_custody.py | 13 +- src/sp_validation/tests/test_custody_e2e.py | 140 +++++++- workflow/README.md | 14 +- workflow/common.py | 23 +- workflow/matplotlibrc | 2 + workflow/rules/cosmo_val.smk | 44 +-- workflow/rules/twopoint.smk | 32 +- workflow/scripts/cv_plot_2pcf.py | 10 +- workflow/scripts/cv_ratio_xi_sys_xi.py | 13 +- workflow/scripts/cv_runner.py | 10 +- workflow/scripts/run_2pcf.py | 82 ++--- workflow/scripts/xi_patches.py | 19 ++ .../data/container_smoke/container_smoke.py | 9 +- workflow/tests/test_container_smoke.py | 6 + workflow/tests/test_dag.py | 41 +++ workflow/tests/test_toy_run.py | 192 +++++++++++ 36 files changed, 1032 insertions(+), 471 deletions(-) create mode 100644 workflow/matplotlibrc create mode 100644 workflow/scripts/xi_patches.py create mode 100644 workflow/tests/test_toy_run.py diff --git a/Dockerfile b/Dockerfile index 844f7e23..0437a929 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,8 @@ RUN apt-get update -y --quiet --fix-missing && \ rm -rf /var/lib/apt/lists/* # TinyTeX pinned to a TeX Live year (frozen tlnet-final mirror); bump both once a year. +# The packages serve matplotlib's usetex figures; sfmath gives them sans-serif +# maths (`\usepackage[cm]{sfmath}`). ENV TEXLIVE_YEAR=2025 \ TINYTEX_VERSION=2026.02 \ TINYTEX_DIR=/opt \ @@ -46,6 +48,7 @@ RUN set -eux; \ amsmath \ amsfonts \ geometry \ + sfmath \ xcolor; \ tlmgr path add; \ latex --version >/dev/null; dvipng --version >/dev/null diff --git a/papers/bmodes/rules/claims.smk b/papers/bmodes/rules/claims.smk index 85891b25..db60a9dc 100644 --- a/papers/bmodes/rules/claims.smk +++ b/papers/bmodes/rules/claims.smk @@ -78,18 +78,18 @@ def _reporting_cov_path(version, blind): def _xi_reporting_path(version): - """Path to reporting-scale 2PCF file.""" + """Path to the reporting-scale ξ± part.""" return ( f"{COSMO_VAL_OUTPUT}/{version}_xi_minsep={FIDUCIAL['min_sep']}" - f"_maxsep={FIDUCIAL['max_sep']}_nbins={FIDUCIAL['nbins']}_npatch={FIDUCIAL['npatch']}.txt" + f"_maxsep={FIDUCIAL['max_sep']}_nbins={FIDUCIAL['nbins']}_npatch={FIDUCIAL['npatch']}.sacc" ) def _xi_integration_path(version): - """Path to fine-binned 2PCF integration file. Unpatched: values only, no covariance.""" + """Path to the fine-binned ξ± integration part (unpatched).""" return ( f"{COSMO_VAL_OUTPUT}/{version}_xi_minsep={FIDUCIAL['min_sep_int']}" - f"_maxsep={FIDUCIAL['max_sep_int']}_nbins={FIDUCIAL['nbins_int']}_npatch=1.txt" + f"_maxsep={FIDUCIAL['max_sep_int']}_nbins={FIDUCIAL['nbins_int']}_npatch=1.sacc" ) @@ -202,7 +202,7 @@ rule cosebis_binning_comparison: xi_1k=_xi_integration_path(FIDUCIAL_VERSION), xi_10k=( f"{COSMO_VAL_OUTPUT}/{FIDUCIAL_VERSION}_xi_minsep={FIDUCIAL['min_sep_int']}" - f"_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch={FIDUCIAL['npatch']}.txt" + f"_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch={FIDUCIAL['npatch']}.sacc" ), cov_1k=_cov_integration_path(FIDUCIAL_VERSION, "A"), output: diff --git a/papers/bmodes/scripts/bb_covariance_blind_independence.py b/papers/bmodes/scripts/bb_covariance_blind_independence.py index 256a79b1..8868baab 100644 --- a/papers/bmodes/scripts/bb_covariance_blind_independence.py +++ b/papers/bmodes/scripts/bb_covariance_blind_independence.py @@ -18,12 +18,12 @@ import matplotlib.pyplot as plt import numpy as np -import treecorr import yaml from astropy.io import fits from plotting_utils import PAPER_MPLSTYLE from pseudo_cl_io import load_pseudo_cl_data +from sp_validation import sacc_io from sp_validation.b_modes import calculate_cosebis plt.style.use(PAPER_MPLSTYLE) @@ -83,10 +83,7 @@ def load_cosebis_diagonals( Returns E_n and B_n covariance diagonals. """ # Load fine-binned 2PCF (need the binning info for COSEBIS calculation) - gg = treecorr.GGCorrelation( - min_sep=min_sep_int, max_sep=max_sep_int, nbins=nbins_int, sep_units="arcmin" - ) - gg.read(xi_integration_path) + gg = sacc_io.xi_correlation(sacc_io.load(xi_integration_path)) # Compute COSEBIS with this blind's covariance results = calculate_cosebis( @@ -711,7 +708,7 @@ def _from_cli(argv=None): xi_integration_path = os.path.join( a.cosmo_val_dir, f"{version}_xi_minsep={min_sep_int}_maxsep={max_sep_int}" - f"_nbins={nbins_int}_npatch={npatch}.txt", + f"_nbins={nbins_int}_npatch={npatch}.sacc", ) pseudo_cl_path = os.path.join( a.cosmo_val_dir, f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" diff --git a/papers/bmodes/scripts/compute_cosebis_pte_single.py b/papers/bmodes/scripts/compute_cosebis_pte_single.py index 452fc514..fcfe9ec0 100644 --- a/papers/bmodes/scripts/compute_cosebis_pte_single.py +++ b/papers/bmodes/scripts/compute_cosebis_pte_single.py @@ -15,9 +15,9 @@ from pathlib import Path import numpy as np -import treecorr from plotting_utils import compute_chi2_pte +from sp_validation import sacc_io from sp_validation.b_modes import calculate_cosebis @@ -90,14 +90,7 @@ def main(config, xi_integration, cov_integration, out_dir, version=None, blind=N blind = blind if blind is not None else fid["blind"] nmodes = int(fid["nmodes"]) # 20 for full computation - min_sep_int = fid["min_sep_int"] - max_sep_int = fid["max_sep_int"] - nbins_int = fid["nbins_int"] - - gg = treecorr.GGCorrelation( - min_sep=min_sep_int, max_sep=max_sep_int, nbins=nbins_int, sep_units="arcmin" - ) - gg.read(xi_integration) + gg = sacc_io.xi_correlation(sacc_io.load(xi_integration)) # Reporting theta grid (nbins+1 = 21 edges): geomspace(1', 250', 21) theta_grid = np.geomspace(fid["min_sep"], fid["max_sep"], fid["nbins"] + 1) @@ -185,7 +178,7 @@ def _from_cli(argv=None): ap.add_argument( "--xi-integration", required=True, - help="Fiducial 1000-bin integration-grid TreeCorr xi_pm .txt dump", + help="Fiducial 1000-bin integration-grid ξ± SACC part", ) ap.add_argument( "--cov-integration", diff --git a/papers/bmodes/scripts/cosebis_binning_comparison.py b/papers/bmodes/scripts/cosebis_binning_comparison.py index 1e9ac56f..7802afa9 100644 --- a/papers/bmodes/scripts/cosebis_binning_comparison.py +++ b/papers/bmodes/scripts/cosebis_binning_comparison.py @@ -10,14 +10,12 @@ import argparse import json -import types from datetime import datetime from pathlib import Path import matplotlib.pyplot as plt import numpy as np import seaborn as sns -import treecorr from cosmo_numba.B_modes.cosebis import COSEBIS from plotting_utils import ( FIG_WIDTH_SINGLE, @@ -25,43 +23,12 @@ compute_chi2_pte, ) +from sp_validation import sacc_io from sp_validation.b_modes import calculate_cosebis, scale_cut_to_bins plt.style.use(PAPER_MPLSTYLE) -def _load_gg(xi_path, min_sep, max_sep, nbins, columns_only=False): - """Load a TreeCorr GGCorrelation from a text file. - - If columns_only=True, read only the per-bin columns (meanr, xip, xim) - and skip the patch covariance. This avoids loading a 20000x20000 - covariance matrix for high-nbins files. - """ - gg = treecorr.GGCorrelation( - min_sep=min_sep, - max_sep=max_sep, - nbins=nbins, - sep_units="arcmin", - ) - if columns_only: - # TreeCorr ASCII: ## comment, # col_names, then data rows. - # Read only per-bin columns, skip the huge patch covariance. - # cols: r_nom meanr meanlogr xip xim xip_im xim_im sigma_xip sigma_xim weight npairs - data = np.loadtxt(xi_path, max_rows=nbins) - # Compute bin edges from log-spaced binning (matching TreeCorr) - bin_edges = np.exp(np.linspace(np.log(min_sep), np.log(max_sep), nbins + 1)) - return types.SimpleNamespace( - meanr=data[:, 1], - xip=data[:, 3], - xim=data[:, 4], - left_edges=bin_edges[:-1], - right_edges=bin_edges[1:], - ) - else: - gg.read(xi_path) - return gg - - def _compute_Bn_only(gg, nmodes, scale_cut): """Compute COSEBIS B_n from a GGCorrelation without covariance.""" min_theta, max_theta = scale_cut @@ -95,19 +62,14 @@ def main(config, xi_1k_path, xi_10k_path, cov_1k_path, out_dir): ) scale_cuts = {"fiducial": fiducial_scale_cut, "full": full_scale_cut} - # Integration parameters - min_sep_int = float(config["fiducial"]["min_sep_int"]) - max_sep_int = float(config["fiducial"]["max_sep_int"]) nbins_1k = int(config["fiducial"]["nbins_int"]) nbins_10k = 10_000 # Load both ξ± grids print("Loading 1,000-bin ξ±...") - gg_1k = _load_gg(xi_1k_path, min_sep_int, max_sep_int, nbins_1k) + gg_1k = sacc_io.xi_correlation(sacc_io.load(xi_1k_path)) print("Loading 10,000-bin ξ±...") - gg_10k = _load_gg( - xi_10k_path, min_sep_int, max_sep_int, nbins_10k, columns_only=True - ) + gg_10k = sacc_io.xi_correlation(sacc_io.load(xi_10k_path)) # Compute COSEBIS from 1,000-bin ξ± (with covariance for PTE baseline) print("\nComputing COSEBIS from 1,000-bin ξ±...") @@ -297,12 +259,12 @@ def _from_cli(argv=None): ap.add_argument( "--xi-1k", required=True, - help="Fiducial 1000-bin integration-grid TreeCorr xi_pm .txt", + help="Fiducial 1000-bin integration-grid ξ± SACC part", ) ap.add_argument( "--xi-10k", required=True, - help="Fiducial 10000-bin integration-grid TreeCorr xi_pm .txt", + help="Fiducial 10000-bin integration-grid ξ± SACC part", ) ap.add_argument( "--cov-1k", diff --git a/papers/bmodes/scripts/cosebis_data_vector.py b/papers/bmodes/scripts/cosebis_data_vector.py index dfa0383b..85ac0e9a 100644 --- a/papers/bmodes/scripts/cosebis_data_vector.py +++ b/papers/bmodes/scripts/cosebis_data_vector.py @@ -17,12 +17,12 @@ import matplotlib.pyplot as plt import numpy as np import seaborn as sns -import treecorr from plotting_utils import ( FIG_WIDTH_SINGLE, PAPER_MPLSTYLE, ) +from sp_validation import sacc_io from sp_validation.b_modes import calculate_cosebis plt.style.use(PAPER_MPLSTYLE) @@ -153,21 +153,11 @@ def main(config, xi_integration, cov_integration, out_dir): "full": full_scale_cut, } - min_sep_int = float(config["fiducial"]["min_sep_int"]) - max_sep_int = float(config["fiducial"]["max_sep_int"]) - nbins_int = int(config["fiducial"]["nbins_int"]) - out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) # Load the fiducial fine-binned 2PCF once (integration grid) - gg = treecorr.GGCorrelation( - min_sep=min_sep_int, - max_sep=max_sep_int, - nbins=nbins_int, - sep_units="arcmin", - ) - gg.read(xi_integration) + gg = sacc_io.xi_correlation(sacc_io.load(xi_integration)) # Compute the E_n / B_n data vectors + T^T C_xi T mode covariance at both # scale cuts. Same calculate_cosebis call as the original per-version loop. @@ -236,7 +226,7 @@ def _from_cli(argv=None): ap.add_argument( "--xi-integration", required=True, - help="Fiducial fine-binned (1000-bin) TreeCorr xi_pm .txt dump", + help="Fiducial fine-binned (1000-bin) ξ± SACC part", ) ap.add_argument( "--cov-integration", diff --git a/papers/bmodes/scripts/cosebis_version_comparison.py b/papers/bmodes/scripts/cosebis_version_comparison.py index 9942bb28..9113cf5e 100644 --- a/papers/bmodes/scripts/cosebis_version_comparison.py +++ b/papers/bmodes/scripts/cosebis_version_comparison.py @@ -13,7 +13,6 @@ import matplotlib.pyplot as plt import numpy as np import seaborn as sns -import treecorr from plotting_utils import ( ERRORBAR_DEFAULTS, FIG_WIDTH_FULL, @@ -26,6 +25,7 @@ version_label, ) +from sp_validation import sacc_io from sp_validation.b_modes import calculate_cosebis plt.style.use(PAPER_MPLSTYLE) @@ -148,7 +148,7 @@ def _create_stacked_bmode_figure( def _xi_integration(results_dir, ver): - return f"{results_dir}/{ver}_xi_minsep=0.5_maxsep=300.0_nbins=1000_npatch=1.txt" + return f"{results_dir}/{ver}_xi_minsep=0.5_maxsep=300.0_nbins=1000_npatch=1.sacc" def _cov_integration(cov_dir, ver, blind): @@ -213,10 +213,6 @@ def main( "full": full_scale_cut, } - min_sep_int = float(config["fiducial"]["min_sep_int"]) - max_sep_int = float(config["fiducial"]["max_sep_int"]) - nbins_int = int(config["fiducial"]["nbins_int"]) - # Color/marker assignment: pair by parent version if ecut versions present has_ecut = any("_ecut" in v for v in versions) if has_ecut: @@ -252,13 +248,7 @@ def main( marker = MARKER_STYLES[i] if i < len(MARKER_STYLES) else "o" fillstyle = "full" - gg = treecorr.GGCorrelation( - min_sep=min_sep_int, - max_sep=max_sep_int, - nbins=nbins_int, - sep_units="arcmin", - ) - gg.read(xi_path) + gg = sacc_io.xi_correlation(sacc_io.load(xi_path)) results = calculate_cosebis( gg, @@ -370,7 +360,7 @@ def _from_cli(argv=None): ap.add_argument( "--results-dir", required=True, - help="COSMO_VAL output dir with per-version integration-grid xi_pm .txt dumps", + help="COSMO_VAL output dir with per-version integration-grid ξ± SACC parts", ) ap.add_argument( "--cov-dir", diff --git a/papers/bmodes/scripts/gather_pure_eb_chunks.py b/papers/bmodes/scripts/gather_pure_eb_chunks.py index 887e512f..ddd6eace 100644 --- a/papers/bmodes/scripts/gather_pure_eb_chunks.py +++ b/papers/bmodes/scripts/gather_pure_eb_chunks.py @@ -9,8 +9,8 @@ python gather_pure_eb_chunks.py \ --version SP_v1.4.6.3_leak_corr --blind A \ - --xi-reporting \ - --xi-integration \ + --xi-reporting \ + --xi-integration \ --chunks-dir \ --min-sep 1.0 --max-sep 250.0 --nbins 20 \ --min-sep-int 0.5 --max-sep-int 300.0 --nbins-int 1000 \ @@ -23,11 +23,13 @@ import numpy as np +from sp_validation import sacc_io -def _load_xi(path, nbins): - """Load ξ± from a TreeCorr text dump.""" - data = np.loadtxt(path, comments="#", max_rows=nbins) - return {"meanr": data[:, 1], "xip": data[:, 3], "xim": data[:, 4]} + +def _load_xi(path): + """Load ξ± from its SACC part.""" + gg = sacc_io.xi_correlation(sacc_io.load(path)) + return {"meanr": gg.meanr, "xip": gg.xip, "xim": gg.xim} def gather( @@ -46,8 +48,8 @@ def gather( print(f"Gathering pure E/B for blind {blind}") - gg = _load_xi(xi_reporting, nbins) - gg_int = _load_xi(xi_integration, nbins_int) + gg = _load_xi(xi_reporting) + gg_int = _load_xi(xi_integration) eb_results = get_pure_EB_modes( theta=gg["meanr"], diff --git a/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py b/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py index dd34db22..e37bd34b 100644 --- a/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py +++ b/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py @@ -20,7 +20,6 @@ import matplotlib.pyplot as plt import numpy as np import seaborn as sns -import treecorr import yaml from astropy.io import fits from cosmo_numba.B_modes.cosebis import COSEBIS @@ -34,6 +33,7 @@ ) from pseudo_cl_io import load_pseudo_cl_data +from sp_validation import sacc_io from sp_validation.b_modes import calculate_cosebis plt.style.use(PAPER_MPLSTYLE) @@ -139,17 +139,7 @@ def _compute_config_cosebis(xi_path, cov_path, nmodes, scale_cut, config): Returns (En, Bn, cov). """ - min_sep_int = float(config["fiducial"]["min_sep_int"]) - max_sep_int = float(config["fiducial"]["max_sep_int"]) - nbins_int = int(config["fiducial"]["nbins_int"]) - - gg = treecorr.GGCorrelation( - min_sep=min_sep_int, - max_sep=max_sep_int, - nbins=nbins_int, - sep_units="arcmin", - ) - gg.read(xi_path) + gg = sacc_io.xi_correlation(sacc_io.load(xi_path)) results = calculate_cosebis( gg, nmodes=nmodes, scale_cuts=[scale_cut], cov_path=cov_path @@ -859,7 +849,7 @@ def _from_cli(argv=None): required=True, help=( "COSMO_VAL output dir (pseudo_cl SACC parts, pseudo_cl_cov FITS " - "+ xi_integration txt)" + "+ xi_integration SACC part)" ), ) ap.add_argument( @@ -896,9 +886,9 @@ def _from_cli(argv=None): "--fiducial-xi-path", default=None, help=( - "Explicit path to the fiducial 1000-bin integration xi_pm text file " + "Explicit path to the fiducial 1000-bin integration ξ± SACC part " "reproduced by lc (e.g. SP_v1.4.6.3_leak_corr_xi_minsep=0.5_maxsep=300.0_" - "nbins=1000_npatch=1.txt), overriding the --cosmo-val-dir pattern lookup " + "nbins=1000_npatch=1.sacc), overriding the --cosmo-val-dir pattern lookup " "for --fiducial-version." ), ) @@ -955,7 +945,7 @@ def _from_cli(argv=None): else os.path.join( a.cosmo_val_dir, f"{ver}_xi_minsep={min_sep_int}_maxsep={max_sep_int}" - f"_nbins={nbins_int}_npatch={npatch}.txt", + f"_nbins={nbins_int}_npatch={npatch}.sacc", ) ) inputs[f"cov_{ver}"] = ( diff --git a/papers/bmodes/scripts/precompute_pure_eb_chunk.py b/papers/bmodes/scripts/precompute_pure_eb_chunk.py index 99153096..8ea6982a 100644 --- a/papers/bmodes/scripts/precompute_pure_eb_chunk.py +++ b/papers/bmodes/scripts/precompute_pure_eb_chunk.py @@ -14,8 +14,8 @@ --chunk-id 0 --n-chunks 20 --n-samples 2000 \ --version SP_v1.4.6.3_leak_corr --blind A \ --cat-config /path/cosmo_val/cat_config.yaml \ - --xi-reporting \ - --xi-integration \ + --xi-reporting \ + --xi-integration \ --cov-integration \ --min-sep 1.0 --max-sep 250.0 --nbins 20 \ --min-sep-int 0.5 --max-sep-int 300.0 --nbins-int 1000 \ @@ -29,6 +29,8 @@ import tqdm from scipy import sparse +from sp_validation import sacc_io + def _build_cosmology(cosmo_params): """Build a CCL cosmology from a PLANCK18-style params dict.""" @@ -44,16 +46,13 @@ def _build_cosmology(cosmo_params): def _load_xi(path, min_sep, max_sep, nbins): - """Load ξ± from a TreeCorr text dump and recompute the log bin edges.""" - data = np.loadtxt(path, comments="#", max_rows=nbins) - meanr = data[:, 1] - xip = data[:, 3] - xim = data[:, 4] + """Load ξ± from its SACC part and recompute the log bin edges.""" + gg = sacc_io.xi_correlation(sacc_io.load(path)) bin_edges = np.logspace(np.log10(min_sep), np.log10(max_sep), nbins + 1) return { - "meanr": meanr, - "xip": xip, - "xim": xim, + "meanr": gg.meanr, + "xip": gg.xip, + "xim": gg.xim, "left_edges": bin_edges[:-1], "right_edges": bin_edges[1:], } diff --git a/papers/bmodes/scripts/run_cosebis_ptes_sweep.py b/papers/bmodes/scripts/run_cosebis_ptes_sweep.py index df3b9b79..c00cd16e 100644 --- a/papers/bmodes/scripts/run_cosebis_ptes_sweep.py +++ b/papers/bmodes/scripts/run_cosebis_ptes_sweep.py @@ -43,7 +43,7 @@ def _xi_integration(xi_sweep_dir, ver): return os.path.join( - xi_sweep_dir, f"{ver}_xi_minsep=0.5_maxsep=300.0_nbins=1000_npatch=1.txt" + xi_sweep_dir, f"{ver}_xi_minsep=0.5_maxsep=300.0_nbins=1000_npatch=1.sacc" ) @@ -62,7 +62,7 @@ def _from_cli(argv=None): ap.add_argument( "--xi-sweep-dir", required=True, - help="xi_sweep output dir with per-version 1000-bin integration xi_pm .txt", + help="xi_sweep output dir with per-version 1000-bin integration ξ± parts", ) ap.add_argument( "--cov-sweep-dir", diff --git a/papers/bmodes/scripts/run_xi_sweep.py b/papers/bmodes/scripts/run_xi_sweep.py index 211214e3..639ce417 100644 --- a/papers/bmodes/scripts/run_xi_sweep.py +++ b/papers/bmodes/scripts/run_xi_sweep.py @@ -2,9 +2,9 @@ Loops the [non-fiducial version list](sweep_versions.nonfiducial_versions) and runs the same ``run_2pcf.run_2pcf`` compute the fiducial two_point recipes call, -once per version, writing every version's ξ± text dump into one lc ``{output}`` +once per version, writing every version's ξ± SACC part into one lc ``{output}`` dir under run_2pcf's native, already-canonical name -``{ver}_xi_minsep={min}_maxsep={max}_nbins={nbins}_npatch={npatch}.txt`` — the +``{ver}_xi_minsep={min}_maxsep={max}_nbins={nbins}_npatch={npatch}.sacc`` — the exact pattern ``cosebis_version_comparison._xi_integration`` reconstructs. Both grids are produced per version: the 1000-bin integration grid feeds diff --git a/papers/catalog/2025_10_02_xi_sys_rho_tau.py b/papers/catalog/2025_10_02_xi_sys_rho_tau.py index ab95d69c..3a73c3dd 100644 --- a/papers/catalog/2025_10_02_xi_sys_rho_tau.py +++ b/papers/catalog/2025_10_02_xi_sys_rho_tau.py @@ -13,6 +13,8 @@ from getdist import MCSamples, plots from shear_psf_leakage.rho_tau_stat import PSFErrorFit, RhoStat, TauStat +from sp_validation import sacc_io + plt.style.use("./matplotlib_config/paper.mplstyle") plt.rcParams["text.usetex"] = True @@ -127,15 +129,12 @@ xi_psf_sys_quant = np.quantile(xi_psf_sys_samples, quantiles, axis=0) theta = psf_fitter.rho_stat_handler.rho_stats["theta"] - if os.path.exists( + xi_part = ( base_dir - + f"/../{ver}_xi_minsep={theta_min}_maxsep={theta_max}_nbins={nbins}_npatch=1.txt" - ): - xip = np.loadtxt( - base_dir - + f"/../{ver}_xi_minsep={theta_min}_maxsep={theta_max}_nbins={nbins}_npatch=1.txt", - usecols=3, - )[:nbins] ## This will need modifications as the pipeline evolved + + f"/../{ver}_xi_minsep={theta_min}_maxsep={theta_max}_nbins={nbins}_npatch=1.sacc" + ) + if os.path.exists(xi_part): + xip = sacc_io.xi_correlation(sacc_io.load(xi_part)).xip ratio_mean = xi_psf_sys_mean / xip ratio_quant = xi_psf_sys_quant / xip @@ -275,15 +274,12 @@ xi_psf_sys_quant = np.quantile(xi_psf_sys_samples, quantiles, axis=0) theta = psf_fitter.rho_stat_handler.rho_stats["theta"] - if os.path.exists( + xi_part = ( base_dir - + f"/../{ver}_xi_minsep={theta_min}_maxsep={theta_max}_nbins={nbins}_npatch=1.txt" - ): - xip = np.loadtxt( - base_dir - + f"/../{ver}_xi_minsep={theta_min}_maxsep={theta_max}_nbins={nbins}_npatch=1.txt", - usecols=3, - )[:nbins] ## This will need modifications as the pipeline evolved + + f"/../{ver}_xi_minsep={theta_min}_maxsep={theta_max}_nbins={nbins}_npatch=1.sacc" + ) + if os.path.exists(xi_part): + xip = sacc_io.xi_correlation(sacc_io.load(xi_part)).xip ratio_mean = xi_psf_sys_mean / xip ratio_quant = xi_psf_sys_quant / xip diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 3f9c23ce..3992d415 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -14,7 +14,7 @@ _get_pte_from_scale_cut, find_conservative_scale_cut_key, ) -from ..custody import custody_of, registry_of +from ..custody import CustodyError, base_catalogue, custody_of, registry_of from ..statistics import chi2_and_pte from ..version import __version__ from .catalog_characterization import CatalogCharacterizationMixin @@ -318,6 +318,9 @@ def __init__( # thread count (max(3, ceil(log2 n))) and ξ± depends on the machine. # 6 is what TreeCorr derives on candide's 48- and 64-CPU nodes. "min_top": 6, + # The CPUs this process may use; TreeCorr's own default is the + # node's count, whatever share of it the job holds. + "num_threads": len(os.sched_getaffinity(0)), } self.catalog_config_path = Path(catalog_config) @@ -406,6 +409,8 @@ def ensure_version_exists(ver): # B-mode results storage for summarize_bmodes() self._pure_eb_results = {} self._cosebis_results = {} + # The sealed ξ± parts calculate_2pcf returned, by (version, grid). + self.xi_parts = {} def _output_path(self, *parts): """Absolute path under the catalog config's output directory. @@ -533,6 +538,25 @@ def custody(self, version): self._declared, version, registry=registry_of(self.catalog_config_path) ) + def patch_centers_path(self, version, npatch): + """The jackknife patch centres ``version`` is measured on with ``npatch``. + + One file per base catalogue, under the output directory's ``patches/``: + a catalogue and its variants split at the same centres. + """ + base = base_catalogue(self._declared, version) + return self._output_path("patches", f"{base}_npatch={int(npatch)}.dat") + + def _refuse_if_blinded(self, version, what): + """Refuse ``what``, which reads ``version``'s ξ± in plaintext, if blinded.""" + custody = self.custody(version) + if custody.status == "blinded": + raise CustodyError( + f"{what} works from {version}'s measured ξ± itself, and " + f"{custody.catalogue} is blinded; derive it from the concealed " + "part calculate_2pcf returns" + ) + def basename(self, version, treecorr_config=None, npatch=None): cfg = treecorr_config or self.treecorr_config patches = npatch or self.npatch @@ -580,8 +604,7 @@ def _calibrated_g(self, ver): Applies additive-bias subtraction and the multiplicative response: ``g = (e − c) / R``. For DES the response is the catalog-averaged per-component ``R11``/``R22`` (column names in the config); for every - other version it is the scalar ``R`` from the config. Used identically - by :meth:`calculate_2pcf` and :meth:`calculate_aperture_mass_dispersion`. + other version it is the scalar ``R`` from the config. Must be called inside a ``self.results[ver].temporarily_read_data()`` context, since it reads ``dat_shear`` columns. diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index 312aadcd..e7609d31 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -7,9 +7,11 @@ import numpy as np +from .. import sacc_io from ..b_modes import ( calculate_cosebis, find_conservative_scale_cut_key, + log_bin_edges, plot_cosebis_covariance_matrix, plot_cosebis_modes, plot_cosebis_scale_cut_heatmap, @@ -88,18 +90,21 @@ def calculate_cosebis( """ self.print_start(f"Computing {version} COSEBIs") - # Set up parameters with defaults - npatch = npatch or self.npatch - - # Always use integration binning for COSEBIs calculation (fine binning) - treecorr_config = self._binning(min_sep_int, max_sep_int, nbins_int) - - # Calculate single fine-binned correlation function for COSEBIs + # Calculate single fine-binned correlation function for COSEBIs, as the + # sealed part: on a blinded catalogue the COSEBIs are concealed with it. print( f"Computing fine-binned 2PCF with {nbins_int} bins from {min_sep_int} to " f"{max_sep_int} arcmin" ) - gg = self.calculate_2pcf(version, npatch=npatch, **treecorr_config) + part = self.calculate_2pcf( + version, + grid="integration", + npatch=npatch, + min_sep=min_sep_int, + max_sep=max_sep_int, + nbins=nbins_int, + ) + gg = sacc_io.xi_correlation(part) if scale_cuts is not None: # Explicit scale cuts provided @@ -263,15 +268,10 @@ def plot_cosebis( # Generate scale cut heatmap if we have multiple scale cuts if multiple_scale_cuts and len(results) > 1: - # Create temporary gg object with correct binning for mapping - treecorr_config_temp = self._binning(min_sep, max_sep, nbins) - gg_temp = self.calculate_2pcf( - version, npatch=npatch, **treecorr_config_temp - ) - + binning = self._binning(min_sep, max_sep, nbins) plot_cosebis_scale_cut_heatmap( results, - (gg_temp.left_edges, gg_temp.right_edges), + log_bin_edges(binning["min_sep"], binning["max_sep"], binning["nbins"]), version, out_stub + "_scalecut_ptes.png", fiducial_scale_cut=fiducial_scale_cut, diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 90157456..5f7632dd 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -66,8 +66,7 @@ def calculate_pure_eb( cov_path_int : str, optional Path to the covariance matrix for the reporting binning. Replaces the treecorr covariance matrix if provided, meaning that var_method has no - effect on the results although it is still passed to - CosmologyValidation.calculate_2pcf. + effect on the results. cosmo_cov : pyccl.Cosmology, optional Cosmology object to use for theoretical xi+/xi- predictions in the semi-analytical covariance calculation. Defaults to self.cosmo if not @@ -100,21 +99,26 @@ def calculate_pure_eb( Notes ----- - - A shared patch file is used for the reporting and integration binning, - and is created if it does not exist. + - Both binnings are measured on the version's persisted patch centres + (:meth:`patch_centers_path`). The jackknife works from TreeCorr's own + measurement, so a blinded catalogue is refused. """ self.print_start(f"Computing {version} pure E/B") + self._refuse_if_blinded(version, "The jackknife pure-E/B") - # Set up parameters with defaults - npatch = npatch or self.npatch - - # Create TreeCorr configurations - treecorr_config = self._binning(min_sep, max_sep, nbins) - treecorr_config_int = self._binning(min_sep_int, max_sep_int, nbins_int) - - # Calculate correlation functions - gg = self.calculate_2pcf(version, npatch=npatch, **treecorr_config) - gg_int = self.calculate_2pcf(version, npatch=npatch, **treecorr_config_int) + npatch = int(npatch or self.npatch) + centres = self._patch_centers(version, npatch) + gg = self._measure_xi( + version, npatch, centres, min_sep=min_sep, max_sep=max_sep, nbins=nbins + ) + gg_int = self._measure_xi( + version, + npatch, + centres, + min_sep=min_sep_int, + max_sep=max_sep_int, + nbins=nbins_int, + ) # Get redshift distribution if using analytic covariance z_dist = ( diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 89dad2a9..ca97a980 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -6,6 +6,7 @@ and plots. It depends on TreeCorr. """ +import hashlib import os import matplotlib.pyplot as plt @@ -14,109 +15,176 @@ import treecorr from cs_util import plots as cs_plots +from .. import sacc_io +from ..custody import base_catalogue +from .sacc_writers import xi_to_sacc -class RealSpaceMixin: - def calculate_2pcf(self, ver, npatch=None, **treecorr_config): - """ - Calculate the two-point correlation function (2PCF) ξ± for a given catalog - version with TreeCorr. - By default the class instance's `npatch` and `treecorr_config` entries are - used to - initialize the TreeCorr Catalog and GGCorrelation objects, but may be - overridden - by passing keyword arguments. +class RealSpaceMixin: + def calculate_2pcf( + self, + ver, + *, + grid="reporting", + npatch=None, + patch_centers=None, + out=None, + **treecorr_config, + ): + """ξ± of ``ver`` on one binning, as its SACC part sealed under its custody. + + @sc signal-leaves-sealed + A catalogue's ξ± leaves this object only as a part sealed under the + catalogue's custody (:func:`sp_validation.sacc_io.seal`): concealed + when it is blinded, before it is returned, cached or written. The + blind is opened first, so a blind that cannot open fails before + TreeCorr runs. + + @sc patch-centres-are-inputs + With patches, the catalogue splits at persisted centres, one file per + base catalogue (:meth:`patch_centers_path`, written by + :meth:`write_patch_centers`), never at centres drawn here: the + full-sample ξ± depends on the patch layout, and TreeCorr's k-means on + the machine. The part names the file by its sha256. Parameters: - ver (str): The catalog version to process. - - npatch (int, optional): The number of patches to use for the calculation. - Defaults to the instance's `npatch` attribute. - - **treecorr_config: Additional TreeCorr configuration parameters that will - override the instance's default `treecorr_config`. For example, `min_sep=1`. + ver (str): The catalogue version to measure. + grid (str): The grid tag the part's points carry. + npatch (int, optional): Jackknife patches; the instance's + ``npatch`` by default. With patches the part carries the + jackknife covariance, without them the shot-noise diagonal. + patch_centers (str, optional): The centres file; by default + :meth:`patch_centers_path`. + out (str, optional): Where to write the part as well. + **treecorr_config: Overrides of the instance's ``treecorr_config``, + e.g. ``min_sep=1``. Returns: - treecorr.GGCorrelation: The TreeCorr GGCorrelation object containing the - computed 2PCF results. - - Notes: - - If the output file for the given configuration already exists, the - calculation is skipped, and the results are loaded from the file. - - If a patch file for the given configuration does not exist, it is - created during the process. - - The ``.txt`` TreeCorr dump is the only raw byproduct written here. + sacc.Sacc: The sealed part, also kept in ``self.xi_parts[ver, grid]``. """ - self.print_magenta(f"Computing {ver} ξ±") - - npatch = npatch or self.npatch - treecorr_config = { - **self._binning(**treecorr_config), - "var_method": "jackknife" if int(npatch) > 1 else "shot", - } - - gg = treecorr.GGCorrelation(treecorr_config) - - # If the output file already exists, skip the calculation - out_fname = self._output_path( - f"{ver}_xi_minsep={treecorr_config['min_sep']}_maxsep={treecorr_config['max_sep']}_nbins={treecorr_config['nbins']}_npatch={npatch}.txt" + npatch = int(npatch or self.npatch) + custody = self.custody(ver) + if custody.status == "blinded": + from .. import blinding + + blinding.open_blind(custody) + + metadata = {**self.sacc_metadata(ver), "npatch": npatch} + patch_centers = self._patch_centers(ver, npatch, patch_centers) + if patch_centers is not None: + with open(patch_centers, "rb") as f: + metadata["patch_centers_sha256"] = hashlib.sha256(f.read()).hexdigest() + gg = self._measure_xi(ver, npatch, patch_centers, **treecorr_config) + + jackknife = npatch > 1 + s = xi_to_sacc( + self.sacc_nz(ver), + metadata, + gg.meanr, + gg.xip, + gg.xim, + grid=grid, + theta_nom=gg.rnom, + npairs=gg.npairs, + weight=gg.weight, + covariance=gg.cov if jackknife else None, + variances=None if jackknife else np.concatenate([gg.varxip, gg.varxim]), ) + part = ( + sacc_io.save(s, out, custody=custody) if out else sacc_io.seal(s, custody) + ) + self.xi_parts[ver, grid] = part + self.print_done("Done 2PCF") + return part - if os.path.exists(out_fname): - self.print_done(f"Skipping 2PCF calculation, {out_fname} exists") - gg.read(out_fname) - - else: - # Load data and create a catalog - with self.results[ver].temporarily_read_data(): - g1, g2 = self._calibrated_g(ver) - w = self._read_shear_cols(ver, "w_col") - - # Use patch file if it exists - patch_file = self._output_path(f"{ver}_patches_npatch={npatch}.dat") - - cat_gal = treecorr.Catalog( - ra=self.results[ver].dat_shear["RA"], - dec=self.results[ver].dat_shear["Dec"], - g1=g1, - g2=g2, - w=w, - ra_units=self.treecorr_config["ra_units"], - dec_units=self.treecorr_config["dec_units"], - npatch=npatch, - patch_centers=patch_file if os.path.exists(patch_file) else None, - ) + def _measure_xi(self, ver, npatch, patch_centers=None, **treecorr_config): + """TreeCorr's ξ± of ``ver``, in plaintext: for this object's use only. - # If no patch file exists, save the current patches - if not os.path.exists(patch_file): - cat_gal.write_patch_centers(patch_file) + Jackknife variances with patches, shot noise without. With patches and + no ``patch_centers``, TreeCorr draws its own centres. + """ + gg = treecorr.GGCorrelation( + { + **self._binning(**treecorr_config), + "var_method": "jackknife" if npatch > 1 else "shot", + } + ) + with self.results[ver].temporarily_read_data(): + g1, g2 = self._calibrated_g(ver) + catalogue = treecorr.Catalog( + ra=self.results[ver].dat_shear["RA"], + dec=self.results[ver].dat_shear["Dec"], + g1=g1, + g2=g2, + w=self._read_shear_cols(ver, "w_col"), + ra_units=self.treecorr_config["ra_units"], + dec_units=self.treecorr_config["dec_units"], + npatch=npatch, + patch_centers=patch_centers, + ) + gg.process(catalogue) + return gg - # Process the catalog & write the correlation functions - gg.process(cat_gal) - # Columns only. The covariance matrix lives in the SACC part; a - # per-patch ξ± realisation is an unblinded data vector nothing reads; - # and TreeCorr cannot read back a text file carrying the matrix - # without the per-patch results. - gg.write(out_fname, write_patch_results=False, write_cov=False) + def _patch_centers(self, ver, npatch, path=None): + """The existing centres file ``ver`` splits at, or None without patches.""" + if npatch <= 1: + return None + path = path or self.patch_centers_path(ver, npatch) + if not os.path.exists(path): + raise FileNotFoundError( + f"{ver} has no patch centres at {path}; write them once with " + f"write_patch_centers({base_catalogue(self._declared, ver)!r}, " + f"{npatch}) (rule xi_patches)" + ) + return path - # Add correlation object to class - if not hasattr(self, "cat_ggs"): - self.cat_ggs = {} - self.cat_ggs[ver] = gg + def write_patch_centers(self, catalogue, npatch, path=None): + """Draw ``npatch`` jackknife patch centres for a base catalogue, once. - self.print_done("Done 2PCF") + TreeCorr's k-means over the catalogue's positions and weights, written + to ``path`` (default :meth:`patch_centers_path`); every measurement of + the catalogue and its variants splits at them. + """ + base = base_catalogue(self._declared, catalogue) + if base != catalogue: + raise ValueError( + f"patch centres belong to {base}, the base catalogue of " + f"{catalogue}; write them from {base}" + ) + path = path or self.patch_centers_path(catalogue, npatch) + # A Catalog's k-means runs on TreeCorr's process-wide thread count. + treecorr.set_omp_threads(self.treecorr_config["num_threads"]) + with self.results[catalogue].temporarily_read_data(): + positions = treecorr.Catalog( + ra=self.results[catalogue].dat_shear["RA"], + dec=self.results[catalogue].dat_shear["Dec"], + w=self._read_shear_cols(catalogue, "w_col"), + ra_units=self.treecorr_config["ra_units"], + dec_units=self.treecorr_config["dec_units"], + npatch=int(npatch), + ) + os.makedirs(os.path.dirname(path), exist_ok=True) + positions.write_patch_centers(path) + return path - return gg + def _reporting_xi(self, ver): + """``ver``'s reporting-grid ξ±, from its part, measured if not yet held.""" + part = self.xi_parts.get((ver, "reporting")) + if part is None: + part = self.calculate_2pcf(ver) + return sacc_io.xi_correlation(part) def plot_2pcf(self): + """The reporting-grid ξ± of every version, drawn from their parts.""" + xi = {ver: self._reporting_xi(ver) for ver in self.versions} + # Plot of n_pairs plt.subplots(ncols=1, nrows=1) for ver in self.versions: - self.calculate_2pcf(ver) plt.plot( - self.cat_ggs[ver].meanr, - self.cat_ggs[ver].npairs, + xi[ver].meanr, + xi[ver].npairs, label=ver, ls=self.cc[ver]["ls"], color=self.cc[ver]["colour"], @@ -133,9 +201,9 @@ def plot_2pcf(self): plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) for idx, ver in enumerate(self.versions): plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, fx=1.05, nx=len(ver)), - self.cat_ggs[ver].xip, - yerr=np.sqrt(self.cat_ggs[ver].varxip), + xi[ver].meanr * cs_plots.dx(idx, fx=1.05, nx=len(ver)), + xi[ver].xip, + yerr=np.sqrt(xi[ver].varxip), label=ver, ls=self.cc[ver]["ls"], color=self.cc[ver]["colour"], @@ -156,9 +224,9 @@ def plot_2pcf(self): plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) for idx, ver in enumerate(self.versions): plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, fx=1.05, nx=len(ver)), - self.cat_ggs[ver].xim, - yerr=np.sqrt(self.cat_ggs[ver].varxim), + xi[ver].meanr * cs_plots.dx(idx, fx=1.05, nx=len(ver)), + xi[ver].xim, + yerr=np.sqrt(xi[ver].varxim), label=ver, ls=self.cc[ver]["ls"], color=self.cc[ver]["colour"], @@ -179,9 +247,9 @@ def plot_2pcf(self): plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) for idx, ver in enumerate(self.versions): plt.errorbar( - self.cat_ggs[ver].meanr, - self.cat_ggs[ver].xip * self.cat_ggs[ver].meanr, - yerr=np.sqrt(self.cat_ggs[ver].varxip) * self.cat_ggs[ver].meanr, + xi[ver].meanr, + xi[ver].xip * xi[ver].meanr, + yerr=np.sqrt(xi[ver].varxip) * xi[ver].meanr, label=ver, ls=self.cc[ver]["ls"], color=self.cc[ver]["colour"], @@ -201,9 +269,9 @@ def plot_2pcf(self): plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) for idx, ver in enumerate(self.versions): plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xim * self.cat_ggs[ver].meanr, - yerr=np.sqrt(self.cat_ggs[ver].varxim) * self.cat_ggs[ver].meanr, + xi[ver].meanr * cs_plots.dx(idx, len(ver)), + xi[ver].xim * xi[ver].meanr, + yerr=np.sqrt(xi[ver].varxim) * xi[ver].meanr, label=ver, ls=self.cc[ver]["ls"], color=self.cc[ver]["colour"], @@ -225,15 +293,15 @@ def plot_2pcf(self): for idx, ver in enumerate(self.versions): plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xip, - yerr=np.sqrt(self.cat_ggs[ver].varxim), + xi[ver].meanr * cs_plots.dx(idx, len(ver)), + xi[ver].xip, + yerr=np.sqrt(xi[ver].varxim), label=r"$\xi_+$", ls="solid", color="green", ) plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), + xi[ver].meanr * cs_plots.dx(idx, len(ver)), self.xi_psf_sys[ver]["mean"], yerr=np.sqrt(self.xi_psf_sys[ver]["var"]), label=r"$\xi^{\rm psf}_{+, {\rm sys}}$", @@ -241,11 +309,9 @@ def plot_2pcf(self): color="red", ) plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xip + self.xi_psf_sys[ver]["mean"], - yerr=np.sqrt( - self.cat_ggs[ver].varxip + self.xi_psf_sys[ver]["var"] - ), + xi[ver].meanr * cs_plots.dx(idx, len(ver)), + xi[ver].xip + self.xi_psf_sys[ver]["mean"], + yerr=np.sqrt(xi[ver].varxip + self.xi_psf_sys[ver]["var"]), label=r"$\xi_+ + \xi^{\rm psf}_{+, {\rm sys}}$", ls="dashdot", color="magenta", @@ -265,13 +331,11 @@ def plot_2pcf(self): self.print_done(f"xi_plus_xi_psf_sys {ver} plot saved to {out_path}") def plot_ratio_xi_sys_xi(self, threshold=0.1, offset=0.02): - plt.subplots(ncols=1, nrows=1, figsize=(10, 7)) for idx, ver in enumerate(self.versions): - self.calculate_2pcf(ver) xi_psf_sys = self.xi_psf_sys[ver] - gg = self.cat_ggs[ver] + gg = self._reporting_xi(ver) ratio = xi_psf_sys["mean"] / gg.xip ratio_err = np.sqrt( @@ -333,36 +397,12 @@ def calculate_aperture_mass_dispersion( theta_map = np.geomspace(theta_min * 5, theta_max / 2, nbins_map) self._map2["theta_map"] = theta_map - treecorr_config = self._binning(theta_min, theta_max, nbins) - for ver in self.versions: self.print_magenta(ver) - - gg = treecorr.GGCorrelation(treecorr_config) - - out_fname = self._output_path(f"xi_for_map2_{ver}.txt") - if os.path.exists(out_fname): - self.print_green(f"Skipping xi for Map2, {out_fname} exists") - gg.read(out_fname) - else: - with self.results[ver].temporarily_read_data(): - g1, g2 = self._calibrated_g(ver) - cat_gal = treecorr.Catalog( - ra=self.results[ver].dat_shear["RA"], - dec=self.results[ver].dat_shear["Dec"], - g1=g1, - g2=g2, - w=self._read_shear_cols(ver, "w_col"), - ra_units=self.treecorr_config["ra_units"], - dec_units=self.treecorr_config["dec_units"], - npatch=npatch, - ) - - gg.process(cat_gal) - gg.write(out_fname) - del cat_gal - del g1 - del g2 + self._refuse_if_blinded(ver, "The aperture-mass dispersion") + gg = self._measure_xi( + ver, npatch, min_sep=theta_min, max_sep=theta_max, nbins=nbins + ) mapsq, mapsq_im, mxsq, mxsq_im, varmapsq = gg.calculateMapSq( R=theta_map, diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 5ec9d6f8..5a8750a9 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -85,7 +85,9 @@ itself is not a dependency. """ +import functools import os +import types import numpy as np import sacc @@ -581,6 +583,71 @@ def get_xi(s, bins, *, grid): return _get_pm(s, XI_PLUS, XI_MINUS, _pair(bins), grid=grid) +class _XiView(types.SimpleNamespace): + """What :func:`xi_correlation` returns; ``cov`` is sliced out when first read.""" + + @functools.cached_property + def cov(self): + return ( + None if self._covariance is None else self._covariance.get_block(self._rows) + ) + + +def xi_correlation(s, bins=(0, 0), grid=None): + """One ξ± series of ``s``, shaped like the TreeCorr ``GGCorrelation`` it came from. + + Carries ``meanr``, ``rnom``, ``xip``, ``xim``, ``varxip``, ``varxim``, + ``cov``, ``npairs``, ``weight``, ``left_edges``, ``right_edges`` and + ``npatch1``, so code written against a measurement reads a part unchanged. + ``grid`` may be left out when the pair's ξ± lies on one grid. The edges are + the log-binning edges about the nominal centres (``theta_nom``), and ``cov`` + is the series' block of the part's covariance. + """ + tracers = _pair(bins) + if grid is None: + grids = {s.data[i].tags.get("grid") for i in _indices(s, XI_PLUS, tracers)} + if len(grids) > 1: + raise ValueError( + f"ξ± of {tracers} lies on grids {sorted(grids)}; pass grid=" + ) + (grid,) = grids + tags = {} if grid is None else {"grid": grid} + plus, minus = (_indices(s, t, tracers, **tags) for t in (XI_PLUS, XI_MINUS)) + rows = np.concatenate([plus, minus]) + + def tag(name): + values = [s.data[i].tags.get(name) for i in plus] + return None if None in values else np.array(values, float) + + covariance = s.covariance + if covariance is None: + variances = np.full(len(rows), np.nan) + elif isinstance(covariance, sacc.covariance.DiagonalCovariance): + variances = np.asarray(covariance.diag)[rows] + else: + variances = np.diagonal(covariance.dense)[rows] + rnom = tag("theta_nom") + edges = (None, None) + if rnom is not None and len(rnom) > 1: + half_bin = np.log(rnom[-1] / rnom[0]) / (len(rnom) - 1) / 2 + edges = (rnom * np.exp(-half_bin), rnom * np.exp(half_bin)) + return _XiView( + meanr=tag("theta"), + rnom=rnom, + xip=s.mean[plus], + xim=s.mean[minus], + varxip=variances[: len(plus)], + varxim=variances[len(plus) :], + npairs=tag("npairs"), + weight=tag("weight"), + left_edges=edges[0], + right_edges=edges[1], + npatch1=int(s.metadata.get("npatch", 1)), + _covariance=covariance, + _rows=rows, + ) + + def get_pseudo_cl(s, bins): """Return ``(ell_eff, cl_ee, cl_bb, cl_eb, window)`` for one tracer pair. diff --git a/src/sp_validation/tests/_synthetic.py b/src/sp_validation/tests/_synthetic.py index b17bab3d..fcd16e99 100644 --- a/src/sp_validation/tests/_synthetic.py +++ b/src/sp_validation/tests/_synthetic.py @@ -94,6 +94,9 @@ def write_synthetic_catalogs( entry = { "subdir": str(cat_dir), "pipeline": "SP", + "colour": "orange", + "ls": "-", + "marker": "o", "shear": { "path": "shear.fits", "redshift_path": str(nz_dir / "dndz_SP_A.txt"), diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 7812cf26..f7168130 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -16,6 +16,7 @@ import yaml from _synthetic import write_synthetic_catalogs +from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation # These tests load real UNIONS catalogues from the cluster filesystem. Skip them @@ -275,13 +276,15 @@ def test_v1_4_6_glass_mock_default_seed(self, base_config): # ------------------------------------------------------------------ def test_calculate_2pcf_runs_on_synthetic_catalog(self, tmp_path): - """calculate_2pcf wires catalog+config into treecorr GGCorrelation. + """calculate_2pcf wires catalog+config into a ξ± part. - Smoke-integration: assert the xi+/- data vector is computed with the - configured number of angular bins and is finite. Numerical values are - deliberately not asserted (could be tightened to allclose vs. a - committed reference later for value-drift coverage). + Smoke-integration: the part's ξ± has the configured number of angular + bins and is finite, and the part reads back with the binning's edges, + by which scale cuts select bins. Numerical values are deliberately not + asserted. """ + from sp_validation.b_modes import log_bin_edges + pytest.importorskip("treecorr") params, version = write_synthetic_catalogs(tmp_path) @@ -295,9 +298,11 @@ def test_calculate_2pcf_runs_on_synthetic_catalog(self, tmp_path): **params, ) - gg = cv.calculate_2pcf(version) + gg = sacc_io.xi_correlation(cv.calculate_2pcf(version)) - # treecorr GGCorrelation with xi+/- on the configured angular grid + edges = log_bin_edges(5.0, 100.0, nbins) + np.testing.assert_allclose(gg.left_edges, edges[0], rtol=1e-12) + np.testing.assert_allclose(gg.right_edges, edges[1], rtol=1e-12) assert gg.xip.shape == (nbins,) assert gg.xim.shape == (nbins,) assert np.all(np.isfinite(gg.xip)) @@ -311,23 +316,26 @@ def test_xi_part_carries_the_covariance_the_measurement_estimated( ): """run_2pcf's ξ± part carries TreeCorr's covariance, whoever calls it. - The jackknife covariance with patches, the shot-noise diagonal without, - so every part has variances whether the rule or the CLI measured it. + The jackknife covariance (correlated bins) with patches, the shot-noise + diagonal without, so every part has variances whether the rule or the + CLI measured it. """ import importlib.util import sacc - from sp_validation import sacc_io - script = Path(__file__).resolve().parents[3] / "workflow/scripts/run_2pcf.py" spec = importlib.util.spec_from_file_location("run_2pcf_part", script) run_2pcf = importlib.util.module_from_spec(spec) spec.loader.exec_module(run_2pcf) params, version = write_synthetic_catalogs(tmp_path) + if npatch > 1: + CosmologyValidation(versions=[version], **params).write_patch_centers( + version, npatch + ) part = tmp_path / "part.sacc" - gg = run_2pcf.run_2pcf( + run_2pcf.run_2pcf( ver=version, min_sep=5.0, max_sep=100.0, @@ -339,40 +347,19 @@ def test_xi_part_carries_the_covariance_the_measurement_estimated( ) cov = sacc_io.load(str(part)).covariance + assert np.all(np.diag(cov.dense) > 0) if npatch > 1: assert isinstance(cov, sacc.covariance.FullCovariance) - np.testing.assert_array_equal(cov.dense, gg.cov) + assert np.any(cov.dense[~np.eye(12, dtype=bool)] != 0) else: assert isinstance(cov, sacc.covariance.DiagonalCovariance) - np.testing.assert_array_equal( - cov.diag, np.concatenate([gg.varxip, gg.varxim]) - ) - - def test_a_patched_xi_dump_reads_back(self, tmp_path): - """calculate_2pcf reads back the text dump a patched measurement wrote. - - The ξ± figure rules re-enter calculate_2pcf on the reporting grid, which - has patches, and are handed the dump rule xi wrote (to its precision). - """ - params, version = write_synthetic_catalogs(tmp_path) - binning = dict(npatch=4, min_sep=5.0, max_sep=100.0, nbins=6) - measured = CosmologyValidation(versions=[version], **params).calculate_2pcf( - version, **binning - ) - read = CosmologyValidation(versions=[version], **params).calculate_2pcf( - version, **binning - ) - for column in ("meanr", "npairs", "xip", "xim", "varxip", "varxim"): - np.testing.assert_allclose( - getattr(read, column), getattr(measured, column), rtol=1e-4 - ) def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): """calculate_2pcf's ξ± is the same on 4 and on 48 TreeCorr threads. Production binning (default bin_slop/angle_slop), both runs on the - jackknife patches the first one writes, each from a fresh Catalog; they - must agree to far below the jackknife σ. + persisted patch centres, each from a fresh Catalog; they must agree to + far below the jackknife σ. """ import treecorr @@ -387,13 +374,13 @@ def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): nbins=6, **params, ) + cv.write_patch_centers(version, 8) xi = {} for n_threads in (4, 48): - # calculate_2pcf reads back an existing text dump instead of measuring. - for dump in Path(params["output_dir"]).glob(f"{version}_xi_*.txt"): - dump.unlink() - gg = cv.calculate_2pcf(version, num_threads=n_threads) + gg = sacc_io.xi_correlation( + cv.calculate_2pcf(version, num_threads=n_threads) + ) assert treecorr.get_omp_threads() == n_threads # the count took effect xi[n_threads] = np.concatenate([gg.xip, gg.xim]) sigma = np.sqrt(np.concatenate([gg.varxip, gg.varxim])) @@ -401,6 +388,26 @@ def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): shift = np.max(np.abs(xi[48] - xi[4]) / sigma) assert shift < 1e-6, f"ξ± moves by {shift:.3g}σ between 4 and 48 threads" + def test_a_patched_measurement_splits_at_the_persisted_centres(self, tmp_path): + """[patch-centres-are-inputs] With patches, calculate_2pcf splits at the + base catalogue's centres file and names it in the part; without the + file it refuses rather than drawing centres of its own.""" + import hashlib + + params, version = write_synthetic_catalogs(tmp_path) + cv = CosmologyValidation(versions=[version], npatch=4, **params) + with pytest.raises(FileNotFoundError, match="write_patch_centers"): + cv.calculate_2pcf(version) + + centres = Path(cv.patch_centers_path(version, 4)) + cv.write_patch_centers(version, 4) + part = cv.calculate_2pcf(version) + assert part.metadata["npatch"] == 4 + assert ( + part.metadata["patch_centers_sha256"] + == hashlib.sha256(centres.read_bytes()).hexdigest() + ) + def test_calculate_scale_dependent_leakage_runs_on_synthetic_catalog( self, tmp_path ): @@ -454,7 +461,7 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( ξ±: exact binning (bin_slop = angle_slop = 0) makes ξ± a plain pair sum, independent of the tree and so of the jackknife patches, whose k-means - centres this test does not fix. + centres depend on the machine. """ pytest.importorskip("treecorr") pytest.importorskip("cosmo_numba") @@ -476,6 +483,7 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( **params, ) cv.treecorr_config.update(bin_slop=0, angle_slop=0) + cv.write_patch_centers(version, npatch) kernel, kernel_calls = b_modes.pure_eb_from_xi, [] monkeypatch.setattr( @@ -520,3 +528,83 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( cov = np.asarray(results["cov"]) assert cov.shape == (6 * nbins, 6 * nbins) assert results["n_eff"] == npatch + + +# --------------------------------------------------------------------------- # +# I14: a blinded catalogue's ξ± leaves CosmologyValidation only concealed +# --------------------------------------------------------------------------- # +@pytest.fixture +def blinded_and_twin(tmp_path): + """TOY, blinded under `toy`, and TOY_OPEN: the same files, unblinded.""" + import json + + from sp_validation import blinding + + params, _ = write_synthetic_catalogs( + tmp_path, + n_gal=4000, + coherent_shear=True, + catalogues={"TOY": None, "TOY_OPEN": "unblinded"}, + ) + fast = tmp_path / "fast.json" + fast.write_text(json.dumps({"theory": {"transfer_function": "eisenstein_hu"}})) + blinding.main( + [ + "init", + "toy", + "TOY", + "--cat-config", + params["catalog_config"], + "--config", + str(fast), + ] + ) + return CosmologyValidation( + versions=["TOY", "TOY_OPEN"], + npatch=1, + theta_min=5.0, + theta_max=60.0, + nbins=6, + **params, + ) + + +def test_a_blinded_catalogues_xi_leaves_concealed(blinded_and_twin): + """[signal-leaves-sealed] calculate_2pcf returns, and caches, a blinded + catalogue's ξ± shifted from its unblinded twin's by exactly the blind's shift.""" + from sp_validation import blinding + + cv = blinded_and_twin + blinded, twin = (cv.calculate_2pcf(v) for v in ("TOY", "TOY_OPEN")) + shift = blinding.conceal(twin, blinding.open_blind(cv.custody("TOY"))).mean + shift = shift - twin.mean + + assert np.all(shift != 0) + np.testing.assert_allclose( + blinded.mean - twin.mean, shift, rtol=1e-8, atol=1e-12 * np.abs(shift).max() + ) + assert blinded.metadata["blinding"] == "blinded" + assert cv.xi_parts["TOY", "reporting"] is blinded + + +@pytest.mark.parametrize( + "measure", + [ + lambda cv: cv.calculate_pure_eb("TOY", npatch=1), + lambda cv: cv.calculate_aperture_mass_dispersion(npatch=1), + ], + ids=["jackknife-pure-eb", "aperture-mass"], +) +def test_plaintext_measurements_refuse_a_blinded_catalogue( + blinded_and_twin, measure, monkeypatch +): + """The jackknife pure-E/B and the aperture mass work from TreeCorr's ξ± + itself, so on a blinded catalogue they refuse before measuring anything.""" + from sp_validation.custody import CustodyError + + def measured(*args, **kwargs): + raise AssertionError("measured a blinded catalogue's ξ± in plaintext") + + monkeypatch.setattr(CosmologyValidation, "_measure_xi", measured) + with pytest.raises(CustodyError, match="TOY is blinded"): + measure(blinded_and_twin) diff --git a/src/sp_validation/tests/test_custody.py b/src/sp_validation/tests/test_custody.py index 8e1eb4c9..be80e0d2 100644 --- a/src/sp_validation/tests/test_custody.py +++ b/src/sp_validation/tests/test_custody.py @@ -253,7 +253,12 @@ def _toy_checkout(tmp_path): def test_host_and_job_resolve_the_same_custody(tmp_path): - """`common.custody_token(v)` equals `CosmologyValidation.custody(v).token`.""" + """The host and a job agree on each version's custody and patch centres. + + `common.custody_token(v)` equals `CosmologyValidation.custody(v).token`, and + the centres the xi rule declares are the file the job splits at, one per + base catalogue. + """ from sp_validation.cosmo_val import CosmologyValidation root = _toy_checkout(tmp_path) @@ -268,8 +273,12 @@ def test_host_and_job_resolve_the_same_custody(tmp_path): cv = CosmologyValidation( versions=versions, catalog_config=common.CAT_CONFIG, - output_dir=str(tmp_path / "out"), + output_dir=str(common.COSMO_VAL), ) for version in versions: assert common.custody_token(version) == cv.custody(version).token, version + assert common.patches_path(version, 100) == cv.patch_centers_path( + version, 100 + ), version assert common.custody_token("TOY").startswith("blinded:TOY:toy:") + assert len({common.patches_path(v, 100) for v in versions}) == 3 diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py index 656434aa..01e43797 100644 --- a/src/sp_validation/tests/test_custody_e2e.py +++ b/src/sp_validation/tests/test_custody_e2e.py @@ -3,15 +3,17 @@ A synthetic catalogue is declared three ways (``TOY`` blinded, ``TOY_OPEN`` unblinded, ``TOY_MOCK`` mock). A blind is drawn for ``TOY`` with ``blinding init``; the rule scripts run as Snakemake runs them (``runpy`` with a -``snakemake`` object) for ``TOY`` and ``TOY_leak_corr``: both ξ± grids, a -two-bin writer, pseudo-Cℓ on an nside-32 NaMaster workspace, ρ/τ, COSEBIs, -pure-E/B and assembly. The blind is then revealed, the declaration flipped, the -chain re-run, and the audit must prove blinded − true = shift(seed) on every -part the reveal archived. +``snakemake`` object) for ``TOY`` and ``TOY_leak_corr``: the patch centres, both +ξ± grids, the ξ± figures, a two-bin writer, pseudo-Cℓ on an nside-32 NaMaster +workspace, ρ/τ, COSEBIs, pure-E/B and assembly. The blind is then revealed, the +declaration flipped, the chain re-run, and the audit must prove blinded − true = +shift(seed) on every part the reveal archived. Neither the blinded run's files +nor its figures may hold a true ξ± value. """ import json import os +import re import runpy import types from pathlib import Path @@ -30,7 +32,7 @@ SCRIPTS = REPO / "workflow" / "scripts" GRIDS = { - "reporting": {"min_sep": 5.0, "max_sep": 60.0, "nbins": 6, "npatch": 1}, + "reporting": {"min_sep": 5.0, "max_sep": 60.0, "nbins": 6, "npatch": 4}, "integration": {"min_sep": 1.0, "max_sep": 150.0, "nbins": 300, "npatch": 1}, } SCALE_CUT = [12.0, 60.0] @@ -132,6 +134,24 @@ def _covariances(root): return paths +def patch_centres(cat_config, version, out, npatch): + """Rule xi_patches for ``version``'s base catalogue, unless its file exists.""" + base = cu.base_catalogue(yaml.safe_load(Path(cat_config).read_text()), version) + centres = out / "patches" / f"{base}_npatch={npatch}.dat" + if not centres.exists(): + run_rule( + "xi_patches.py", + output={"patches": str(centres)}, + params={ + "catalogue": base, + "npatch": npatch, + "cat_config": str(cat_config), + "output_dir": str(out), + }, + ) + return centres + + def run_chain(cat_config, version, out, cov, *, grid_parts=None): """The cosmo_val chain for one version, into ``out``; returns the part paths.""" out.mkdir(parents=True, exist_ok=True) @@ -139,16 +159,15 @@ def run_chain(cat_config, version, out, cov, *, grid_parts=None): xi = {} for grid, b in GRIDS.items(): xi[grid] = out / f"{version}_xi_{grid}.sacc" - binning = ( - f"minsep={b['min_sep']}_maxsep={b['max_sep']}_nbins={b['nbins']}" - f"_npatch={b['npatch']}" + patches = ( + {"patches": str(patch_centres(cat_config, version, out, b["npatch"]))} + if b["npatch"] > 1 + else {} ) run_rule( "run_2pcf.py", - output={ - "txt": str(out / f"{version}_xi_{binning}.txt"), - "sacc": str(xi[grid]), - }, + input=patches, + output={"sacc": str(xi[grid])}, params={ "ver": version, **b, @@ -255,6 +274,73 @@ def assemble(cat_config, version, out, paths, cov, token): ) +def plot_2pcf(cat_config, out, xi): + """Rule cv_plot_2pcf over the reporting parts ``xi`` ({version: path}).""" + run_rule( + "cv_plot_2pcf.py", + input={"xi": [str(path) for path in xi.values()]}, + output={"sentinel": str(out / "plot_2pcf.done")}, + params={ + "cv_init": { + "versions": list(xi), + "catalog_config": str(cat_config), + "output_dir": str(out), + } + }, + ) + + +def recording_figures(monkeypatch): + """Record every array a figure draws; returns the list it fills.""" + import matplotlib.axes + + drawn = [] + for name in ("plot", "errorbar"): + + def draw(self, *args, _draw=getattr(matplotlib.axes.Axes, name), **kwargs): + drawn.extend(np.asarray(a, float) for a in args if np.ndim(a) == 1) + return _draw(self, *args, **kwargs) + + monkeypatch.setattr(matplotlib.axes.Axes, name, draw) + return drawn + + +def _near(x, values, rtol): + """Whether any finite, nonzero ``x`` lies within ``rtol`` of a sorted ``values``.""" + keep = np.isfinite(x) & (x != 0) + x, rtol = x[keep], np.broadcast_to(rtol, keep.shape)[keep] + i = np.clip(np.searchsorted(values, x), 1, len(values) - 1) + gap = np.minimum(np.abs(x - values[i - 1]), np.abs(x - values[i])) + return bool(np.any(gap <= rtol * np.abs(x))) + + +def plaintext(blobs, values): + """Where the bytes of ``blobs`` ({name: bytes}) hold any of ``values``. + + A value is found as a float64 of either byte order at any offset, to the + float noise of a re-measurement, or as a number written in exponent + notation, to half a unit of its last digit. + """ + values = np.sort(np.asarray(values, float)) + found = [] + for name, blob in blobs.items(): + for order in "<>": + for offset in range(8): + count = (len(blob) - offset) // 8 + if count > 0 and _near( + np.frombuffer(blob, f"{order}f8", count, offset), values, 1e-9 + ): + found.append(f"{name}: float64 {order} at offset {offset}") + tokens = re.findall(rb"(-?\d\.(\d+)e[-+]\d+)", blob) + if tokens and _near( + np.array([float(t) for t, _ in tokens]), + values, + np.array([0.51 * 10.0 ** -len(digits) for _, digits in tokens]), + ): + found.append(f"{name}: as text") + return found + + def stamps(root): """``{relative path: stamp}`` of every SACC under ``root``.""" return { @@ -311,8 +397,16 @@ def test_a_blinded_catalogue_from_birth_to_audit(toy, monkeypatch): assert blinded.status == "blinded" # --- a blinded run ------------------------------------------------------- - for version in ("TOY", "TOY_leak_corr"): + versions = ("TOY", "TOY_leak_corr") + for version in versions: run_chain(toy.cat_config, version, out, toy.cov) + with monkeypatch.context() as m: + drawn = recording_figures(m) + plot_2pcf( + toy.cat_config, out, {v: out / f"{v}_xi_reporting.sacc" for v in versions} + ) + concealed = sio.xi_correlation(sio.load(out / "TOY_xi_reporting.sacc")) + assert any(np.array_equal(a, concealed.xip) for a in drawn) # drawn from the part born = stamps(out) assert len(born) == 2 * (2 + len(PARTS)), sorted(born) assert all(s == blinded.stamp for s in born.values()), born # one commitment @@ -419,6 +513,11 @@ def second_block_fails(*args, **kwargs): # --- the reveal: archive, flip the declaration, re-measure, audit -------- (out / "two_bin.sacc").unlink() + blinded_run = { + str(p.relative_to(toy.root)): p.read_bytes() + for p in out.rglob("*") + if p.is_file() + } assert ( bd.main( ["reveal", "toy", "--root", str(out), "--cat-config", str(toy.cat_config)] @@ -458,6 +557,19 @@ def second_block_fails(*args, **kwargs): )["ok"] revealed.write_text(published) + # --- no true ξ± value was ever written or drawn by the blinded run ------- + true_xi = np.concatenate( + [ + np.concatenate([gg.xip, gg.xim]) + for v in versions + for grid in GRIDS + for gg in [sio.xi_correlation(sio.load(out / f"{v}_xi_{grid}.sacc"))] + ] + ) + figures = {"figures": np.concatenate(drawn).tobytes()} + assert not plaintext({**blinded_run, **figures}, true_xi) + assert not list(toy.root.rglob("*_xi_*.txt")) + def test_a_mock_never_opens_a_blind(toy, monkeypatch): def refuse(custody): diff --git a/workflow/README.md b/workflow/README.md index 08732e2d..ae94d5f6 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -38,7 +38,10 @@ For a blinded catalogue, every ξ± and pseudo-Cℓ_EE value is shifted by a hid cosmology before it is first written. COSEBIs and pure-E/B computed from the shifted ξ± carry the blind with them; B-modes stay usable. Every SACC file records the custody it was born under, and assembly refuses parts under any -other. +other. ξ± leaves `CosmologyValidation.calculate_2pcf` only as its sealed part +(no text dump), so the ξ± figures and notebooks draw what the part holds; the +jackknife `calculate_pure_eb` and the aperture mass, which need TreeCorr's own +measurement, refuse a blinded catalogue. A blinded catalogue needs a blind, drawn once, by a person. The launch stops with this command when none covers it: @@ -162,6 +165,15 @@ mkdir -p /data ln -s /n17data/cdaley/unions/code/sp_validation/cosmo_inference/data/mask /data/ ``` +Jackknife patch centres are drawn once per base catalogue, by rule +`xi_patches` into `COSMO_VAL/patches/`, and every patched ξ± measurement of the +catalogue and its variants splits at them (in a notebook: +`CosmologyValidation.write_patch_centers`). + +Jobs draw their figures with matplotlib's defaults: `common.py` points each +job's `MATPLOTLIBRC` at `workflow/matplotlibrc`, so your own matplotlibrc +shapes your interactive work in the container but never a rule's figure. + Other rules write elsewhere: masks under the run directory's `output/masks/`, `papers/bmodes`' figures and macros under its run directory's `docs/`, the image sims under their `grids_base`. diff --git a/workflow/common.py b/workflow/common.py index c7a9aaba..3345db8b 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -70,6 +70,13 @@ def _load_checkout_module(name): # Without it, jobs use the home directory's cache, which every node mounts. os.environ.pop("XDG_CACHE_HOME", None) +# Jobs draw their figures with matplotlib's defaults, whoever launches them. +# Apptainer binds $HOME, so a job would otherwise read the launching user's +# ~/.config/matplotlib/matplotlibrc, and a LaTeX preamble there that the image +# cannot typeset stops every figure rule. MATPLOTLIBRC outranks that file, and +# APPTAINERENV_ carries it past --cleanenv. +os.environ["APPTAINERENV_MATPLOTLIBRC"] = str(REPO_ROOT / "workflow/matplotlibrc") + # Output roots are env-overridable so a reproduction run can write into a # fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL @@ -107,6 +114,7 @@ def _load_checkout_module(name): # silent failures. Apply with: wildcard_constraints: **WILDCARD_CONSTRAINTS WILDCARD_CONSTRAINTS = { "version": r"SP_v[\d.]+(_w_iv)?(_ecut\d+)?(_leak_corr)?", + "catalogue": r"SP_v[\d.]+(_w_iv)?(_ecut\d+)?", "blind": r"[ABC]", "nbins": r"\d+", "min_sep": r"[0-9.]+", @@ -403,6 +411,15 @@ def base_version(version): return _custody.base_catalogue(CATALOG_CONFIG, version) +def patches_path(version, npatch): + """The jackknife patch centres ``version`` is measured on with ``npatch``. + + One file per base catalogue, so a catalogue and its variants split alike. + """ + name = f"{base_version(version)}_npatch={int(npatch)}.dat" + return str(COSMO_VAL / "patches" / name) + + def build_redshift_path(version, blind): """Construct n(z) filepath for given catalog version and blind.""" base = base_version(version) @@ -489,9 +506,9 @@ def pseudo_cl_tag(config): return f"blind={fiducial['blind']}_{fiducial['binning']}_nbins={fiducial['nbins']}" -def get_shear_catalog(wildcards): - """Resolve shear catalog path from config for a given version.""" - cat_config = CATALOG_CONFIG[_custody.entry_of(wildcards.version)] +def shear_catalog(version): + """The shear catalogue file of ``version``, from its catalogue entry.""" + cat_config = CATALOG_CONFIG[_custody.entry_of(version)] shear_path = cat_config["shear"]["path"] if shear_path.startswith("/"): return shear_path diff --git a/workflow/matplotlibrc b/workflow/matplotlibrc new file mode 100644 index 00000000..6a13d270 --- /dev/null +++ b/workflow/matplotlibrc @@ -0,0 +1,2 @@ +# The matplotlibrc every workflow job reads (common.py names it): empty, so a +# job's figures take matplotlib's defaults. diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index ebb6835e..8c614fad 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -7,21 +7,21 @@ # products they write under COSMO_VAL (= cosmo_val/output): # # catalogue ──→ xi (one job per grid: reporting, integration) -# ├─ reporting part ──────→ pure_eb (part, npz, figures) -# ├─ integration part ─┬──→ pure_eb -# │ └──→ cosebis (part, npz, figures) -# └─ reporting .txt ──→ 2pcf plot, ratio_xi_sys_xi +# ├─ reporting part ───┬──→ pure_eb (part, npz, figures) +# │ └──→ 2pcf plot, ratio_xi_sys_xi +# └─ integration part ─┬──→ pure_eb +# └──→ cosebis (part, npz, figures) # CosmoCov ξ±, integration grid ──→ pure_eb, cosebis (their covariances) # catalogue ──→ pseudo_cl (part) ──┬─→ pseudo-Cl figures # CosmoCov ξ±, reporting grid ─────┤ # rho/tau (part + FITS) ───────────┼─→ summarize_bmodes (reads the products) # └─→ assemble_sacc ──→ {version}.sacc # -# The B-mode rules are ingests: pure_eb, cosebis, the pseudo-Cl figures and the -# summary all work from the parts and the covariance inputs, never from a -# catalogue, so a blinded part keeps everything downstream blinded. The -# analytic covariances (CosmoCov ξ±, NaMaster pseudo-Cℓ) are what assembly puts -# in the terminal file, replacing the estimates a part was born with. +# The B-mode rules and the ξ± figures are ingests: they work from the parts and +# the covariance inputs, never from a catalogue, so a blinded part keeps +# everything downstream blinded. The analytic covariances (CosmoCov ξ±, +# NaMaster pseudo-Cℓ) are what assembly puts in the terminal file, replacing +# the estimates a part was born with. # # Methods that only emit figures, or whose figure paths derive from internal # handler state (rho/tau plots, rho_tau_fits, objectwise leakage, 2pcf @@ -50,15 +50,6 @@ CV_BINNING = ( ) -def cv_xi_txt(version): - """Path to the 2pcf data vector calculate_2pcf writes for a version. - - Mirrors the out_fname f-string in cosmo_val.calculate_2pcf: - {ver}_xi_minsep=..._maxsep=..._nbins=..._npatch=...txt - """ - return str(COSMO_VAL / f"{version}_xi_{xi_binning('reporting')}.txt") - - def cv_rho_stats(version): return str( COSMO_VAL / "rho_tau_stats" / f"rho_stats_{cv_basename(version, CV_FIDUCIAL)}.fits" @@ -211,12 +202,11 @@ CV_INIT = cv_init_params(config) # --------------------------------------------------------------------------- # PSF diagnostics: rho/tau statistics and the PSF-error fit # --------------------------------------------------------------------------- -# The rho/tau FITS and the xi data vector are produced by the generic compute -# rules `rho_tau_stats` and `xi` already defined in workflow/rules/twopoint.smk -# (same filename convention, same CosmologyValidation methods). The cosmo_val -# suite consumes those outputs rather than re-declaring colliding rules. The -# helper paths cv_rho_stats / cv_tau_stats / cv_xi_txt resolve to exactly the -# files those rules write, so requesting them triggers the existing compute. +# The rho/tau FITS and the ξ± parts are produced by the generic compute rules +# `rho_tau_stats` and `xi` in workflow/rules/twopoint.smk. The cosmo_val suite +# consumes those outputs rather than re-declaring colliding rules: the helper +# paths cv_rho_stats / cv_tau_stats / cv_xi_sacc resolve to exactly the files +# those rules write, so requesting them triggers the existing compute. rule cv_plot_rho_stats: @@ -330,9 +320,9 @@ rule cv_additive_bias: rule cv_plot_2pcf: - """n_pairs / xi± overlay across versions.""" + """n_pairs / xi± overlay across versions, from the reporting parts.""" input: - xi=[cv_xi_txt(v) for v in CV_VERSIONS], + xi=[cv_xi_sacc(v, "reporting") for v in CV_VERSIONS], output: sentinel=str(CV_SENTINELS / "plot_2pcf.done"), params: @@ -346,7 +336,7 @@ rule cv_plot_2pcf: rule cv_ratio_xi_sys_xi: """Ratio of PSF systematics (xi_psf_sys) to the cosmic-shear signal (xi+).""" input: - xi=[cv_xi_txt(v) for v in CV_VERSIONS], + xi=[cv_xi_sacc(v, "reporting") for v in CV_VERSIONS], rho=[cv_rho_stats(v) for v in CV_VERSIONS], tau=[cv_tau_stats(v) for v in CV_VERSIONS], output: diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 4b3836aa..35edf69d 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -14,15 +14,16 @@ def xi_binning(grid): rule xi: - """TreeCorr ξ±(θ) for one version on one angular grid. + """TreeCorr ξ±(θ) for one version on one angular grid, as its sealed SACC part. One rule for every grid: outputs are named by their binning, so a request - binds the wildcards and the grid label resolves from them. + binds the wildcards and the grid label resolves from them. With patches, + the measurement splits at the base catalogue's persisted centres. """ input: - catalog=get_shear_catalog, + catalog=lambda w: shear_catalog(w.version), + patches=lambda w: patches_path(w.version, w.npatch) if int(w.npatch) > 1 else [], output: - txt=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), sacc=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 24 params: @@ -45,6 +46,29 @@ rule xi: "../scripts/run_2pcf.py" +rule xi_patches: + """Jackknife patch centres of one base catalogue, drawn once for its variants. + + TreeCorr's k-means over the catalogue's positions and weights; they carry no + shear, so no custody. + """ + input: + catalog=lambda w: shear_catalog(w.catalogue), + output: + patches=str(COSMO_VAL / "patches" / "{catalogue}_npatch={npatch}.dat"), + threads: 12 + params: + catalogue="{catalogue}", + npatch="{npatch}", + cat_config=CAT_CONFIG, + output_dir=str(COSMO_VAL), + resources: + mem_mb=30000, + runtime=120, + script: + "../scripts/xi_patches.py" + + rule rho_tau_stats: output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), diff --git a/workflow/scripts/cv_plot_2pcf.py b/workflow/scripts/cv_plot_2pcf.py index 5251e7a5..1f6925c7 100644 --- a/workflow/scripts/cv_plot_2pcf.py +++ b/workflow/scripts/cv_plot_2pcf.py @@ -1,14 +1,14 @@ """Rule cv_plot_2pcf: n_pairs / xi± overlay across versions. -Reads each version's xi txt (declared inputs, produced by cv_2pcf); calls -plot_2pcf, which re-reads the existing txt files rather than recomputing. -Writes figures under the output dir. Sentinel-tracked: plot_2pcf emits several -figures whose names are internal. +Draws each version's reporting ξ± part (the declared inputs). Writes figures +under the output dir. Sentinel-tracked: plot_2pcf emits several figures whose +names are internal. """ -from cv_runner import _unbuffer_streams, make_cv, touch_sentinels +from cv_runner import _unbuffer_streams, make_cv, take_reporting_parts, touch_sentinels _unbuffer_streams() cv = make_cv(snakemake) +take_reporting_parts(cv, snakemake.input["xi"]) cv.plot_2pcf() touch_sentinels(snakemake) diff --git a/workflow/scripts/cv_ratio_xi_sys_xi.py b/workflow/scripts/cv_ratio_xi_sys_xi.py index 79afd89a..7f8936dd 100644 --- a/workflow/scripts/cv_ratio_xi_sys_xi.py +++ b/workflow/scripts/cv_ratio_xi_sys_xi.py @@ -1,15 +1,16 @@ """Rule cv_ratio_xi_sys_xi: ratio of PSF systematics to cosmic-shear signal. -Joins two upstream chains: the 2pcf data vector (xi txt, reloaded via -calculate_2pcf) and xi_psf_sys (recomputed in memory from the rho/tau FITS via -the lazy cv.xi_psf_sys property — the PSF-error fit is not persisted by -cosmo_val.py). Both are declared as inputs so the DAG shows the join. Writes -ratio_xi_sys_xi.png at a fixed path (declared output). +Joins two upstream chains: each version's reporting ξ± part and xi_psf_sys +(recomputed in memory from the rho/tau FITS via the lazy cv.xi_psf_sys property +-- the PSF-error fit is not persisted by cosmo_val.py). Both are declared as +inputs so the DAG shows the join. Writes ratio_xi_sys_xi.png at a fixed path +(declared output). """ -from cv_runner import _unbuffer_streams, make_cv, verify_outputs +from cv_runner import _unbuffer_streams, make_cv, take_reporting_parts, verify_outputs _unbuffer_streams() cv = make_cv(snakemake) +take_reporting_parts(cv, snakemake.input["xi"]) cv.plot_ratio_xi_sys_xi(offset=snakemake.params.get("offset", 0.1)) verify_outputs(snakemake) diff --git a/workflow/scripts/cv_runner.py b/workflow/scripts/cv_runner.py index 601f2e14..bb67cdfb 100644 --- a/workflow/scripts/cv_runner.py +++ b/workflow/scripts/cv_runner.py @@ -7,7 +7,7 @@ common machinery so each rule script stays a few readable lines. The in-memory ``cv`` object is re-instantiated per rule. The DAG links rules -through the *real* data products each method writes (rho/tau FITS, xi text, +through the *real* data products each method writes (rho/tau FITS, ξ± parts, pseudo-Cl FITS, E/B npz, leakage pkl); the lazy ``cv`` properties that aren't persisted (``c1``/``c2``, ``xi_psf_sys``, ``rho_tau_fits``) are recomputed in whichever rule needs them, which is cheap next to the science compute they @@ -36,6 +36,14 @@ def make_cv(snakemake): return CosmologyValidation(**dict(snakemake.params["cv_init"])) +def take_reporting_parts(cv, paths): + """Hand ``cv`` the reporting ξ± parts at ``paths``, one per version, in order.""" + from sp_validation import sacc_io + + for version, path in zip(cv.versions, paths, strict=True): + cv.xi_parts[version, "reporting"] = sacc_io.load(path) + + def verify_outputs(snakemake): """Fail loudly if any declared output is missing after the method ran.""" missing = [str(o) for o in snakemake.output if not Path(o).exists()] diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 0fa59c46..ded79da4 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -13,13 +13,11 @@ --out The measurement is binning-agnostic: the reporting and the fine integration -grids are the same compute with different ``--min-sep/--max-sep/--nbins``. -``CosmologyValidation.calculate_2pcf`` writes the ``.txt`` dump (a raw -byproduct); the ξ± data product is born as SACC here, a *part* named by its -binning and tagged with its ``--grid``, sealed under the catalogue's custody. -The part carries the covariance the measurement estimated: the dense jackknife -covariance when it had patches, the shot-noise ``varxip``/``varxim`` diagonal -when it had none. +grids are the same compute with different ``--min-sep/--max-sep/--nbins``. The +ξ± is born as a SACC part, named by its binning, tagged with its ``--grid`` and +sealed under the catalogue's custody by ``CosmologyValidation.calculate_2pcf``; +nothing else is written. With patches, the measurement splits at the base +catalogue's persisted centres (rule xi_patches). ``output_dir`` is passed explicitly so lc can point each run at its own ``{output}`` tree. @@ -28,11 +26,7 @@ import argparse import os -import numpy as np - -from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation -from sp_validation.cosmo_val.sacc_writers import xi_to_sacc from sp_validation.custody import confirm @@ -47,65 +41,48 @@ def run_2pcf( sacc_out=None, grid="reporting", custody=None, + patch_centers=None, ): - """Measure ξ±(θ) for ``ver`` and write its born-as-SACC part. + """Measure ξ±(θ) for ``ver`` and write its sealed SACC part. Parameters mirror the TreeCorr reporting/integration grids: ``min_sep`` / ``max_sep`` in arcmin, ``nbins`` logarithmic bins, ``npatch`` spatial patches (1 for the paper fiducial). ``cat_config`` is an absolute path to the catalog configuration; ``output_dir`` overrides - ``cat_config['paths']['output']`` so the ``.txt`` byproduct lands where lc - expects. ``sacc_out`` is the exact destination for the SACC part (the - Snakemake-declared output); it defaults to a binning-derived name under - the resolved output directory for the CLI path. ``custody`` is the custody - token Snakemake resolved for ``ver`` (the rule's ``params.custody``); the - part is not written under any other. + ``cat_config['paths']['output']``. ``sacc_out`` is the exact destination for + the part (the Snakemake-declared output); it defaults to a binning-derived + name under the resolved output directory for the CLI path. ``custody`` is + the custody token Snakemake resolved for ``ver`` (the rule's + ``params.custody``); the part is not written under any other. + ``patch_centers`` is the centres file to split at, by default the base + catalogue's under ``output_dir``. Returns ------- - treecorr.GGCorrelation - The measured correlation object (also the source of the SACC part). + sacc.Sacc + The part as written. """ cv = CosmologyValidation( - versions=[ver], - catalog_config=cat_config, - output_dir=output_dir, - # so the SACC provenance metadata stamps the npatch actually measured - npatch=npatch, + versions=[ver], catalog_config=cat_config, output_dir=output_dir ) - declared = cv.custody(ver) if custody is not None: - confirm(declared, custody) - gg = cv.calculate_2pcf( - ver=ver, + confirm(cv.custody(ver), custody) + out_path = sacc_out or os.path.join( + output_dir or cv.cc["paths"]["output"], + f"{ver}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc", + ) + part = cv.calculate_2pcf( + ver, + grid=grid, npatch=npatch, + patch_centers=patch_centers, + out=out_path, min_sep=min_sep, max_sep=max_sep, nbins=nbins, ) - - # Born-as-SACC ξ± part. theta = meanr; theta_nom = rnom. - jackknife = gg.var_method == "jackknife" - s = xi_to_sacc( - cv.sacc_nz(ver), - cv.sacc_metadata(ver), - gg.meanr, - gg.xip, - gg.xim, - grid=grid, - theta_nom=gg.rnom, - npairs=gg.npairs, - weight=gg.weight, - covariance=gg.cov if jackknife else None, - variances=None if jackknife else np.concatenate([gg.varxip, gg.varxim]), - ) - out_path = sacc_out or os.path.join( - output_dir or cv.cc["paths"]["output"], - f"{ver}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc", - ) - sacc_io.save(s, out_path, custody=declared) print(f"Wrote {grid} ξ± SACC part: {out_path}") - return gg + return part def _from_snakemake(smk): @@ -119,10 +96,9 @@ def _from_snakemake(smk): cat_config=p["cat_config"], output_dir=p["output_dir"], grid=p.get("grid", "reporting"), - # The SACC part goes exactly where the rule declares it; the .txt - # byproduct still lands under the resolved output dir. sacc_out=smk.output["sacc"], custody=p["custody"], + patch_centers=smk.input.get("patches") or None, ) diff --git a/workflow/scripts/xi_patches.py b/workflow/scripts/xi_patches.py new file mode 100644 index 00000000..bd7b76bc --- /dev/null +++ b/workflow/scripts/xi_patches.py @@ -0,0 +1,19 @@ +"""Rule xi_patches: the jackknife patch centres of one base catalogue. + +TreeCorr's k-means over the catalogue's positions and weights, written once to +the declared output; every ξ± measurement of the catalogue and its variants +splits at these centres. +""" + +from cv_runner import _unbuffer_streams + +from sp_validation.cosmo_val import CosmologyValidation + +_unbuffer_streams() +p = snakemake.params +cv = CosmologyValidation( + versions=[p["catalogue"]], + catalog_config=p["cat_config"], + output_dir=p["output_dir"], +) +cv.write_patch_centers(p["catalogue"], int(p["npatch"]), snakemake.output["patches"]) diff --git a/workflow/tests/data/container_smoke/container_smoke.py b/workflow/tests/data/container_smoke/container_smoke.py index effc0b0b..a73033aa 100644 --- a/workflow/tests/data/container_smoke/container_smoke.py +++ b/workflow/tests/data/container_smoke/container_smoke.py @@ -2,7 +2,7 @@ Cheap sanity check of the profile-driven container path -- same executor (slurm), same software-deployment-method (apptainer), same apptainer-args -binds, same container image every real rule uses. Four things it proves, each +binds, same container image every real rule uses. What it proves, each written to the output YAML: * the job really ran inside the image (``APPTAINER_CONTAINER``, set by @@ -12,7 +12,8 @@ * the numeric stack works (numpy eigh on a small fixed matrix). ``OMP_NUM_THREADS`` is recorded but not asserted -- see the assertions; * which commit of this checkout is running (git rev-parse from inside the - container -- proves /home is bound and usable, not just readable). + container -- proves /home is bound and usable, not just readable); + * which matplotlibrc the job's figures would read. Driven by the co-located Snakefile; the assertions on the output YAML live in workflow/tests/test_container_smoke.py (candide only). @@ -66,6 +67,9 @@ except (subprocess.CalledProcessError, FileNotFoundError) as exc: commit = f"unavailable ({exc})" +# --- the matplotlibrc a figure rule would read ----------------------------- +import matplotlib # noqa: E402 + provenance = { "repo_dir": repo_dir, "commit": commit, @@ -80,6 +84,7 @@ "sp_validation": sp_validation_info, "numeric": numeric_info, "provenance": provenance, + "matplotlibrc": matplotlib.matplotlib_fname(), }, f, sort_keys=False, diff --git a/workflow/tests/test_container_smoke.py b/workflow/tests/test_container_smoke.py index 8045103d..66c999ba 100644 --- a/workflow/tests/test_container_smoke.py +++ b/workflow/tests/test_container_smoke.py @@ -83,6 +83,12 @@ def test_container_smoke(): module_file ) + # Figures read the workflow's matplotlibrc, never the user's. + assert ( + Path(report["matplotlibrc"]).resolve() + == (REPO / "workflow/matplotlibrc").resolve() + ), report["matplotlibrc"] + # The numeric stack agrees with the same computation run here. np.testing.assert_allclose( report["numeric"]["eigenvalues"], diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 4406816e..000b18ad 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -1,6 +1,7 @@ """DAG properties, checked through the host launcher (see conftest.py).""" import os +import re import subprocess import sys from pathlib import Path @@ -79,6 +80,46 @@ def test_one_integration_grid(toy): assert {part, covariance} <= by_rule["cv_pure_eb"], by_rule["cv_pure_eb"] +def test_xi_leaves_a_measurement_only_as_a_part(toy): + """No job reads or writes a ξ± text dump; the ξ± figures draw the parts.""" + result = toy.snakemake("-n", "all") + assert result.returncode == 0, result.stdout + jobs = parse_jobs(result.stdout) + dumps = [ + f for j in jobs for f in j.input + j.output if re.search(r"_xi_.*\.txt$", f) + ] + assert not dumps, dumps + + grids = toy.common.xi_grids(toy.config, toy.config["fiducial"]) + reporting = toy.common.grid_binning(grids["reporting"]) + for rule in ("cv_plot_2pcf", "cv_ratio_xi_sys_xi"): + (job,) = [j for j in jobs if j.rule == rule] + assert {Path(f).name for f in job.input if "_xi_" in Path(f).name} == { + f"{v}_xi_{reporting}.sacc" for v in VERSIONS + }, job.input + + +def test_patch_centres_are_an_input_shared_by_variants(toy): + """Every patched ξ± job splits at its base catalogue's centres, drawn once.""" + result = toy.snakemake("-n", "assemble_sacc_all") + assert result.returncode == 0, result.stdout + jobs = parse_jobs(result.stdout) + npatch = toy.common.xi_grids(toy.config, toy.config["fiducial"])["reporting"][ + "npatch" + ] + assert npatch > 1 + centres = str(toy.cosmo_val / "patches" / f"{VERSIONS[0]}_npatch={npatch}.dat") + + assert [j.output for j in jobs if j.rule == "xi_patches"] == [[centres]] + xi = [j for j in jobs if j.rule == "xi"] + assert {j.wildcards["version"] for j in xi} == set(VERSIONS) + for job in xi: + patched = int(job.wildcards["npatch"]) > 1 + assert [f for f in job.input if "/patches/" in f] == ( + [centres] if patched else [] + ), job + + @pytest.mark.parametrize("named", [True, False], ids=["named", "unnamed"]) def test_outputs_stay_in_the_output_roots(toy, named): """Nothing the suite declares lands outside the configured output roots. diff --git a/workflow/tests/test_toy_run.py b/workflow/tests/test_toy_run.py new file mode 100644 index 00000000..7a06c6d1 --- /dev/null +++ b/workflow/tests/test_toy_run.py @@ -0,0 +1,192 @@ +"""Rule xi, run for real through apptainer, before and after a blind is drawn. + +The launch is the README's with the machine-independent default profile: the +host Snakemake, the image `spv-container` manages, jobs on this node. A toy +checkout under your home directory (which the profile binds) carries copies of +workflow/, papers/cosmo_val/ and src/, and one synthetic catalogue declared +unblinded. Its reporting parts and their figure are made; then the catalogue is +declared blinded, a blind is drawn for it, and the same launch re-measures the +parts concealed. The jobs run under a matplotlibrc asking for a LaTeX package +no image has, as a user's own may. +""" + +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import pytest +import yaml +from conftest import REPO, container, on_candide + +VERSIONS = ("SP_v0.1", "SP_v0.1_leak_corr") +REPORTING = {"theta_min": 5.0, "theta_max": 60.0, "nbins": 6, "npatch": 4} +BINNING = "minsep=5.0_maxsep=60.0_nbins=6_npatch=4" + + +def _in_image(image, root, *command): + """``command``'s stdout, run in ``image`` on the toy checkout's src/.""" + return subprocess.run( + [ + "apptainer", + "exec", + "--cleanenv", + "--env", + f"PYTHONPATH={root / 'src'}", + str(image), + *command, + ], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout + + +def _stamps(image, root, parts): + """The custody stamp of each SACC part, read in the image.""" + script = ( + "import json, sys\n" + "from sp_validation import custody, sacc_io\n" + "print(json.dumps([custody.read_stamp(sacc_io.load(p).metadata).stamp" + " for p in sys.argv[1:]]))" + ) + return json.loads(_in_image(image, root, "python", "-c", script, *map(str, parts))) + + +def _toy_checkout(root, image): + """A checkout with one synthetic catalogue, SP_v0.1, declared unblinded.""" + skip = shutil.ignore_patterns(".snakemake", "__pycache__", "tests") + shutil.copytree(REPO / "workflow", root / "workflow", ignore=skip) + shutil.copytree( + REPO / "papers" / "cosmo_val", root / "papers" / "cosmo_val", ignore=skip + ) + shutil.copytree( + REPO / "src", root / "src", ignore=shutil.ignore_patterns("__pycache__") + ) + (root / "cosmo_val").mkdir() + _in_image( + image, + root, + "python", + "-c", + "import sys\n" + f"sys.path.insert(0, {str(root / 'src/sp_validation/tests')!r})\n" + "from pathlib import Path\n" + "from _synthetic import write_synthetic_catalogs\n" + f"write_synthetic_catalogs(Path({str(root / 'cosmo_val')!r})," + f" catalogues={{{VERSIONS[0]!r}: 'unblinded'}})", + ) + + config_path = root / "papers" / "cosmo_val" / "config" / "config.yaml" + config = yaml.safe_load(config_path.read_text()) + config["versions"] = list(VERSIONS) + config["fiducial"]["version"] = VERSIONS[1] + config["fiducial"]["mock_version"] = VERSIONS[0] + config["cosmo_val"].update(REPORTING) + config_path.write_text(yaml.safe_dump(config, sort_keys=False)) + + # A user's matplotlibrc typesetting with a package the image lacks. + rc = root / "xdg" / "matplotlib" / "matplotlibrc" + rc.parent.mkdir(parents=True) + rc.write_text( + "text.usetex: True\n" + "text.latex.preamble: \\usepackage{spvalidationabsentpackage}\n" + ) + (root / "fast.json").write_text( + json.dumps({"theory": {"transfer_function": "eisenstein_hu"}}) + ) + + +@pytest.mark.candide +@on_candide +def test_xi_before_and_after_its_catalogue_is_blinded(): + image, kind = container.resolve_image() + assert kind != "tag", "no local image; run `spv-container pull`" + root = Path(tempfile.mkdtemp(prefix="toy_run_", dir=Path.home())) + _toy_checkout(root, image) + out = root / "out" + cat_config = root / "cosmo_val" / "cat_config.yaml" + parts = [out / f"{v}_xi_{BINNING}.sacc" for v in VERSIONS] + target = str(out / "snakemake_sentinels" / "plot_2pcf.done") + + env = { + k: v + for k, v in os.environ.items() + if k not in ("SNAKEMAKE_PROFILE", "APPTAINERENV_PYTHONPATH") + } + env.update( + COSMO_VAL=str(out), + COSMO_INFERENCE=str(root / "inference"), + XDG_CACHE_HOME=str(root / "cache"), + APPTAINERENV_XDG_CONFIG_HOME=str(root / "xdg"), + PYTHONNOUSERSITE="1", + PYTHONUNBUFFERED="1", + ) + + def launch(*args): + return subprocess.run( + [ + sys.executable, + "-m", + "snakemake", + "--profile", + str(root / "workflow" / "profiles" / "default"), + "--cores", + "4", + *args, + "--config", + f"container={image}", + "--", + target, + ], + cwd=root / "papers" / "cosmo_val", + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=1800, + check=False, + ) + + # Declared unblinded: parts stamped unblinded, and their figure drawn. + result = launch() + assert result.returncode == 0, result.stdout + assert [s["blinding"] for s in _stamps(image, root, parts)] == ["unblinded"] * 2 + assert (out / "patches" / "SP_v0.1_npatch=4.dat").is_file() + assert (out / "xi_p.png").is_file() + + # Declared blinded, and its blind drawn: the parts' params changed. + catalogues = yaml.safe_load(cat_config.read_text()) + catalogues[VERSIONS[0]]["blinding"] = "blinded" + cat_config.write_text(yaml.safe_dump(catalogues, sort_keys=False)) + _in_image( + image, + root, + "python", + "-m", + "sp_validation.blinding", + "init", + "toy", + VERSIONS[0], + "--cat-config", + str(cat_config), + "--config", + str(root / "fast.json"), + ) + dry = launch("-n") + assert dry.returncode == 0, dry.stdout + assert "Params have changed" in dry.stdout, dry.stdout + + result = launch() + assert result.returncode == 0, result.stdout + record = json.loads((root / "cosmo_val/blinds/toy/commitment.json").read_text()) + for stamp in _stamps(image, root, parts): + assert stamp["blinding"] == "blinded", stamp + assert stamp["blinding_commitment"] == record["seed_commitment"], stamp + assert not list(out.rglob("*_xi_*.txt")) + assert not list((root / "cosmo_val" / "output").iterdir()) + + shutil.rmtree(root) # kept on failure, for post-mortem From 851d413e492f195c7f009495bb5c0441e3452912 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 00:38:17 +0200 Subject: [PATCH 106/160] papers/bmodes: the pure-E/B sweep reads the xi sweep's SACC parts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_xi_sweep.py writes each version's ξ± as a SACC part and the pure-E/B chunk scripts read parts, so the sweep driver looks for the .sacc files. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- papers/bmodes/scripts/run_pure_eb_semianalytic.sh | 2 +- papers/bmodes/scripts/run_pure_eb_sweep.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/papers/bmodes/scripts/run_pure_eb_semianalytic.sh b/papers/bmodes/scripts/run_pure_eb_semianalytic.sh index 472cd81d..88ddbbc1 100644 --- a/papers/bmodes/scripts/run_pure_eb_semianalytic.sh +++ b/papers/bmodes/scripts/run_pure_eb_semianalytic.sh @@ -9,7 +9,7 @@ # Usage: # run_pure_eb_semianalytic.sh --version SP_v1.4.6.3_leak_corr --blind A \ # --cat-config \ -# --xi-reporting --xi-integration \ +# --xi-reporting --xi-integration \ # --cov-integration \ # --out [--n-chunks 20] [--n-samples 2000] [--nproc 16] set -euo pipefail diff --git a/papers/bmodes/scripts/run_pure_eb_sweep.sh b/papers/bmodes/scripts/run_pure_eb_sweep.sh index 8195f150..8b8587d4 100755 --- a/papers/bmodes/scripts/run_pure_eb_sweep.sh +++ b/papers/bmodes/scripts/run_pure_eb_sweep.sh @@ -40,8 +40,8 @@ mkdir -p "$OUT" VERSIONS=$(sweep_versions "$CONFIG") for ver in $VERSIONS; do - xirep="$XISWEEP/${ver}_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.txt" - xiint="$XISWEEP/${ver}_xi_minsep=0.5_maxsep=300.0_nbins=1000_npatch=1.txt" + xirep="$XISWEEP/${ver}_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.sacc" + xiint="$XISWEEP/${ver}_xi_minsep=0.5_maxsep=300.0_nbins=1000_npatch=1.sacc" covbase="covariance_${ver}_${BLIND}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" covint="$COVSWEEP/$covbase/${covbase}_processed.txt" for f in "$xirep" "$xiint" "$covint"; do From 4cdab707da73489a79633b37ee0d8bbf82bcd7d5 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 00:50:06 +0200 Subject: [PATCH 107/160] Patch centres are drawn by hand, never by a rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TreeCorr's k-means cannot be reproduced, and a rule that drew the centres re-drew them whenever Snakemake re-ran it: on -F, and under the profiles' code trigger whenever its script was newer than the file. Every patched ξ± then moved (a few percent), and a reveal's audit could no longer match the blinded parts' patch_centers_sha256. Rule xi_patches is gone. The centres are drawn once per base catalogue by `python -m sp_validation.cosmo_val.patch_centers --cat-config … --output-dir `, which refuses to replace a file. Rule xi reads them as an input no rule produces; a launch into a tree without them stops with that command (common.patches_input), and calculate_2pcf's refusal names the same command, which works from an object holding only a variant. Tests: -F schedules no job writing a centres file (red with the rule restored); a tree without centres stops the launch naming the command (red without the existence check); the command named by calculate_2pcf's refusal draws the centres and refuses to replace them (red with the old message, and without the refusal); P14 runs the command the first launch names, in the image; P4 dry-runs into a tree of stand-in centres. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/cosmo_val/patch_centers.py | 33 +++++++++++ src/sp_validation/cosmo_val/real_space.py | 28 ++++++--- src/sp_validation/tests/test_cosmo_val.py | 26 ++++++--- src/sp_validation/tests/test_custody_e2e.py | 33 +++++------ workflow/README.md | 18 ++++-- workflow/common.py | 25 +++++++- workflow/rules/twopoint.smk | 28 +-------- workflow/scripts/run_2pcf.py | 2 +- workflow/scripts/xi_patches.py | 19 ------- workflow/tests/conftest.py | 15 ++++- workflow/tests/test_dag.py | 60 ++++++++++++++++---- workflow/tests/test_toy_run.py | 22 +++++-- 12 files changed, 208 insertions(+), 101 deletions(-) create mode 100644 src/sp_validation/cosmo_val/patch_centers.py delete mode 100644 workflow/scripts/xi_patches.py diff --git a/src/sp_validation/cosmo_val/patch_centers.py b/src/sp_validation/cosmo_val/patch_centers.py new file mode 100644 index 00000000..54275be7 --- /dev/null +++ b/src/sp_validation/cosmo_val/patch_centers.py @@ -0,0 +1,33 @@ +"""Draw a base catalogue's jackknife patch centres into an output tree, once. + + python -m sp_validation.cosmo_val.patch_centers \\ + --cat-config cosmo_val/cat_config.yaml --output-dir + +writes ``/patches/_npatch=.dat`` +(:meth:`CosmologyValidation.write_patch_centers`). It reads the whole +catalogue, so run it on a compute node. +""" + +import argparse + +from .core import CosmologyValidation + + +def main(argv=None): + parser = argparse.ArgumentParser( + prog="python -m sp_validation.cosmo_val.patch_centers", + description=__doc__.split("\n")[0], + ) + parser.add_argument("catalogue", help="a base catalogue of the catalogue config") + parser.add_argument("npatch", type=int) + parser.add_argument("--cat-config", required=True) + parser.add_argument("--output-dir", required=True, help="the output tree") + a = parser.parse_args(argv) + cv = CosmologyValidation( + versions=[a.catalogue], catalog_config=a.cat_config, output_dir=a.output_dir + ) + print(cv.write_patch_centers(a.catalogue, a.npatch)) + + +if __name__ == "__main__": + main() diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index ca97a980..36abb25e 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -42,10 +42,12 @@ def calculate_2pcf( @sc patch-centres-are-inputs With patches, the catalogue splits at persisted centres, one file per - base catalogue (:meth:`patch_centers_path`, written by - :meth:`write_patch_centers`), never at centres drawn here: the - full-sample ξ± depends on the patch layout, and TreeCorr's k-means on - the machine. The part names the file by its sha256. + base catalogue (:meth:`patch_centers_path`), never at centres drawn + here: the full-sample ξ± depends on the patch layout, and TreeCorr's + k-means cannot be reproduced. So the centres are drawn once, by hand + (``python -m sp_validation.cosmo_val.patch_centers``, which never + replaces a file), and never by a workflow rule, which a forced or + re-triggered run would re-draw. The part names the file by its sha256. Parameters: ver (str): The catalogue version to measure. @@ -132,10 +134,13 @@ def _patch_centers(self, ver, npatch, path=None): return None path = path or self.patch_centers_path(ver, npatch) if not os.path.exists(path): + base = base_catalogue(self._declared, ver) raise FileNotFoundError( - f"{ver} has no patch centres at {path}; write them once with " - f"write_patch_centers({base_catalogue(self._declared, ver)!r}, " - f"{npatch}) (rule xi_patches)" + f"{ver} splits at {base}'s patch centres, and {path} does not " + "exist. Draw them once: python -m " + f"sp_validation.cosmo_val.patch_centers {base} {npatch} " + f"--cat-config {os.path.abspath(self.catalog_config_path)} " + f"--output-dir {self._output_path()}" ) return path @@ -143,8 +148,8 @@ def write_patch_centers(self, catalogue, npatch, path=None): """Draw ``npatch`` jackknife patch centres for a base catalogue, once. TreeCorr's k-means over the catalogue's positions and weights, written - to ``path`` (default :meth:`patch_centers_path`); every measurement of - the catalogue and its variants splits at them. + to ``path`` (default :meth:`patch_centers_path`). An existing file is + never replaced. """ base = base_catalogue(self._declared, catalogue) if base != catalogue: @@ -153,6 +158,11 @@ def write_patch_centers(self, catalogue, npatch, path=None): f"{catalogue}; write them from {base}" ) path = path or self.patch_centers_path(catalogue, npatch) + if os.path.exists(path): + raise FileExistsError( + f"{path} exists: {catalogue}'s patch centres are drawn once, and " + "its patched ξ± split at them. Delete the file to draw new ones." + ) # A Catalog's k-means runs on TreeCorr's process-wide thread count. treecorr.set_omp_threads(self.treecorr_config["num_threads"]) with self.results[catalogue].temporarily_read_data(): diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index f7168130..1f89236d 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -390,22 +390,34 @@ def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): def test_a_patched_measurement_splits_at_the_persisted_centres(self, tmp_path): """[patch-centres-are-inputs] With patches, calculate_2pcf splits at the - base catalogue's centres file and names it in the part; without the - file it refuses rather than drawing centres of its own.""" + base catalogue's centres file and names it in the part. Without the + file it refuses, naming the command that draws it, which works from an + object holding only a variant and never replaces the file.""" import hashlib + import shlex - params, version = write_synthetic_catalogs(tmp_path) + from sp_validation.cosmo_val import patch_centers + + params, base = write_synthetic_catalogs(tmp_path) + version = f"{base}_leak_corr" cv = CosmologyValidation(versions=[version], npatch=4, **params) - with pytest.raises(FileNotFoundError, match="write_patch_centers"): + with pytest.raises(FileNotFoundError) as refused: cv.calculate_2pcf(version) + command = "python -m sp_validation.cosmo_val.patch_centers " + assert command in str(refused.value), refused.value + argv = shlex.split(str(refused.value).split(command, 1)[1]) + patch_centers.main(argv) centres = Path(cv.patch_centers_path(version, 4)) - cv.write_patch_centers(version, 4) + drawn = centres.read_bytes() + with pytest.raises(FileExistsError): + patch_centers.main(argv) + assert centres.read_bytes() == drawn + part = cv.calculate_2pcf(version) assert part.metadata["npatch"] == 4 assert ( - part.metadata["patch_centers_sha256"] - == hashlib.sha256(centres.read_bytes()).hexdigest() + part.metadata["patch_centers_sha256"] == hashlib.sha256(drawn).hexdigest() ) def test_calculate_scale_dependent_leakage_runs_on_synthetic_catalog( diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py index 01e43797..fa62b476 100644 --- a/src/sp_validation/tests/test_custody_e2e.py +++ b/src/sp_validation/tests/test_custody_e2e.py @@ -3,12 +3,12 @@ A synthetic catalogue is declared three ways (``TOY`` blinded, ``TOY_OPEN`` unblinded, ``TOY_MOCK`` mock). A blind is drawn for ``TOY`` with ``blinding init``; the rule scripts run as Snakemake runs them (``runpy`` with a -``snakemake`` object) for ``TOY`` and ``TOY_leak_corr``: the patch centres, both -ξ± grids, the ξ± figures, a two-bin writer, pseudo-Cℓ on an nside-32 NaMaster -workspace, ρ/τ, COSEBIs, pure-E/B and assembly. The blind is then revealed, the -declaration flipped, the chain re-run, and the audit must prove blinded − true = -shift(seed) on every part the reveal archived. Neither the blinded run's files -nor its figures may hold a true ξ± value. +``snakemake`` object) for ``TOY`` and ``TOY_leak_corr``, on patch centres drawn +by their command: both ξ± grids, the ξ± figures, a two-bin writer, pseudo-Cℓ on +an nside-32 NaMaster workspace, ρ/τ, COSEBIs, pure-E/B and assembly. The blind +is then revealed, the declaration flipped, the chain re-run, and the audit must +prove blinded − true = shift(seed) on every part the reveal archived. Neither +the blinded run's files nor its figures may hold a true ξ± value. """ import json @@ -27,6 +27,7 @@ from sp_validation import custody as cu from sp_validation import sacc_io as sio from sp_validation.cosmo_val import CosmologyValidation +from sp_validation.cosmo_val.patch_centers import main as draw_patch_centers REPO = Path(__file__).resolve().parents[3] SCRIPTS = REPO / "workflow" / "scripts" @@ -135,19 +136,19 @@ def _covariances(root): def patch_centres(cat_config, version, out, npatch): - """Rule xi_patches for ``version``'s base catalogue, unless its file exists.""" + """``version``'s base catalogue's centres, drawn by their command if absent.""" base = cu.base_catalogue(yaml.safe_load(Path(cat_config).read_text()), version) centres = out / "patches" / f"{base}_npatch={npatch}.dat" if not centres.exists(): - run_rule( - "xi_patches.py", - output={"patches": str(centres)}, - params={ - "catalogue": base, - "npatch": npatch, - "cat_config": str(cat_config), - "output_dir": str(out), - }, + draw_patch_centers( + [ + base, + str(npatch), + "--cat-config", + str(cat_config), + "--output-dir", + str(out), + ] ) return centres diff --git a/workflow/README.md b/workflow/README.md index ae94d5f6..fbca6c6e 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -165,10 +165,20 @@ mkdir -p /data ln -s /n17data/cdaley/unions/code/sp_validation/cosmo_inference/data/mask /data/ ``` -Jackknife patch centres are drawn once per base catalogue, by rule -`xi_patches` into `COSMO_VAL/patches/`, and every patched ξ± measurement of the -catalogue and its variants splits at them (in a notebook: -`CosmologyValidation.write_patch_centers`). +Jackknife patch centres are drawn once per base catalogue into +`COSMO_VAL/patches/`, and every patched ξ± measurement of the catalogue and its +variants splits at them. TreeCorr's k-means cannot be reproduced, so no rule +draws them: `-F` re-measures on the same centres, and a launch that needs +centres its tree lacks stops with the command that draws them, to run once on a +compute node: + +```bash +APPTAINERENV_PYTHONPATH=$PWD/src spv-container exec python -m sp_validation.cosmo_val.patch_centers \ + --cat-config cosmo_val/cat_config.yaml --output-dir +``` + +Or copy the file from the tree whose ξ± yours should match. The command never +replaces a file; to re-draw, delete it on purpose. Jobs draw their figures with matplotlib's defaults: `common.py` points each job's `MATPLOTLIBRC` at `workflow/matplotlibrc`, so your own matplotlibrc diff --git a/workflow/common.py b/workflow/common.py index 3345db8b..aa937fc8 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -114,7 +114,6 @@ def _load_checkout_module(name): # silent failures. Apply with: wildcard_constraints: **WILDCARD_CONSTRAINTS WILDCARD_CONSTRAINTS = { "version": r"SP_v[\d.]+(_w_iv)?(_ecut\d+)?(_leak_corr)?", - "catalogue": r"SP_v[\d.]+(_w_iv)?(_ecut\d+)?", "blind": r"[ABC]", "nbins": r"\d+", "min_sep": r"[0-9.]+", @@ -420,6 +419,30 @@ def patches_path(version, npatch): return str(COSMO_VAL / "patches" / name) +def patches_input(version, npatch): + """Rule xi's centres input for ``version`` with ``npatch``; none unpatched. + + No rule draws centres (``patch-centres-are-inputs``), so a missing file + stops the launch with the command that draws it. + """ + if int(npatch) <= 1: + return [] + path = patches_path(version, npatch) + if os.path.exists(path): + return path + from snakemake.exceptions import WorkflowError + + base = base_version(version) + raise WorkflowError( + f"{version} splits at {base}'s jackknife patch centres, {path}, which do " + "not exist; no rule draws them. Draw them once, on a compute node:\n" + f" APPTAINERENV_PYTHONPATH={REPO_SRC} spv-container exec python -m " + f"sp_validation.cosmo_val.patch_centers {base} {int(npatch)} " + f"--cat-config {CAT_CONFIG} --output-dir {COSMO_VAL}\n" + f"or copy {base}'s centres from the output tree whose ξ± yours should match." + ) + + def build_redshift_path(version, blind): """Construct n(z) filepath for given catalog version and blind.""" base = base_version(version) diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 35edf69d..65973c75 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -18,11 +18,12 @@ rule xi: One rule for every grid: outputs are named by their binning, so a request binds the wildcards and the grid label resolves from them. With patches, - the measurement splits at the base catalogue's persisted centres. + the measurement splits at the base catalogue's persisted centres, drawn by + hand (common.patches_input). """ input: catalog=lambda w: shear_catalog(w.version), - patches=lambda w: patches_path(w.version, w.npatch) if int(w.npatch) > 1 else [], + patches=lambda w: patches_input(w.version, w.npatch), output: sacc=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 24 @@ -46,29 +47,6 @@ rule xi: "../scripts/run_2pcf.py" -rule xi_patches: - """Jackknife patch centres of one base catalogue, drawn once for its variants. - - TreeCorr's k-means over the catalogue's positions and weights; they carry no - shear, so no custody. - """ - input: - catalog=lambda w: shear_catalog(w.catalogue), - output: - patches=str(COSMO_VAL / "patches" / "{catalogue}_npatch={npatch}.dat"), - threads: 12 - params: - catalogue="{catalogue}", - npatch="{npatch}", - cat_config=CAT_CONFIG, - output_dir=str(COSMO_VAL), - resources: - mem_mb=30000, - runtime=120, - script: - "../scripts/xi_patches.py" - - rule rho_tau_stats: output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index ded79da4..ca431a66 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -17,7 +17,7 @@ ξ± is born as a SACC part, named by its binning, tagged with its ``--grid`` and sealed under the catalogue's custody by ``CosmologyValidation.calculate_2pcf``; nothing else is written. With patches, the measurement splits at the base -catalogue's persisted centres (rule xi_patches). +catalogue's persisted centres (``python -m sp_validation.cosmo_val.patch_centers``). ``output_dir`` is passed explicitly so lc can point each run at its own ``{output}`` tree. diff --git a/workflow/scripts/xi_patches.py b/workflow/scripts/xi_patches.py deleted file mode 100644 index bd7b76bc..00000000 --- a/workflow/scripts/xi_patches.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Rule xi_patches: the jackknife patch centres of one base catalogue. - -TreeCorr's k-means over the catalogue's positions and weights, written once to -the declared output; every ξ± measurement of the catalogue and its variants -splits at these centres. -""" - -from cv_runner import _unbuffer_streams - -from sp_validation.cosmo_val import CosmologyValidation - -_unbuffer_streams() -p = snakemake.params -cv = CosmologyValidation( - versions=[p["catalogue"]], - catalog_config=p["cat_config"], - output_dir=p["output_dir"], -) -cv.write_patch_centers(p["catalogue"], int(p["npatch"]), snakemake.output["patches"]) diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py index 720e5ae6..45cbce25 100644 --- a/workflow/tests/conftest.py +++ b/workflow/tests/conftest.py @@ -15,9 +15,10 @@ ``cosmo_val/cat_config.yaml`` declaring one catalogue per custody state over a touched catalogue file, a blind registry of hand-written records (the host never decrypts, so no record needs a real seed), the processed CosmoCov -covariances already in place (their inputs live on candide), and both output -roots in tmp. Its runs use a fake image whose Python and Snakemake match the -running ones, so the launch-time parity check passes without apptainer. +covariances already in place (their inputs live on candide), the toy +catalogue's patch centres, and both output roots in tmp. Its runs use a fake +image whose Python and Snakemake match the running ones, so the launch-time +parity check passes without apptainer. """ import copy @@ -266,6 +267,14 @@ def toy(tmp_path_factory): path.parent.mkdir(parents=True, exist_ok=True) path.touch() + # The toy catalogue's patch centres, drawn by hand, in both output roots a + # launch may name. + npatch = grids["reporting"]["npatch"] + for cosmo_val in (Path(env["COSMO_VAL"]), root / "cosmo_val" / "output"): + centres = cosmo_val / "patches" / f"{VERSIONS[0]}_npatch={npatch}.dat" + centres.parent.mkdir(parents=True) + centres.touch() + return Toy( root=root, rundir=rundir, diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 000b18ad..3ab03e5c 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -99,9 +99,10 @@ def test_xi_leaves_a_measurement_only_as_a_part(toy): }, job.input -def test_patch_centres_are_an_input_shared_by_variants(toy): - """Every patched ξ± job splits at its base catalogue's centres, drawn once.""" - result = toy.snakemake("-n", "assemble_sacc_all") +def test_patch_centres_are_an_input_no_rule_draws(toy): + """Every patched ξ± job splits at its base catalogue's centres, and even a + forced run writes none: a re-draw cannot be reproduced.""" + result = toy.snakemake("-F", "-n", "assemble_sacc_all") assert result.returncode == 0, result.stdout jobs = parse_jobs(result.stdout) npatch = toy.common.xi_grids(toy.config, toy.config["fiducial"])["reporting"][ @@ -110,7 +111,6 @@ def test_patch_centres_are_an_input_shared_by_variants(toy): assert npatch > 1 centres = str(toy.cosmo_val / "patches" / f"{VERSIONS[0]}_npatch={npatch}.dat") - assert [j.output for j in jobs if j.rule == "xi_patches"] == [[centres]] xi = [j for j in jobs if j.rule == "xi"] assert {j.wildcards["version"] for j in xi} == set(VERSIONS) for job in xi: @@ -118,6 +118,26 @@ def test_patch_centres_are_an_input_shared_by_variants(toy): assert [f for f in job.input if "/patches/" in f] == ( [centres] if patched else [] ), job + drawn = [f for j in jobs for f in j.output if "/patches/" in f] + assert not drawn, drawn + + +def test_a_tree_without_centres_stops_the_launch(toy, tmp_path): + """A launch into a tree without the centres names the command that draws + them, for the base catalogue.""" + npatch = toy.common.xi_grids(toy.config, toy.config["fiducial"])["reporting"][ + "npatch" + ] + result = toy.snakemake( + "-n", "assemble_sacc_all", env={**toy.env, "COSMO_VAL": str(tmp_path)} + ) + plain = toy.common._plain + assert result.returncode != 0, result.stdout + assert ( + f"python -m sp_validation.cosmo_val.patch_centers {VERSIONS[0]} {npatch} " + f"--cat-config {plain(toy.root / 'cosmo_val' / 'cat_config.yaml')} " + f"--output-dir {plain(tmp_path)}" + ) in result.stdout, result.stdout @pytest.mark.parametrize("named", [True, False], ids=["named", "unnamed"]) @@ -177,8 +197,8 @@ def test_output_roots_take_the_plain_spelling(toy): tree / "inference", ) grids = toy.common.xi_grids(toy.config, toy.config["fiducial"]) - reporting = toy.common.grid_binning(grids["reporting"]) - target = tree / "val" / f"{VERSIONS[0]}_xi_{reporting}.sacc" + integration = toy.common.grid_binning(grids["integration"]) + target = tree / "val" / f"{VERSIONS[0]}_xi_{integration}.sacc" result = toy.snakemake("-n", str(target), env=env) assert result.returncode == 0, result.stdout declared = [f for j in parse_jobs(result.stdout) for f in j.input + j.output] @@ -377,9 +397,26 @@ def test_unblinding_a_concealed_catalogue_needs_the_reveal(toy): assert "blinding reveal stale" in result.stdout -def _real_dry_run(paper, targets): +def _stand_in_centres(paper, cosmo_val): + """Touch the centres ``paper``'s patched ξ± grids read; a dry-run reads none.""" + common = _load_module( + REPO / "workflow" / "common.py", f"{paper}_common", {"COSMO_VAL": cosmo_val} + ) + common.CATALOG_CONFIG = yaml.safe_load(Path(common.CAT_CONFIG).read_text()) + config = yaml.safe_load( + (REPO / "papers" / paper / "config" / "config.yaml").read_text() + ) + for grid in common.xi_grids(config, config["fiducial"]).values(): + for version in config["versions"] if grid["npatch"] > 1 else (): + centres = Path(common.patches_path(version, grid["npatch"])) + centres.parent.mkdir(parents=True, exist_ok=True) + centres.touch() + + +def _real_dry_run(paper, targets, cosmo_val): env = {k: v for k, v in os.environ.items() if k != "SNAKEMAKE_PROFILE"} - env.update(PYTHONUNBUFFERED="1", PYTHONNOUSERSITE="1") + env.update(PYTHONUNBUFFERED="1", PYTHONNOUSERSITE="1", COSMO_VAL=cosmo_val) + _stand_in_centres(paper, cosmo_val) return subprocess.run( [ sys.executable, @@ -410,13 +447,14 @@ def _real_dry_run(paper, targets): ], ids=["cosmo_val", "bmodes"], ) -def test_papers_resolve_on_candide(paper, targets): +def test_papers_resolve_on_candide(paper, targets, tmp_path): """The real paper DAGs resolve against the real catalogues and your image. The e-cut catalogue reads its parent's catalogue entry. Your image (the SIF - or sandbox `spv-container` manages) is read, so parity was checked. + or sandbox `spv-container` manages) is read, so parity was checked. The + products land in a tree of their own, holding stand-in patch centres. """ - result = _real_dry_run(paper, targets) + result = _real_dry_run(paper, targets, str(tmp_path)) assert result.returncode == 0, result.stdout assert "parity unchecked" not in result.stdout, ( "no local image was read; run `spv-container pull`\n" + result.stdout diff --git a/workflow/tests/test_toy_run.py b/workflow/tests/test_toy_run.py index 7a06c6d1..10e6b16c 100644 --- a/workflow/tests/test_toy_run.py +++ b/workflow/tests/test_toy_run.py @@ -4,14 +4,16 @@ host Snakemake, the image `spv-container` manages, jobs on this node. A toy checkout under your home directory (which the profile binds) carries copies of workflow/, papers/cosmo_val/ and src/, and one synthetic catalogue declared -unblinded. Its reporting parts and their figure are made; then the catalogue is -declared blinded, a blind is drawn for it, and the same launch re-measures the -parts concealed. The jobs run under a matplotlibrc asking for a LaTeX package -no image has, as a user's own may. +unblinded. The first launch stops for want of its patch centres and names the +command that draws them; after it has run, its reporting parts and their figure +are made; then the catalogue is declared blinded, a blind is drawn for it, and +the same launch re-measures the parts concealed. The jobs run under a +matplotlibrc asking for a LaTeX package no image has, as a user's own may. """ import json import os +import shlex import shutil import subprocess import sys @@ -151,11 +153,21 @@ def launch(*args): check=False, ) + # No centres: the launch names the command that draws them, which works. + result = launch() + assert result.returncode != 0, result.stdout + (command,) = [ + line.split("spv-container exec ", 1)[1] + for line in result.stdout.splitlines() + if "sp_validation.cosmo_val.patch_centers" in line + ] + _in_image(image, root, *shlex.split(command)) + assert (out / "patches" / "SP_v0.1_npatch=4.dat").is_file() + # Declared unblinded: parts stamped unblinded, and their figure drawn. result = launch() assert result.returncode == 0, result.stdout assert [s["blinding"] for s in _stamps(image, root, parts)] == ["unblinded"] * 2 - assert (out / "patches" / "SP_v0.1_npatch=4.dat").is_file() assert (out / "xi_p.png").is_file() # Declared blinded, and its blind drawn: the parts' params changed. From 85eefaef6348c1ef57a7dfefb766c340e3d05b05 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 01:42:16 +0200 Subject: [PATCH 108/160] workflow/tests: the candide profile's job bound is checked on any host A real launch through the committed candide profile, of an up-to-date target, submits nothing, so it runs in CI and on an allocation; the same launch through the profile without `jobs` is refused. test_container_smoke submits a real job and needs sbatch, which only a login node has; it is documented as the login-node check it is. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- workflow/README.md | 4 +- workflow/tests/test_container_smoke.py | 9 ++-- workflow/tests/test_dag.py | 60 +++++++++++++++++++++++--- 3 files changed, 62 insertions(+), 11 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index d60ad6b3..b6058026 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -303,7 +303,9 @@ uv run --isolated --no-project --python 3.12 --with snakemake==9.23.1 \ pytest workflow/tests ``` -CI runs the same suite with `-m "not candide"`. +CI runs the same suite with `-m "not candide"`. `test_container_smoke` +submits one real SLURM job through the candide profile, so it runs only where +`sbatch` exists — a candide login node — and skips on compute nodes. ### `snakemake` in `script:` files diff --git a/workflow/tests/test_container_smoke.py b/workflow/tests/test_container_smoke.py index 8045103d..faae81ad 100644 --- a/workflow/tests/test_container_smoke.py +++ b/workflow/tests/test_container_smoke.py @@ -4,8 +4,9 @@ bound come from that profile, launched as the README launches a target; the test Snakefile composes workflow/ as the entry Snakefiles do, so the job runs the image that launch resolves (your SIF or sandbox) in the environment that -launch hands its jobs. That contract is what's under test, so it runs only on a -candide submit host. +launch hands its jobs. That contract is what's under test, so it runs only where +``sbatch`` exists: a candide login node. Compute nodes have none, so in a job +step on an allocation it skips. The job writes a YAML report (see data/container_smoke/container_smoke.py); the assertions below check what it reports. @@ -36,7 +37,9 @@ def _reference_eigenvalues() -> np.ndarray: @pytest.mark.candide @on_candide -@pytest.mark.skipif(shutil.which("sbatch") is None, reason="needs a SLURM submit host") +@pytest.mark.skipif( + shutil.which("sbatch") is None, reason="submits a SLURM job: run on a login node" +) def test_container_smoke(): assert container.resolve_image()[1] != "tag", ( "no local image; run `spv-container pull`" diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index bbd21622..e2218f1e 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -137,6 +137,19 @@ def test_launch_reads_the_image_under_home(toy, tmp_path): assert "uv tool install --force --python 3.13 snakemake==" in result.stdout +def _apptainer_stub(tmp_path): + """A PATH entry that answers where apptainer is not installed. + + The candide profile deploys with apptainer, whose version Snakemake reads + even when it runs no job. + """ + apptainer = tmp_path / "bin" / "apptainer" + apptainer.parent.mkdir() + apptainer.write_text("#!/bin/sh\necho apptainer version 1.3.4\n") + apptainer.chmod(0o755) + return apptainer.parent + + def test_a_job_needs_no_launch_cache(toy, tmp_path): """A job starts on a node where the launch's XDG cache cannot exist. @@ -147,12 +160,6 @@ def test_a_job_needs_no_launch_cache(toy, tmp_path): """ blocker = tmp_path / "a-file" blocker.touch() - # The profile deploys with apptainer, whose version Snakemake reads even in - # a dry-run; this stub answers where apptainer is not installed. - apptainer = tmp_path / "bin" / "apptainer" - apptainer.parent.mkdir() - apptainer.write_text("#!/bin/sh\necho apptainer version 1.3.4\n") - apptainer.chmod(0o755) candide = toy.root / "workflow" / "profiles" / "candide" result = toy.snakemake( "-n", @@ -162,7 +169,7 @@ def test_a_job_needs_no_launch_cache(toy, tmp_path): env=toy.env | { "XDG_CACHE_HOME": str(blocker / "cache"), - "PATH": f"{apptainer.parent}{os.pathsep}{toy.env['PATH']}", + "PATH": f"{_apptainer_stub(tmp_path)}{os.pathsep}{toy.env['PATH']}", }, ) assert result.returncode == 0, result.stdout @@ -186,6 +193,45 @@ def test_a_job_needs_no_launch_cache(toy, tmp_path): assert (tmp_path / "env.txt").read_text() == "" +def test_the_candide_profile_bounds_its_jobs(tmp_path): + """A real launch through the candide profile needs no --jobs. + + Snakemake refuses a real run on a remote executor without a job bound. The + target is up to date, so the launch submits nothing and runs on any host; + the same launch through the profile stripped of its bound shows the refusal. + """ + (tmp_path / "Snakefile").write_text( + 'rule done:\n output: "done.txt"\n shell: "touch {output}"\n' + ) + (tmp_path / "done.txt").touch() + env = {k: v for k, v in os.environ.items() if k != "SNAKEMAKE_PROFILE"} + env["PATH"] = f"{_apptainer_stub(tmp_path)}{os.pathsep}{env['PATH']}" + candide = REPO / "workflow" / "profiles" / "candide" + unbounded = tmp_path / "unbounded" + unbounded.mkdir() + profile = yaml.safe_load((candide / "config.yaml").read_text()) + profile.pop("jobs", None) + (unbounded / "config.yaml").write_text(yaml.safe_dump(profile)) + + def launch(profile_dir): + return subprocess.run( + [sys.executable, "-m", "snakemake", "--profile", str(profile_dir)], + cwd=tmp_path, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=300, + check=False, + ) + + result = launch(candide) + assert result.returncode == 0, result.stdout + assert "Nothing to be done" in result.stdout, result.stdout + refused = launch(unbounded) + assert refused.returncode != 0 and "--jobs" in refused.stdout, refused.stdout + + def test_image_sims_checks_parity_at_launch(toy, tmp_path): """The standalone image-sims workflow stops on a mismatched image too.""" run = { From 37d5a66ff0c9e44526f09a911d4590465e8f2835 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 01:49:31 +0200 Subject: [PATCH 109/160] sacc_io: under a blind, every data type needs a blinding rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Which rows a blinded catalogue's SACC may hold is an allow-list: ξ± and Cℓ_EE (shifted), Cℓ_BB and Cℓ_EB (unshifted), COSEBIs and pure-E/B (derived) and ρ/τ (no signal). A data type in none of these, such as galaxy–galaxy lensing γ_t or CMB κ × shear, is refused at birth and in a blinded derivation, instead of being stamped blinded with its true values because its name does not start with galaxy_shear_. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/sacc_io.py | 54 +++++++++++++++++------- src/sp_validation/tests/test_blinding.py | 46 ++++++++++++++++++++ workflow/README.md | 4 +- 3 files changed, 88 insertions(+), 16 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 5a8750a9..4c9529c3 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -87,6 +87,7 @@ import functools import os +import re import types import numpy as np @@ -975,26 +976,44 @@ def _check_grid_consistency(s, angle): # --------------------------------------------------------------------------- # # The file door: every SACC is born sealed, derived with one stamp, or refused # --------------------------------------------------------------------------- # -# Signal: the shear data types a cosmology shift reaches. -SIGNAL_PREFIX = "galaxy_shear_" -# The signal rows a blind conceals. +# The blinding rule of each data type a blinded catalogue's SACC may hold. A +# data type with none is refused under a blind, so a new statistic fails closed. +# Signal the blind shifts: SHIFTABLE = (XI_PLUS, XI_MINUS, CL_EE) -# Signal rows a pure E-mode shift leaves unchanged. +# signal a pure E-mode shift leaves unchanged: UNSHIFTED = (CL_BB, CL_EB) +# signal derived from ξ±, which moves only through the rows it is computed from: +DERIVED = (COSEBI_EE, COSEBI_BB, *PURE_TYPES.values()) +SIGNAL = SHIFTABLE + UNSHIFTED + DERIVED +# and no signal: the PSF's ρ and the galaxy–PSF τ statistics. +_SIGNAL_FREE = re.compile( + "|".join(t.format(k=r"\d+") for t in (RHO_PLUS, RHO_MINUS, TAU_PLUS, TAU_MINUS)) +) def is_signal(data_type): """Whether ``data_type`` carries cosmological signal.""" - return data_type.startswith(SIGNAL_PREFIX) + return data_type in SIGNAL def is_derived(data_type): - """Whether ``data_type`` is a derived statistic (COSEBIs, pure-E/B, …). + """Whether ``data_type`` is a derived statistic (COSEBIs, pure-E/B).""" + return data_type in DERIVED - Signal the blind neither conceals nor leaves unchanged: it moves only - through the shiftable rows it is computed from. - """ - return is_signal(data_type) and data_type not in SHIFTABLE + UNSHIFTED + +def _refuse_unruled(s): + """Refuse rows of a data type with no blinding rule (under a blind).""" + unruled = sorted( + t + for t in {dp.data_type for dp in s.data} + if not is_signal(t) and not _SIGNAL_FREE.fullmatch(t) + ) + if unruled: + raise ValueError( + f"{unruled}: no blinding rule for these data types, so a blinded " + "catalogue's SACC cannot hold them; give a new statistic its rule in " + "sacc_io (and its shift in blinding) first" + ) def _stamped(s): @@ -1013,16 +1032,18 @@ def seal(s, custody): @sc born-sealed A catalogue-born SACC leaves memory only through here. Under a blinded custody every ξ± and Cℓ_EE row is shifted on a copy before the stamp is - minted; a SACC with no signal is stamped without opening the blind; any - other signal (COSEBIs, pure-E/B) is a derived statistic, refused here and - saved with ``derived_from``. An already-stamped SACC is re-written only as - a derivation. + minted; a SACC with no signal (ρ/τ) is stamped without opening the blind; a + derived statistic (COSEBIs, pure-E/B) is refused here and saved with + ``derived_from``; a data type with no blinding rule is refused. An + already-stamped SACC is re-written only as a derivation. """ if _stamped(s): raise ValueError( "this SACC is already stamped; a loaded or sealed SACC is re-written " "only as a derivation (save(..., derived_from=[...]))" ) + if custody.status == "blinded": + _refuse_unruled(s) types = {dp.data_type for dp in s.data} derived = {t for t in types if is_derived(t)} if custody.status == "blinded" and derived: @@ -1061,6 +1082,8 @@ def _derive(s, parts, custody): f"parts are stamped {stamp.token}, but {custody.catalogue} is " f"declared {custody.token}" ) + if stamp.status == "blinded": + _refuse_unruled(s) inputs = {_row_key(dp) for p in parts for dp in p.data if dp.data_type in SHIFTABLE} stray = [ i @@ -1094,7 +1117,8 @@ def save(s, path, *, custody=None, derived_from=None): @sc derived-inherit A derivation's inputs must share one stamp, and each of its ξ±/Cℓ_EE rows must be a copy of an input row (type, tracers, value, tags; windows by - index), so plaintext cannot be saved under a concealed stamp. + index), so plaintext cannot be saved under a concealed stamp; under a + blinded stamp every data type needs a blinding rule, as at birth. """ if derived_from is not None: out = _derive(s, list(derived_from), custody) diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 9dd46df9..ab23174d 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -484,6 +484,52 @@ def refuse(custody): assert cu.read_stamp(sealed.metadata).stamp == blinds["TOY"].stamp +# Data types no blind has a rule for, galaxy–galaxy lensing and CMB κ × shear, +# with the tag SACC requires of each. +UNRULED = { + "galaxy_shearDensity_xi_t": "theta", + "cmbGalaxy_convergenceShear_cl_e": "ell", +} + + +def _with_unruled(s, data_type): + for x in (5.0, 20.0, 80.0): + s.add_data_point( + data_type, ("source_0", "source_0"), 1e-5, **{UNRULED[data_type]: x} + ) + return s + + +@pytest.mark.parametrize("data_type", UNRULED) +def test_a_data_type_without_a_blinding_rule_is_refused_under_a_blind( + blinds, tmp_path, data_type +): + """Under a blind, a data type is born or derived only with a blinding rule. + + Unblinded and mock SACCs are stamped whatever they hold. + """ + for version in ("TOY_OPEN", "TOY_MOCK"): + s = _with_unruled(two_bin_sacc(cl=False, covariance=False), data_type) + assert np.array_equal( + np.asarray(sio.seal(s, blinds[version]).mean), np.asarray(s.mean) + ) + with pytest.raises(ValueError, match="no blinding rule"): + sio.seal( + _with_unruled(two_bin_sacc(cl=False, covariance=False), data_type), + blinds["TOY"], + ) + xi = sio.seal(two_bin_sacc(cl=False, covariance=False), blinds["TOY"]) + for custody in (None, blinds["TOY"]): + with pytest.raises(ValueError, match="no blinding rule"): + sio.save( + _with_unruled(xi.copy(), data_type), + tmp_path / "x.sacc", + derived_from=[xi], + custody=custody, + ) + assert not list(tmp_path.iterdir()) + + def test_a_stamped_sacc_is_not_born_again(blinds, tmp_path): sealed = sio.seal(two_bin_sacc(cl=False), blinds["TOY_OPEN"]) with pytest.raises(ValueError, match="already stamped"): diff --git a/workflow/README.md b/workflow/README.md index c69ec40a..19fa7747 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -38,7 +38,9 @@ For a blinded catalogue, every ξ± and pseudo-Cℓ_EE value is shifted by a hid cosmology before it is first written. COSEBIs and pure-E/B computed from the shifted ξ± carry the blind with them; B-modes stay usable. Every SACC file records the custody it was born under, and assembly refuses parts under any -other. ξ± leaves `CosmologyValidation.calculate_2pcf` only as its sealed part +other. Under a blind a SACC holds only data types with a blinding rule +(`sacc_io.SIGNAL` and ρ/τ), so a new statistic is refused until it has one. ξ± +leaves `CosmologyValidation.calculate_2pcf` only as its sealed part (no text dump), so the ξ± figures and notebooks draw what the part holds; the jackknife `calculate_pure_eb` and the aperture mass, which need TreeCorr's own measurement, refuse a blinded catalogue. From 29cb3f9af5ee67c808cf18bc36b03e53f094652b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 02:06:59 +0200 Subject: [PATCH 110/160] custody: a blind conceals data, whatever entry names it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Catalogues reading one shear file are concealed under one blind or none. custody_of refuses a catalogue whose file another catalogue reads under another custody, and init and share refuse to draw or extend a blind that would leave such a twin behind, naming the twins and the fix: draw the blind for them together, share it, or make them variants with `base:`. On the committed config, declaring SP_v1.4.6.3 blinded now stops init on its six twins (_A, _B, _C and the three _uncal entries) instead of leaving them to measure its ξ± in plaintext. An entry named as a variant (`_leak_corr`, `_seed`) shares its parent's custody, as the variant it is named for does; it may repeat the parent's `blinding` or `base`, as overwrite_config's copies do, and a declaration of its own is refused instead of ignored. The suffixes strip in either order, as CosmologyValidation composes them, and the workflow reads a version's shear file through the same resolution. Test fixtures give each catalogue its own copy of the galaxies. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j --- src/sp_validation/blinding.py | 45 ++++--- src/sp_validation/custody.py | 125 ++++++++++++++++++-- src/sp_validation/tests/_synthetic.py | 17 +-- src/sp_validation/tests/test_blinding.py | 26 ++++ src/sp_validation/tests/test_cosmo_val.py | 2 +- src/sp_validation/tests/test_custody.py | 68 +++++++++++ src/sp_validation/tests/test_custody_e2e.py | 2 +- workflow/README.md | 6 +- workflow/common.py | 7 +- workflow/tests/conftest.py | 62 +++++----- 10 files changed, 283 insertions(+), 77 deletions(-) diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index fe80e756..68045fa4 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -364,21 +364,31 @@ def _theory_stack(): return stack -def _declared_blinded(catalogues, registry, base): - """Refuse a base ``init`` or ``share`` cannot give a blind.""" - if _custody.base_catalogue(catalogues, base) != base: - raise _custody.CustodyError( - f"{base} is a variant of {_custody.base_catalogue(catalogues, base)}; " - "a blind covers base catalogues" - ) - declared = _custody.declaration(catalogues, base) - if declared in ("unblinded", "mock"): - raise _custody.CustodyError( - f"{base} is declared {declared}; declare it blinded first" - ) - covering = [r.name for r in _custody.records(registry).values() if base in r.bases] - if covering: - raise _custody.CustodyError(f"{base} is already covered by blind {covering[0]}") +def _may_cover(catalogues, recs, name, bases): + """Refuse ``bases`` blind ``name`` cannot cover: a variant, a catalogue + declared unblinded or mock or covered already, or one whose shear file + another catalogue would go on reading otherwise concealed.""" + for base in bases: + if _custody.base_catalogue(catalogues, base) != base: + raise _custody.CustodyError( + f"{base} is a variant of " + f"{_custody.base_catalogue(catalogues, base)}; a blind covers base " + "catalogues" + ) + declared = _custody.declaration(catalogues, base) + if declared in ("unblinded", "mock"): + raise _custody.CustodyError( + f"{base} is declared {declared}; declare it blinded first" + ) + covering = [r.name for r in recs.values() if base in r.bases] + if covering: + raise _custody.CustodyError( + f"{base} is already covered by blind {covering[0]}" + ) + covered = recs[name].bases if name in recs else () + after = {**recs, name: _custody.Record(name, {}, (*covered, *bases), None)} + for base in bases: + _custody.refuse_twins(catalogues, after, base) def _conceal_one_row(name, seed, record): @@ -418,8 +428,7 @@ def init(name, bases, *, cat_config, config=None): f"{path} exists; a blind is drawn once (delete a staging " "directory an interrupted init left)" ) - for base in bases: - _declared_blinded(catalogues, registry, base) + _may_cover(catalogues, _custody.records(registry), name, bases) seed = secrets.token_hex(16) key = Fernet.generate_key() @@ -462,7 +471,7 @@ def share(name, base, *, cat_config): raise _custody.CustodyError(f"no blind {name} in {registry}") if records[name].revealed is not None: raise _custody.CustodyError(f"blind {name} is revealed; draw a new one") - _declared_blinded(_catalogues(cat_config), registry, base) + _may_cover(_catalogues(cat_config), records, name, [base]) with open(registry / name / "bases", "a") as f: f.write(f"{base}\n") print(f"[blinding] {base} shares blind {name}; commit {registry / name / 'bases'}") diff --git a/src/sp_validation/custody.py b/src/sp_validation/custody.py index 0bcc6369..0be80614 100644 --- a/src/sp_validation/custody.py +++ b/src/sp_validation/custody.py @@ -7,6 +7,7 @@ import hashlib import json +import os import re from dataclasses import dataclass from pathlib import Path @@ -84,20 +85,55 @@ def registry_of(cat_config): return Path(cat_config).absolute().parent / "blinds" -def entry_of(version): - """The catalogue-config entry describing ``version``'s data. +def _variant_chain(version): + """``version``, then each name stripping a trailing ``_leak_corr`` or + ``_seed`` leaves: the variants CosmologyValidation materialises from an + entry, in any order.""" + chain = [version] + while True: + name = chain[-1] + if name.endswith(_LEAK_SUFFIX): + stripped = name[: -len(_LEAK_SUFFIX)] + else: + stripped = _SEED_SUFFIX.sub("", name) + if stripped in ("", name): + return chain + chain.append(stripped) + + +def base_catalogue(catalogues, version): + """The base catalogue of ``version``: the entry it is a variant of, or its + own, then that entry's ``base:`` links. - Strips the variants CosmologyValidation materialises from an entry: - ``_leak_corr``, then ``_seed``. + A config entry named as a variant of another (``_leak_corr``, + ``_seed``) shares that entry's custody, as the variant it is + named for would; it may repeat that entry's ``blinding`` or ``base``, never + declare another. """ - if version.endswith(_LEAK_SUFFIX): - version = version[: -len(_LEAK_SUFFIX)] - return _SEED_SUFFIX.sub("", version) + chain = _variant_chain(version) + base = _follow_base(catalogues, chain[-1], version) + for alias in chain[:-1]: + entry = catalogues.get(alias) + if not isinstance(entry, dict): + continue + status = declaration(catalogues, base) or "blinded" + if "base" in entry and _follow_base(catalogues, entry["base"], alias) != base: + claim = f"base: {entry['base']}" + elif entry.get("blinding", status) != status: + claim = f"blinding: {entry['blinding']}" + else: + continue + raise CustodyError( + f"{alias} is a variant of {chain[-1]}, so it shares the custody of " + f"{base} ({status}), but declares {claim}; declare custody on {base}, " + f"or rename {alias} to make it a catalogue of its own" + ) + return base -def base_catalogue(catalogues, version): - """The base catalogue of ``version``: its entry, then its ``base:`` links.""" - name, seen = entry_of(version), [] +def _follow_base(catalogues, name, version): + """The entry the ``base:`` links from ``name`` end at.""" + seen = [] while True: if name in NOT_CATALOGUES: raise CustodyError(f"{name} is not a catalogue") @@ -161,6 +197,65 @@ def records(registry): return found +def _reads(entry): + """The shear catalogue file a catalogue entry reads, if it names one.""" + shear = entry.get("shear") if isinstance(entry, dict) else None + if not isinstance(shear, dict) or "path" not in shear: + return None + return os.path.normpath(os.path.join(str(entry.get("subdir", "")), shear["path"])) + + +def shear_file(catalogues, version): + """The shear catalogue file ``version`` reads: that of the first entry of + its variant chain the config holds, as CosmologyValidation reads it.""" + chain = _variant_chain(version) + return _reads(catalogues[next((n for n in chain if n in catalogues), chain[-1])]) + + +def _concealer(catalogues, recs, base): + """What conceals ``base``'s data: the unrevealed blind covering it; ``""`` + when it is declared blinded and none does; ``None`` when it is public.""" + covering = sorted( + r.name for r in recs.values() if base in r.bases and not r.revealed + ) + if covering: + return ", ".join(covering) + return None if declaration(catalogues, base) in ("unblinded", "mock") else "" + + +def refuse_twins(catalogues, recs, version): + """Refuse when a catalogue reading ``version``'s shear file is concealed + otherwise than ``version`` is, under the blind records ``recs``. + + A blind conceals data, not a name: every catalogue reading one shear file + is concealed under one blind, or none is. + """ + base = base_catalogue(catalogues, version) + here = shear_file(catalogues, version) + readers = { + name: base_catalogue(catalogues, name) + for name, entry in catalogues.items() + if here is not None and name not in NOT_CATALOGUES and _reads(entry) == here + } + hiding = { + b: _concealer(catalogues, recs, b) + for b in dict.fromkeys([base, *readers.values()]) + } + twins = [name for name, b in readers.items() if hiding[b] != hiding[base]] + if twins: + states = "; ".join( + f"{b}: {'public' if c is None else f'blind {c}' if c else 'no blind yet'}" + for b, c in hiding.items() + ) + raise CustodyError( + f"{version} and {', '.join(twins)} read one shear file, {here}, but are " + f"not concealed alike ({states}). Catalogues reading one file share " + "one blind: declare them blinded and draw it for them together " + "(blinding init) or share it (blinding share), or make them variants " + "of one base with `base:`" + ) + + def _no_blind(version, base, declared, registry): why = ( "declared blinded" @@ -196,7 +291,8 @@ def custody_of(catalogues, version, *, registry): change it. An entry declaring nothing is blinded, so a new catalogue fails closed. Variants share their base's custody and blind: ``_leak_corr`` and ``_seed`` versions, and entries naming their parent with ``base:``. A - blinded base is covered by exactly one blind. + blinded base is covered by exactly one blind, which also conceals every + catalogue reading its shear file (:func:`refuse_twins`). Raises ------ @@ -204,12 +300,15 @@ def custody_of(catalogues, version, *, registry): With the one thing to do, when the declaration and the registry disagree: a blinded catalogue with no blind, a revealed blind still declared blinded, an unblinded catalogue under a concealed blind, a - mock under a blind, or two blinds over one catalogue. + mock under a blind, two blinds over one catalogue, or catalogues + reading one shear file concealed otherwise. """ registry = Path(registry) base = base_catalogue(catalogues, version) declared = declaration(catalogues, base) - covering = [r for r in records(registry).values() if base in r.bases] + recs = records(registry) + refuse_twins(catalogues, recs, version) + covering = [r for r in recs.values() if base in r.bases] if len(covering) > 1: names = ", ".join(r.name for r in covering) raise CustodyError(f"{base} is covered by several blinds: {names}") diff --git a/src/sp_validation/tests/_synthetic.py b/src/sp_validation/tests/_synthetic.py index fcd16e99..9c6bef7e 100644 --- a/src/sp_validation/tests/_synthetic.py +++ b/src/sp_validation/tests/_synthetic.py @@ -4,7 +4,8 @@ (RA/Dec/e1/e2/w), a PSF star catalogue with the columns the leakage and ρ/τ seams read, a cs_util-readable dndz, and a ``cat_config.yaml`` beside a ``blinds/`` registry directory, as in the repository layout. Every catalogue -entry in the config describes the same files and declares its own custody. +entry in the config reads its own copy of the same galaxies and declares its +own custody. """ import copy @@ -35,9 +36,10 @@ def write_synthetic_catalogs( Add a ``psf`` block (ρ/τ and pseudo-Cℓ read it via ``get_params_rho_tau``). catalogues : dict, optional - ``{version: declaration}``, one catalogue entry each over the same - files. A declaration is a ``blinding`` value, or ``None`` for an entry - that declares none. Defaults to one mock, ``TestCatalog``. + ``{version: declaration}``, one catalogue entry each, reading its own + copy of the shear catalogue (catalogues reading one file share one + custody). A declaration is a ``blinding`` value, or ``None`` for an + entry that declares none. Defaults to one mock, ``TestCatalog``. Returns ------- @@ -65,9 +67,7 @@ def write_synthetic_catalogs( e1 = rng.normal(0, 0.25, n_gal) e2 = rng.normal(0, 0.25, n_gal) w = rng.uniform(0.5, 1.0, n_gal) - Table({"RA": ra, "Dec": dec, "e1": e1, "e2": e2, "w": w}).write( - cat_dir / "shear.fits", overwrite=True - ) + shear = Table({"RA": ra, "Dec": dec, "e1": e1, "e2": e2, "w": w}) Table( { @@ -98,7 +98,6 @@ def write_synthetic_catalogs( "ls": "-", "marker": "o", "shear": { - "path": "shear.fits", "redshift_path": str(nz_dir / "dndz_SP_A.txt"), "w_col": "w", "e1_col": "e1", @@ -137,6 +136,8 @@ def write_synthetic_catalogs( } for version, declaration in catalogues.items(): config[version] = copy.deepcopy(entry) + config[version]["shear"]["path"] = f"shear_{version}.fits" + shear.write(cat_dir / f"shear_{version}.fits", overwrite=True) if declaration is not None: config[version]["blinding"] = declaration config_path = tmp_path / "cat_config.yaml" diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index ab23174d..d18faecb 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -823,6 +823,32 @@ def test_share_adds_a_base_to_a_concealed_blind(fresh): bd.share("toy", "LATER", cat_config=cat_config) +def test_a_blind_is_drawn_for_every_catalogue_reading_its_file(tmp_path): + """init and share refuse a catalogue whose shear file another catalogue + reads under another custody, before anything is written; drawn for both, + the blind conceals both.""" + _cat_config(tmp_path) + toy, later = ({"shear": {"path": str(tmp_path / f)}} for f in ("a.fits", "b.fits")) + _declare( + tmp_path, + TOY=toy, + TOY_TWIN={**toy, "blinding": "unblinded"}, + LATER=later, + LATER_TWIN={**later, "blinding": "unblinded"}, + ) + with pytest.raises(cu.CustodyError, match="TOY_TWIN"): + _init(tmp_path, "toy", "TOY") + assert not (tmp_path / "blinds").exists() + + _declare(tmp_path, TOY_TWIN=toy) + _init(tmp_path, "toy", "TOY", "TOY_TWIN") + assert _custody(tmp_path, "TOY_TWIN").blind == "toy" + with pytest.raises(cu.CustodyError, match="LATER_TWIN"): + bd.share("toy", "LATER", cat_config=str(tmp_path / "cat_config.yaml")) + bases = (tmp_path / "blinds" / "toy" / "bases").read_text().split() + assert bases == ["TOY", "TOY_TWIN"] + + # --------------------------------------------------------------------------- # # I10: an opened blind is the one committed # --------------------------------------------------------------------------- # diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 1f89236d..f3d932e2 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -547,7 +547,7 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( # --------------------------------------------------------------------------- # @pytest.fixture def blinded_and_twin(tmp_path): - """TOY, blinded under `toy`, and TOY_OPEN: the same files, unblinded.""" + """TOY, blinded under `toy`, and TOY_OPEN: the same galaxies, unblinded.""" import json from sp_validation import blinding diff --git a/src/sp_validation/tests/test_custody.py b/src/sp_validation/tests/test_custody.py index be80e0d2..86671144 100644 --- a/src/sp_validation/tests/test_custody.py +++ b/src/sp_validation/tests/test_custody.py @@ -182,12 +182,80 @@ def test_variants_share_their_base(tmp_path): "SP_v9_leak_corr", "SP_v9_seed00042", "SP_v9_seed00042_leak_corr", + "SP_v9_leak_corr_seed00042", "SP_v9_ecut07", "SP_v9_ecut07_leak_corr", ): assert cu.custody_of(cats, version, registry=registry) == base, version +def test_an_entry_named_as_a_variant_shares_its_parents_custody(tmp_path): + """An entry named `_leak_corr` or `_seed` resolves through + that entry, as the variant it is named for does. It may repeat the entry's + `blinding` or `base`, as the copies `overwrite_config` writes do, but a + declaration of its own is refused, never ignored.""" + registry = tmp_path / "blinds" + _write_blind(registry, "y3", ["TOY"]) + cats = _catalogues(PARENT="unblinded", TOY=None, EC={"base": "PARENT"}) + copies = dict( + cats, PARENT_leak_corr=dict(cats["PARENT"]), EC_seed7=dict(cats["EC"]) + ) + for version in ("PARENT_leak_corr", "EC_seed7"): + custody = cu.custody_of(copies, version, registry=registry) + assert custody.token == "unblinded:PARENT", version + for alias, claim in { + "PARENT_leak_corr": {"blinding": "blinded"}, + "PARENT_seed7": {"base": "TOY"}, + }.items(): + with pytest.raises(cu.CustodyError, match="declare custody on PARENT"): + cu.custody_of({**cats, alias: claim}, alias, registry=registry) + + +def _reading(cats, path, *names): + """Point ``names``' entries at one shear file, spelled two ways.""" + for i, name in enumerate(names): + cats[name]["shear"]["path"] = path.name if i % 2 else str(path) + cats[name]["subdir"] = str(path.parent) + return cats + + +def test_catalogues_reading_one_file_share_one_blind(tmp_path): + """A blind conceals a shear file, whatever entry reads it: a catalogue + reading a concealed catalogue's file under any other custody is refused, + on either side, until the blind covers it too.""" + registry = tmp_path / "blinds" + seed = _write_blind(registry, "y3", ["TOY"]) + cats = _catalogues(TOY=None, TOY_A="unblinded", TOY_V={"base": "TOY"}) + _reading(cats, tmp_path / "toy.fits", "TOY", "TOY_A", "TOY_V") + for version in ("TOY_A", "TOY", "TOY_leak_corr"): + with pytest.raises(cu.CustodyError, match="not concealed alike") as err: + cu.custody_of(cats, version, registry=registry) + assert "TOY: blind y3" in str(err.value), version + assert "TOY_A: public" in str(err.value), version + + two = _reading( + _catalogues(TOY=None, TOY_B=None), tmp_path / "toy.fits", "TOY", "TOY_B" + ) + _write_blind(tmp_path / "two", "y3", ["TOY"]) + _write_blind(tmp_path / "two", "y4", ["TOY_B"]) + with pytest.raises(cu.CustodyError, match="TOY: blind y3; TOY_B: blind y4"): + cu.custody_of(two, "TOY", registry=tmp_path / "two") + + mock = _catalogues(OPEN="unblinded", OPEN_MOCK="mock") + _reading(mock, tmp_path / "open.fits", "OPEN", "OPEN_MOCK") + assert cu.custody_of(mock, "OPEN", registry=registry).status == "unblinded" + + del cats["TOY_A"]["blinding"] + (registry / "y3" / "bases").write_text("TOY\nTOY_A\n") + for version in ("TOY", "TOY_A", "TOY_V"): + assert cu.custody_of(cats, version, registry=registry).blind == "y3" + + (registry / "y3" / "revealed.json").write_text(json.dumps({"seed": seed})) + cats["TOY"]["blinding"] = "unblinded" + cats["TOY_A"]["blinding"] = "unblinded" + assert cu.custody_of(cats, "TOY_A", registry=registry).status == "unblinded" + + def test_base_links_are_followed_and_checked(): cats = _catalogues( A="unblinded", B={"base": "A"}, C={"base": "B"}, D={"base": "nowhere"} diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py index fa62b476..57394c9e 100644 --- a/src/sp_validation/tests/test_custody_e2e.py +++ b/src/sp_validation/tests/test_custody_e2e.py @@ -429,7 +429,7 @@ def test_a_blinded_catalogue_from_birth_to_audit(toy, monkeypatch): written = sio.save(two_bin, out / "two_bin.sacc", custody=blinded) assert np.all(np.asarray(written.mean) != np.asarray(two_bin.mean)) - # The true vector of TOY is TOY_OPEN's, the same files declared unblinded. + # The true vector of TOY is TOY_OPEN's, the same galaxies declared unblinded. open_parts = run_chain( toy.cat_config, "TOY_OPEN", toy.root / "open", toy.cov, grid_parts="only" ) diff --git a/workflow/README.md b/workflow/README.md index 19fa7747..561174f4 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -30,7 +30,11 @@ each namespaces cleanly under `results//`. Every catalogue in `cosmo_val/cat_config.yaml` declares its custody on its base entry, `blinding: blinded | unblinded | mock`; an entry that declares none is blinded. Variants (`_leak_corr`, `_seed`, and entries naming their parent -with `base:`) share their base's custody and blind. A launch prints one +with `base:`) share their base's custody and blind; an entry named like a +variant is one, and may repeat its parent's `blinding` or `base`, never declare +another. A blind conceals data, not a name: catalogues reading one shear file +are concealed under one blind or none, so a blind is drawn (or shared) for all +of them. A launch prints one `[custody]` line per catalogue it runs; no `--config` or config file changes a declaration. diff --git a/workflow/common.py b/workflow/common.py index aa937fc8..ef9c41f3 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -531,12 +531,7 @@ def pseudo_cl_tag(config): def shear_catalog(version): """The shear catalogue file of ``version``, from its catalogue entry.""" - cat_config = CATALOG_CONFIG[_custody.entry_of(version)] - shear_path = cat_config["shear"]["path"] - if shear_path.startswith("/"): - return shear_path - subdir = cat_config.get("subdir", "") - return str(Path(subdir) / shear_path) + return _custody.shear_file(CATALOG_CONFIG, version) # --------------------------------------------------------------------------- diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py index 45cbce25..dbe1c21b 100644 --- a/workflow/tests/conftest.py +++ b/workflow/tests/conftest.py @@ -12,8 +12,8 @@ The ``toy`` fixture is a disposable checkout: copies of ``workflow/`` and ``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, a -``cosmo_val/cat_config.yaml`` declaring one catalogue per custody state over a -touched catalogue file, a blind registry of hand-written records (the host +``cosmo_val/cat_config.yaml`` declaring one catalogue per custody state, each +over its own touched catalogue file, a blind registry of hand-written records (the host never decrypts, so no record needs a real seed), the processed CosmoCov covariances already in place (their inputs live on candide), the toy catalogue's patch centres, and both output roots in tmp. Its runs use a fake @@ -21,7 +21,6 @@ parity check passes without apptainer. """ -import copy import dataclasses import hashlib import importlib.util @@ -157,29 +156,36 @@ def snakemake( ) -def _cat_config(catalogue): - entry = { - "subdir": str(catalogue.parent), - "pipeline": "SP", - "colour": "orange", - "marker": "^", - "cov_th": {"A": 100.0, "n_e": 5.0, "sigma_e": 0.3}, - "shear": { - "path": str(catalogue), - "redshift_path": str(catalogue.parent / "nz_SP_v0.1_A.txt"), - "w_col": "w", - "e1_col": "e1", - "e2_col": "e2", - "e1_col_corrected": "e1_leak_corrected", - "e2_col_corrected": "e2_leak_corrected", - }, - } +def _cat_config(data): + """One catalogue per custody state, each reading its own touched file.""" + + def entry(name, **declaration): + catalogue = data / f"{name}.fits" + catalogue.touch() + return { + "subdir": str(data), + "pipeline": "SP", + "colour": "orange", + "marker": "^", + "cov_th": {"A": 100.0, "n_e": 5.0, "sigma_e": 0.3}, + "shear": { + "path": str(catalogue), + "redshift_path": str(data / "nz_SP_v0.1_A.txt"), + "w_col": "w", + "e1_col": "e1", + "e2_col": "e2", + "e1_col_corrected": "e1_leak_corrected", + "e2_col_corrected": "e2_leak_corrected", + }, + **declaration, + } + return { - VERSIONS[0]: entry, - "SP_v0.2": dict(copy.deepcopy(entry), blinding="unblinded"), - "SP_v0.3": dict(copy.deepcopy(entry), blinding="mock"), - UNCOVERED: copy.deepcopy(entry), - STALE: dict(copy.deepcopy(entry), blinding="unblinded"), + VERSIONS[0]: entry(VERSIONS[0]), + "SP_v0.2": entry("SP_v0.2", blinding="unblinded"), + "SP_v0.3": entry("SP_v0.3", blinding="mock"), + UNCOVERED: entry(UNCOVERED), + STALE: entry(STALE, blinding="unblinded"), "paths": {"output": "./output"}, } @@ -212,12 +218,10 @@ def toy(tmp_path_factory): ) (root / "src").symlink_to(REPO / "src") - catalogue = root / "data" / "toy_shear.fits" - catalogue.parent.mkdir() - catalogue.touch() + (root / "data").mkdir() (root / "cosmo_val").mkdir() (root / "cosmo_val" / "cat_config.yaml").write_text( - yaml.safe_dump(_cat_config(catalogue)) + yaml.safe_dump(_cat_config(root / "data")) ) _registry(root) From f2eab2f02c9d8d354a470293fead796346974528 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 02:39:42 +0200 Subject: [PATCH 111/160] profiles: wait 60 s for a job's outputs to appear P5 (one real SLURM job through the candide profile, from a login node) passed, but only on its retry: the job finished, its output took more than 5 s to show on the login node's /home, and Snakemake re-ran it. On a multi-hour job that retry costs hours, and a second miss fails the run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- workflow/profiles/candide/config.yaml | 6 ++++-- workflow/profiles/default/config.yaml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 01230785..466d7cd2 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -20,8 +20,10 @@ software-deployment-method: apptainer rerun-triggers: ["mtime", "params", "input", "code"] # Give an appearing output file a moment on networked filesystems before -# Snakemake calls a job failed for a missing output. -latency-wait: 5 +# Snakemake calls a job failed for a missing output. A job's write can take +# more than 5 s to show on the launching host (candide's /home does); a +# shorter wait reads a finished job as failed and re-runs it. +latency-wait: 60 # --- end GENERIC ------------------------------------------------------------ # No ``apptainer-prefix``: the entry Snakefiles resolve ``container:`` to this diff --git a/workflow/profiles/default/config.yaml b/workflow/profiles/default/config.yaml index bf272c03..cd42355a 100644 --- a/workflow/profiles/default/config.yaml +++ b/workflow/profiles/default/config.yaml @@ -25,8 +25,10 @@ software-deployment-method: apptainer rerun-triggers: ["mtime", "params", "input", "code"] # Give an appearing output file a moment on networked filesystems before -# Snakemake calls a job failed for a missing output. -latency-wait: 5 +# Snakemake calls a job failed for a missing output. A job's write can take +# more than 5 s to show on the launching host (candide's /home does); a +# shorter wait reads a finished job as failed and re-runs it. +latency-wait: 60 # --- end GENERIC ------------------------------------------------------------ # Binds are the one thing you almost certainly need to edit for your machine: From 5c8850d29a87d5609edbf2e993211e9bda69de09 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 03:53:08 +0200 Subject: [PATCH 112/160] tests: blinding and custody as property tests over the seal and custody tables hypothesis joins the test extra. One property each now states the contracts the example families spelled out case by case: - seal is the seal table: under a blind, derived and unruled births are refused, and seal(s) - s equals an independent CCL reference on every xi/Cl_EE row whatever tags, pairs and statistics a part holds, with every other value, tag and the covariance bitwise kept; the blind opens only to shift rows (I1, I2, I5) - load opens exactly the stamps the door mints (I6) - a derivation carries its inputs' one stamp and, under a blind, only copies of their rows; each former example is kept as an @example (I7) - the commitment is the fork's and hides its RNG seed, for any seed (I8) - opening refuses an edit to any field the sealed seed binds (I10) - the audit accepts exactly one shift on every block (I11, mixed files) - custody: one table of every custody_of row, and any _leak_corr/_seed variant resolves to its base (I12) Audit tests share one revealed registry. Every DESIGN section 9 mutation and the review's HIGH defects were re-run against the reworked tests: all red. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- pyproject.toml | 1 + src/sp_validation/tests/test_blinding.py | 1128 ++++++++++------------ src/sp_validation/tests/test_custody.py | 216 ++--- uv.lock | 68 +- 4 files changed, 689 insertions(+), 724 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 632793f4..d38ec125 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,6 +127,7 @@ environments = ["sys_platform == 'linux'"] [project.optional-dependencies] test = [ + "hypothesis", "pytest", "pytest-cov", "ruff" diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index d18faecb..6e57135a 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -11,11 +11,14 @@ import json import os import stat +import string from pathlib import Path import numpy as np import pytest import yaml +from hypothesis import assume, example, given, settings +from hypothesis import strategies as st from sp_validation import blinding as bd from sp_validation import custody as cu @@ -26,13 +29,15 @@ DATA = Path(__file__).parent / "data" # The hidden (S8, Ωm) of the blind committed under DATA/blinds/committed. COMMITTED_POINT = (0.8374952413635888, 0.32730573347638175) +VERSIONS = ("TOY", "OTHER", "TOY_OPEN", "OTHER_OPEN", "TOY_MOCK") # --------------------------------------------------------------------------- # -# A registry with real blinds, and custody under them +# Registries with real blinds, and custody under them # --------------------------------------------------------------------------- # -def _cat_config(root): - """A catalogue config declaring one catalogue per custody state.""" +def _registry(root, *blinds): + """A catalogue config declaring one catalogue per custody state, and + ``blinds`` ((name, base) pairs) drawn for it, each from a fixed seed.""" entries = { "TOY": {}, "OTHER": {}, @@ -41,23 +46,20 @@ def _cat_config(root): "TOY_MOCK": {"blinding": "mock"}, "paths": {"output": str(root / "output")}, } - path = root / "cat_config.yaml" - path.write_text(yaml.safe_dump(entries)) + (root / "cat_config.yaml").write_text(yaml.safe_dump(entries)) (root / "fast.json").write_text(json.dumps(FAST)) - return path + with pytest.MonkeyPatch.context() as m: + for blind, base in blinds: + m.setattr(bd.secrets, "token_hex", lambda n, b=blind: f"{b}-seed") + _init(root, blind, base) + return root def _init(root, blind, *bases): + cat_config, config = root / "cat_config.yaml", root / "fast.json" bd.main( - [ - "init", - blind, - *bases, - "--cat-config", - str(root / "cat_config.yaml"), - "--config", - str(root / "fast.json"), - ] + ["init", blind, *bases, "--cat-config", str(cat_config)] + + ["--config", str(config)] ) @@ -68,33 +70,22 @@ def _custody(root, version): def _declare(root, **entries): """Set catalogue entries of ``root``'s catalogue config.""" path = root / "cat_config.yaml" - config = yaml.safe_load(path.read_text()) - config.update(entries) - path.write_text(yaml.safe_dump(config)) + path.write_text(yaml.safe_dump({**yaml.safe_load(path.read_text()), **entries})) @pytest.fixture(scope="module") def blinds(tmp_path_factory): - """Blinds `toy` (TOY) and `other` (OTHER), and the five custodies. - - Their seeds are fixed, so every run tests the same hidden points. - """ - root = tmp_path_factory.mktemp("registry") - _cat_config(root) - with pytest.MonkeyPatch.context() as m: - for blind, base in (("toy", "TOY"), ("other", "OTHER")): - m.setattr(bd.secrets, "token_hex", lambda n, b=blind: f"{b}-seed") - _init(root, blind, base) - versions = ("TOY", "OTHER", "TOY_OPEN", "OTHER_OPEN", "TOY_MOCK") - return {v: _custody(root, v) for v in versions} + """The five custodies: TOY under blind `toy`, OTHER under `other`.""" + root = _registry( + tmp_path_factory.mktemp("registry"), ("toy", "TOY"), ("other", "OTHER") + ) + return {v: _custody(root, v) for v in VERSIONS} @pytest.fixture def fresh(tmp_path): """A registry of its own, for tests that edit or reveal a blind.""" - _cat_config(tmp_path) - _init(tmp_path, "toy", "TOY") - return tmp_path + return _registry(tmp_path, ("toy", "TOY")) # --------------------------------------------------------------------------- # @@ -110,12 +101,18 @@ def _gauss_nz(z0, sigma, n=200): PAIRS = ((0, 0), (0, 1), (1, 1)) # The ξ± tags a part may carry; blocks never depend on them (None: no tag). XI_TAGS = ("reporting", "integration", "cosebis", "mystery", None) +# Data types no blind has a rule for, galaxy–galaxy lensing and CMB κ × shear, +# with the tag SACC requires of each. +UNRULED = { + "galaxy_shearDensity_xi_t": "theta", + "cmbGalaxy_convergenceShear_cl_e": "ell", +} def _xi_template(theta, k=0): - xip = 1e-4 * (1 + 0.1 * k) * (theta / 10.0) ** -0.6 - xim = 0.5e-4 * (1 + 0.1 * k) * (theta / 10.0) ** -0.9 - return xip, xim + return 1e-4 * (1 + 0.1 * k) * (theta / 10.0) ** -0.6, 0.5e-4 * (1 + 0.1 * k) * ( + theta / 10.0 + ) ** -0.9 def _add_xi_rows(s, pair, theta, tag, k=0): @@ -123,17 +120,15 @@ def _add_xi_rows(s, pair, theta, tag, k=0): if tag is not None: sio.add_xi(s, pair, theta, xip, xim, grid=tag) return - tracers = sio._pair(pair) for dtype, values in ((sio.XI_PLUS, xip), (sio.XI_MINUS, xim)): for th, v in zip(theta, values): - s.add_data_point(dtype, tracers, float(v), theta=float(th)) + s.add_data_point(dtype, sio._pair(pair), float(v), theta=float(th)) def _add_cl_rows(s, pair, k=0): ell_eff = np.array([30.0, 80.0, 150.0, 280.0, 450.0]) w_ell = np.arange(2, 501).astype(float) w_mat = np.exp(-0.5 * ((w_ell[:, None] - ell_eff[None, :]) / 40.0) ** 2) - w_mat /= w_mat.sum(axis=0) ee = 1e-8 * (1 + 0.1 * k) * (ell_eff / 100.0) ** -1.2 sio.add_pseudo_cl( s, @@ -143,7 +138,7 @@ def _add_cl_rows(s, pair, k=0): 0.01 * ee, 0.02 * ee, window_ells=w_ell, - window_weights=w_mat, + window_weights=w_mat / w_mat.sum(axis=0), ) @@ -153,16 +148,38 @@ def _add_rho_rows(s): sio.add_tau(s, (0,), 0, theta, np.arange(6) * 3e-7, np.arange(6) * 4e-7) -def two_bin_sacc(*, xi_tags=XI_TAGS, cl=True, rho=False, covariance=True): - """ξ± on every tag in ``xi_tags`` and pseudo-Cℓ, over pairs (0,0), (0,1), (1,1).""" +def two_bin_sacc( + *, + pairs=PAIRS, + xi_tags=XI_TAGS, + cl=True, + rho=False, + derived=(), + unruled=None, + covariance=True, +): + """A catalogue's part: ξ± under each of ``xi_tags`` and pseudo-Cℓ (EE, BB, + EB) over ``pairs`` of two bins, and optionally ρ/τ, COSEBIs and pure-E/B + rows and rows of a data type without a blinding rule.""" s = sio.new_sacc(NZ2, metadata={"catalogue_version": "TOY"}) - for k, pair in enumerate(PAIRS): + for pair in pairs: + k = PAIRS.index(pair) for n, tag in enumerate(xi_tags): _add_xi_rows(s, pair, np.geomspace(2.0, 200.0, 6) * (1 + 0.013 * n), tag, k) if cl: _add_cl_rows(s, pair, k) if rho: _add_rho_rows(s) + if "cosebis" in derived: + sio.add_cosebis(s, (0, 0), np.arange(1, 6) * 1e-10, (12.0, 83.0), Bn=np.ones(5)) + if "pure_eb" in derived: + theta = np.geomspace(2.0, 200.0, 4) + sio.add_pure_eb(s, (0, 0), theta, **{k: np.ones(4) for k in sio.PURE_KEYS}) + if unruled is not None: + for x in (5.0, 20.0, 80.0): + s.add_data_point( + unruled, ("source_0", "source_0"), 1e-5, **{UNRULED[unruled]: x} + ) if covariance: rng = np.random.default_rng(3) s.add_covariance( @@ -171,41 +188,36 @@ def two_bin_sacc(*, xi_tags=XI_TAGS, cl=True, rho=False, covariance=True): return s -def cosebis_sacc(): - s = sio.new_sacc({0: NZ2[0]}, metadata={"catalogue_version": "TOY"}) - sio.add_cosebis(s, (0, 0), np.arange(1, 6) * 1e-10, (12.0, 83.0), Bn=np.ones(5)) - return s - - -def pure_eb_sacc(): - s = sio.new_sacc({0: NZ2[0]}, metadata={"catalogue_version": "TOY"}) - theta = np.geomspace(2.0, 200.0, 4) - sio.add_pure_eb(s, (0, 0), theta, **{k: np.ones(4) for k in sio.PURE_KEYS}) - return s +def derived_sacc(kind): + """A COSEBIs or pure-E/B part alone, as its rule writes it.""" + return two_bin_sacc(pairs=((0, 0),), xi_tags=(), cl=False, derived=(kind,)) def rho_sacc(): - s = sio.new_sacc({0: NZ2[0]}, metadata={"catalogue_version": "TOY"}) - _add_rho_rows(s) - return s + return two_bin_sacc(pairs=(), rho=True) def _rows(s, *types): - return np.array([i for i, dp in enumerate(s.data) if dp.data_type in types]) + return np.array([i for i, dp in enumerate(s.data) if dp.data_type in types], int) + + +def _values(s): + return np.asarray(s.mean) # --------------------------------------------------------------------------- # # An independent CCL reference for the shift # --------------------------------------------------------------------------- # -def _reference_params(seed): - """The hidden and fiducial CCL points, drawn with the fork directly.""" +def _reference_cosmologies(seed): + """The hidden and fiducial CCL cosmologies, the draw made by the fork directly.""" + import pyccl as ccl from smokescreen.param_shifts import draw_param_shifts shift = draw_param_shifts({"S8": 0.075, "Omega_m": 0.1}, seed) - def ccl_point(s8, om): + def cosmology(s8, om): h, ob, mnu = 0.70, 0.0469, 0.06 - return dict( + return ccl.Cosmology( Omega_c=om - ob - mnu / (93.14 * h**2), Omega_b=ob, h=h, @@ -217,40 +229,27 @@ def ccl_point(s8, om): wa=0.0, Neff=3.046, T_CMB=2.7255, + transfer_function="eisenstein_hu", + matter_power_spectrum="halofit", ) - return ( - ccl_point(0.80 + shift["S8"], 0.30 + shift["Omega_m"]), - ccl_point(0.80, 0.30), - ) + return cosmology(0.80 + shift["S8"], 0.30 + shift["Omega_m"]), cosmology(0.80, 0.30) def reference_shift(s, seed): """t(hidden) − t(fiducial) on every ξ± and Cℓ_EE row of ``s``, from scratch.""" import pyccl as ccl - hidden, fiducial = _reference_params(seed) - ell = np.unique( - np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)]).astype(float) - ) - - def cosmo(p): - return ccl.Cosmology( - **p, transfer_function="eisenstein_hu", matter_power_spectrum="halofit" - ) - - def spectra(p, tracers, ells): - c = cosmo(p) - a, b = (s.tracers[t] for t in tracers) - return c, ccl.angular_cl( - c, - ccl.WeakLensingTracer(c, dndz=(a.z, a.nz)), - ccl.WeakLensingTracer(c, dndz=(b.z, b.nz)), - ells, - ) - + cosmologies = _reference_cosmologies(seed) + ell = np.unique(np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)])) out = np.full(len(s.mean), np.nan) for tracers in {dp.tracers for dp in s.data if dp.data_type in sio.SHIFTABLE}: + a, b = (s.tracers[t] for t in tracers) + + def cl(c, ells): + lens = [ccl.WeakLensingTracer(c, dndz=(t.z, t.nz)) for t in (a, b)] + return ccl.angular_cl(c, *lens, ells) + for dtype, kind in ((sio.XI_PLUS, "GG+"), (sio.XI_MINUS, "GG-")): rows = [ i @@ -258,68 +257,71 @@ def spectra(p, tracers, ells): if (dp.data_type, dp.tracers) == (dtype, tracers) ] theta = np.array([s.data[i].tags["theta"] for i in rows]) / 60.0 - xi = [] - for p in (hidden, fiducial): - c, cl = spectra(p, tracers, ell) - xi.append(ccl.correlation(c, ell=ell, C_ell=cl, theta=theta, type=kind)) - out[rows] = xi[0] - xi[1] - rows = [ - i - for i, dp in enumerate(s.data) - if (dp.data_type, dp.tracers) == (sio.CL_EE, tracers) - ] - for i in rows: - window = s.data[i].tags["window"] - ells = np.asarray(window.values, float) - w = np.asarray(window.weight, float)[:, s.data[i].tags["window_ind"]] - out[i] = w @ ( - spectra(hidden, tracers, ells)[1] - spectra(fiducial, tracers, ells)[1] + hidden, fiducial = ( + ccl.correlation(c, ell=ell, C_ell=cl(c, ell), theta=theta, type=kind) + for c in cosmologies ) + out[rows] = hidden - fiducial + for i, dp in enumerate(s.data): + if (dp.data_type, dp.tracers) == (sio.CL_EE, tracers): + window = dp.tags["window"] + ells = np.asarray(window.values, float) + w = np.asarray(window.weight, float)[:, dp.tags["window_ind"]] + out[i] = w @ (cl(cosmologies[0], ells) - cl(cosmologies[1], ells)) return out # --------------------------------------------------------------------------- # -# I1: the shift lands on every ξ± and Cℓ_EE row, from the content +# I1, I2, I5: seal is the seal table; a blinded seal shifts exactly the +# shiftable rows, by the theory difference, and nothing else # --------------------------------------------------------------------------- # -def test_shift_is_the_theory_difference_on_every_shiftable_row(blinds): - """seal(s) − s equals an independent CCL reference on every ξ± and Cℓ_EE row. - - Two bins, all three pairs; ξ± rows under every tag and none, so blocks are - discovered from the data types and tracers, never from grid names. - """ - s = two_bin_sacc() - sealed = sio.seal(s, blinds["TOY"]) - shift = np.asarray(sealed.mean) - np.asarray(s.mean) - expected = reference_shift(s, bd.open_blind(blinds["TOY"]).seed) - - rows = _rows(s, sio.XI_PLUS, sio.XI_MINUS, sio.CL_EE) - assert len(rows) and np.all(np.isfinite(expected[rows])) - for pair in PAIRS: - tracers = sio._pair(pair) - for types in ((sio.XI_PLUS, sio.XI_MINUS), (sio.CL_EE,)): - block = [ - i - for i in rows - if s.data[i].tracers == tracers and s.data[i].data_type in types - ] - scale = np.max(np.abs(expected[block])) - gap = np.max(np.abs(shift[block] - expected[block])) - assert scale > 0 and gap <= 1e-8 * scale, (pair, types, gap / scale) +PARTS = st.fixed_dictionaries( + { + "pairs": st.lists(st.sampled_from(PAIRS), min_size=1, max_size=3, unique=True), + "xi_tags": st.lists(st.sampled_from(XI_TAGS), max_size=3, unique=True), + "cl": st.booleans(), + "rho": st.booleans(), + "derived": st.sampled_from([(), ("cosebis",), ("pure_eb",)]), + "unruled": st.sampled_from([None, *UNRULED]), + } +).filter(lambda p: p["xi_tags"] or p["cl"] or p["rho"] or p["derived"] or p["unruled"]) +EVERY_SHIFTABLE_ROW = dict( + pairs=PAIRS, xi_tags=XI_TAGS, cl=True, rho=True, derived=(), unruled=None +) -# --------------------------------------------------------------------------- # -# I2: nothing else moves -# --------------------------------------------------------------------------- # -def test_only_shiftable_values_move(blinds): - """BB, EB and ρ/τ values, every tag, tracer, row and the covariance are bitwise kept.""" - s = two_bin_sacc(rho=True) - sealed = sio.seal(s, blinds["TOY"]) - moved = _rows(s, sio.XI_PLUS, sio.XI_MINUS, sio.CL_EE) +@settings(max_examples=6, deadline=None) +@given(content=PARTS, version=st.sampled_from(["TOY", "TOY_OPEN", "TOY_MOCK"])) +@example(content=EVERY_SHIFTABLE_ROW, version="TOY") +@example(content={**EVERY_SHIFTABLE_ROW, "xi_tags": [], "cl": False}, version="TOY") +def test_seal_is_the_seal_table(blinds, content, version): + """Under a blind, a birth with derived or unruled rows is refused; any + other has ``seal(s) − s`` equal to an independent CCL reference at (fid, + hidden) on every ξ± and Cℓ_EE row, within 1e-8 of each block's largest + shift, and every other value, tag, tracer and the covariance bitwise kept. + Unblinded and mock births keep every value. The blind opens only to shift + rows, and the input is never stamped in place. + + Blocks come from the data types and tracers: ξ± under every tag and none, + over the two bins' three pairs, whatever subset a part holds. + """ + s, custody = two_bin_sacc(**content), blinds[version] + blinded = custody.status == "blinded" + opened, open_blind = [], bd.open_blind + with pytest.MonkeyPatch.context() as m: + m.setattr(bd, "open_blind", lambda c: opened.append(c) or open_blind(c)) + if blinded and (content["derived"] or content["unruled"]): + with pytest.raises(ValueError, match="derived_from|no blinding rule"): + sio.seal(s, custody) + return + sealed = sio.seal(s, custody) + + assert "blinding" not in s.metadata + assert cu.read_stamp(sealed.metadata).stamp == custody.stamp + moved = _rows(s, *sio.SHIFTABLE) if blinded else np.array([], int) + assert bool(opened) == bool(len(moved)) kept = np.setdiff1d(np.arange(len(s.mean)), moved) - assert len(kept) and np.array_equal( - np.asarray(sealed.mean)[kept], np.asarray(s.mean)[kept] - ) - assert np.all(np.asarray(sealed.mean)[moved] != np.asarray(s.mean)[moved]) + assert np.array_equal(_values(sealed)[kept], _values(s)[kept]) assert np.array_equal(sealed.covariance.dense, s.covariance.dense) for a, b in zip(s.data, sealed.data): assert (a.data_type, a.tracers) == (b.data_type, b.tracers) @@ -329,15 +331,22 @@ def test_only_shiftable_values_move(blinds): for name, tracer in s.tracers.items(): if hasattr(tracer, "nz"): assert np.array_equal(sealed.tracers[name].nz, tracer.nz) + if not len(moved): + return - -def test_derived_statistics_keep_their_values(blinds, tmp_path): - """COSEBIs and pure-E/B written as derivations carry their values unchanged.""" - xi = sio.seal(two_bin_sacc(cl=False), blinds["TOY"]) - for s in (cosebis_sacc(), pure_eb_sacc()): - path = tmp_path / "derived.sacc" - sio.save(s, path, derived_from=[xi]) - assert np.array_equal(np.asarray(sio.load(path).mean), np.asarray(s.mean)) + shift = _values(sealed) - _values(s) + expected = reference_shift(s, open_blind(custody).seed) + for pair in content["pairs"]: + for types in ((sio.XI_PLUS, sio.XI_MINUS), (sio.CL_EE,)): + block = [ + i + for i in moved + if s.data[i].tracers == sio._pair(pair) and s.data[i].data_type in types + ] + if block: + scale = np.max(np.abs(expected[block])) + gap = np.max(np.abs(shift[block] - expected[block])) + assert scale > 0 and gap <= 1e-8 * scale, (pair, types, gap / scale) # --------------------------------------------------------------------------- # @@ -352,9 +361,8 @@ def test_hidden_point_is_uniform_in_the_s8_om_box(): s8 = np.array([h.S8 for h in hidden]) om = np.array([h.Omega_m for h in hidden]) - assert np.all(np.abs(s8 - fid.S8) <= 0.075) and np.all( - np.abs(om - fid.Omega_m) <= 0.1 - ) + assert np.all(np.abs(s8 - fid.S8) <= 0.075) + assert np.all(np.abs(om - fid.Omega_m) <= 0.1) assert stats.kstest((s8 - fid.S8 + 0.075) / 0.15, "uniform").pvalue > 1e-3 assert stats.kstest((om - fid.Omega_m + 0.1) / 0.2, "uniform").pvalue > 1e-3 for h in hidden[:50]: @@ -447,90 +455,12 @@ def delta(grid): # --------------------------------------------------------------------------- # -# I5: the seal table +# I6: the gates # --------------------------------------------------------------------------- # -@pytest.mark.parametrize("version", ["TOY_OPEN", "TOY_MOCK"]) -def test_unblinded_and_mock_births_are_stamped_untouched(blinds, version): - s = two_bin_sacc(rho=True) - sealed = sio.seal(s, blinds[version]) - assert np.array_equal(np.asarray(sealed.mean), np.asarray(s.mean)) - assert cu.read_stamp(sealed.metadata).stamp == blinds[version].stamp - assert "blinding" not in s.metadata # the input is not stamped in place - - -def test_a_blinded_birth_is_concealed_and_stamped(blinds): - s = two_bin_sacc(cl=False) - sealed = sio.seal(s, blinds["TOY"]) - assert not np.array_equal(np.asarray(sealed.mean), np.asarray(s.mean)) - assert cu.read_stamp(sealed.metadata).stamp == blinds["TOY"].stamp - - -@pytest.mark.parametrize( - "make", [cosebis_sacc, pure_eb_sacc], ids=["cosebis", "pure_eb"] -) -def test_a_blinded_birth_of_a_derived_statistic_is_refused(blinds, make): - with pytest.raises(ValueError, match="derived_from"): - sio.seal(make(), blinds["TOY"]) - - -def test_a_blinded_birth_without_signal_never_opens_the_blind(blinds, monkeypatch): - def refuse(custody): - raise AssertionError("ρ/τ opened the blind") - - monkeypatch.setattr(bd, "open_blind", refuse) - s = rho_sacc() - sealed = sio.seal(s, blinds["TOY"]) - assert np.array_equal(np.asarray(sealed.mean), np.asarray(s.mean)) - assert cu.read_stamp(sealed.metadata).stamp == blinds["TOY"].stamp - - -# Data types no blind has a rule for, galaxy–galaxy lensing and CMB κ × shear, -# with the tag SACC requires of each. -UNRULED = { - "galaxy_shearDensity_xi_t": "theta", - "cmbGalaxy_convergenceShear_cl_e": "ell", -} - - -def _with_unruled(s, data_type): - for x in (5.0, 20.0, 80.0): - s.add_data_point( - data_type, ("source_0", "source_0"), 1e-5, **{UNRULED[data_type]: x} - ) - return s - - -@pytest.mark.parametrize("data_type", UNRULED) -def test_a_data_type_without_a_blinding_rule_is_refused_under_a_blind( - blinds, tmp_path, data_type -): - """Under a blind, a data type is born or derived only with a blinding rule. - - Unblinded and mock SACCs are stamped whatever they hold. - """ - for version in ("TOY_OPEN", "TOY_MOCK"): - s = _with_unruled(two_bin_sacc(cl=False, covariance=False), data_type) - assert np.array_equal( - np.asarray(sio.seal(s, blinds[version]).mean), np.asarray(s.mean) - ) - with pytest.raises(ValueError, match="no blinding rule"): - sio.seal( - _with_unruled(two_bin_sacc(cl=False, covariance=False), data_type), - blinds["TOY"], - ) - xi = sio.seal(two_bin_sacc(cl=False, covariance=False), blinds["TOY"]) - for custody in (None, blinds["TOY"]): - with pytest.raises(ValueError, match="no blinding rule"): - sio.save( - _with_unruled(xi.copy(), data_type), - tmp_path / "x.sacc", - derived_from=[xi], - custody=custody, - ) - assert not list(tmp_path.iterdir()) - - -def test_a_stamped_sacc_is_not_born_again(blinds, tmp_path): +def test_nothing_is_written_unstamped_or_stamped_twice(blinds, tmp_path): + with pytest.raises(ValueError, match="custody"): + sio.save(two_bin_sacc(), tmp_path / "x.sacc") + assert not (tmp_path / "x.sacc").exists() sealed = sio.seal(two_bin_sacc(cl=False), blinds["TOY_OPEN"]) with pytest.raises(ValueError, match="already stamped"): sio.seal(sealed, blinds["TOY"]) @@ -540,113 +470,107 @@ def test_a_stamped_sacc_is_not_born_again(blinds, tmp_path): sio.save(sio.load(path), path, custody=blinds["TOY_OPEN"]) -# --------------------------------------------------------------------------- # -# I6: the gates -# --------------------------------------------------------------------------- # -def test_save_needs_a_custody_or_inputs(tmp_path): - with pytest.raises(ValueError, match="custody"): - sio.save(two_bin_sacc(), tmp_path / "x.sacc") - assert not (tmp_path / "x.sacc").exists() - - -def _raw(tmp_path, name, **metadata): - s = two_bin_sacc(cl=False) - s.metadata.update(metadata) - path = tmp_path / f"{name}.sacc" - s.save_fits(str(path), overwrite=True) - return path - - -def test_load_refuses_unstamped_and_malformed_files(blinds, tmp_path): - stamp = blinds["TOY"].stamp - for name, metadata in { - "unstamped": {}, - "unknown": {**stamp, "blinding": "open"}, - "no_catalogue": {"blinding": "unblinded"}, - "no_commitment": {k: v for k, v in stamp.items() if k != "blinding_commitment"}, - "mixed": {**blinds["TOY_OPEN"].stamp, "blinding_blind": "toy"}, - }.items(): +@settings(max_examples=20, deadline=None) +@given( + version=st.sampled_from(["TOY", "TOY_OPEN", "TOY_MOCK"]), + spoil=st.sampled_from([None, "drop", "add", "status"]), + data=st.data(), +) +def test_load_opens_exactly_the_stamps_the_door_mints( + blinds, tmp_path_factory, version, spoil, data +): + """A file stamped as ``save`` stamps it loads with that stamp; one missing + a stamp key, carrying a blind's keys under another status, or naming an + unknown status is refused.""" + stamp = dict(blinds[version].stamp) + assume(not (spoil == "add" and version == "TOY")) + if spoil == "drop": + del stamp[data.draw(st.sampled_from(sorted(stamp)))] + elif spoil == "add": + key = data.draw(st.sampled_from(cu.STAMP_KEYS[2:])) + stamp[key] = blinds["TOY"].stamp[key] + elif spoil == "status": + status = st.text(string.ascii_letters, min_size=1, max_size=8) + stamp["blinding"] = data.draw(status.filter(lambda t: t not in cu.STATUSES)) + path = tmp_path_factory.mktemp("raw") / "part.sacc" + _write(path, two_bin_sacc(cl=False), stamp) + if spoil is None: + assert cu.read_stamp(sio.load(path).metadata).stamp == stamp + else: with pytest.raises(ValueError, match="stamp"): - sio.load(_raw(tmp_path, name, **metadata)) - - -@pytest.mark.parametrize("version", ["TOY", "TOY_OPEN", "TOY_MOCK"]) -def test_every_custody_loads(blinds, tmp_path, version): - path = tmp_path / "part.sacc" - written = sio.save(two_bin_sacc(), path, custody=blinds[version]) - loaded = sio.load(path) - assert cu.read_stamp(loaded.metadata).stamp == blinds[version].stamp - assert np.array_equal(np.asarray(loaded.mean), np.asarray(written.mean)) + sio.load(path) # --------------------------------------------------------------------------- # -# I7: derivations inherit one stamp; assembly equals the declaration +# I7: a derivation carries its inputs' one stamp; an assembly its declaration # --------------------------------------------------------------------------- # -def test_a_derivation_inherits_its_inputs_stamp(blinds, tmp_path): - xi = sio.seal(two_bin_sacc(cl=False), blinds["TOY"]) - written = sio.save(cosebis_sacc(), tmp_path / "c.sacc", derived_from=[xi, xi]) - assert cu.read_stamp(written.metadata).stamp == blinds["TOY"].stamp - - -@pytest.mark.parametrize( - "first, second", - [ - ("TOY_OPEN", "TOY_MOCK"), # two statuses - ("TOY", "OTHER"), # two blinds - ("TOY_OPEN", "OTHER_OPEN"), # two unblinded catalogues - ], -) -def test_inputs_under_two_stamps_are_refused(blinds, tmp_path, first, second): - a = sio.seal(two_bin_sacc(cl=False), blinds[first]) - b = sio.seal(two_bin_sacc(cl=False), blinds[second]) - with pytest.raises(ValueError, match="stamps"): - sio.save(cosebis_sacc(), tmp_path / "c.sacc", derived_from=[a, b]) - with pytest.raises(ValueError, match="stamps"): - sio.save( - a.copy(), tmp_path / "a.sacc", derived_from=[a, b], custody=blinds[first] - ) - assert not list(tmp_path.iterdir()) +@pytest.fixture(scope="module") +def parts(blinds): + """A ξ± part born under each custody.""" + return { + v: sio.seal(two_bin_sacc(xi_tags=("reporting",), cl=False), blinds[v]) + for v in VERSIONS + } -def test_plaintext_cannot_be_laundered_under_a_concealed_stamp(blinds, tmp_path): - """A derivation's ξ± and Cℓ_EE rows must be copies of its inputs' rows.""" - plain = two_bin_sacc() - concealed = sio.seal(plain, blinds["TOY"]) - with pytest.raises(ValueError, match="not copies"): - sio.save(plain, tmp_path / "x.sacc", derived_from=[concealed]) - assert not (tmp_path / "x.sacc").exists() - sio.save(concealed.copy(), tmp_path / "y.sacc", derived_from=[concealed]) +DERIVATIONS = ("copy", "cosebis", "pure_eb", "plaintext", "unruled") -@pytest.mark.parametrize( - "parts_under, declared", - [ - ("TOY", "TOY_OPEN"), # a stale concealed part after a reveal - ("OTHER", "TOY"), # another blind and catalogue - ("TOY_MOCK", "TOY_OPEN"), # a mock inside data - ("TOY_OPEN", "TOY"), # unblinded inside blinded - ], +@settings(max_examples=25, deadline=None) +@given( + inputs=st.lists(st.sampled_from(VERSIONS), min_size=1, max_size=3), + declared=st.sampled_from([None, *VERSIONS]), + content=st.sampled_from(DERIVATIONS), ) -def test_assembly_refuses_parts_under_another_custody( - blinds, tmp_path, parts_under, declared +@example(inputs=["TOY"], declared=None, content="plaintext") # laundering +@example(inputs=["TOY"], declared="TOY_OPEN", content="copy") # stale after a reveal +@example(inputs=["OTHER"], declared="TOY", content="copy") # another blind +@example(inputs=["TOY_MOCK"], declared="TOY_OPEN", content="copy") # mock into data +@example(inputs=["TOY_OPEN"], declared="TOY", content="copy") # unblinded into blinded +@example(inputs=["TOY_OPEN", "TOY_MOCK"], declared=None, content="cosebis") +@example(inputs=["TOY", "OTHER"], declared=None, content="cosebis") +@example(inputs=["TOY_OPEN", "OTHER_OPEN"], declared=None, content="cosebis") +@example(inputs=["TOY"], declared=None, content="unruled") +@example(inputs=["TOY", "TOY"], declared="TOY", content="copy") # an assembly +@example(inputs=["TOY_OPEN"], declared="TOY_OPEN", content="plaintext") +@example(inputs=["TOY_MOCK"], declared=None, content="pure_eb") +def test_a_derivation_carries_its_inputs_one_stamp( + blinds, parts, tmp_path_factory, inputs, declared, content ): - part = sio.seal(two_bin_sacc(cl=False), blinds[parts_under]) - with pytest.raises(ValueError, match="declared"): - sio.save( - part.copy(), - tmp_path / "a.sacc", - derived_from=[part], - custody=blinds[declared], - ) - assert not (tmp_path / "a.sacc").exists() - - -def test_assembly_under_the_declaration(blinds, tmp_path): - part = sio.seal(two_bin_sacc(cl=False), blinds["TOY"]) - written = sio.save( - part.copy(), tmp_path / "a.sacc", derived_from=[part], custody=blinds["TOY"] + """``save(s, derived_from=inputs, custody=declared)`` writes ``s``'s values + under the inputs' stamp exactly when the inputs share one stamp, it is the + declared custody's (if any), and, under a blind, every ξ± row of ``s`` is a + copy of an input row and every data type has a blinding rule. Otherwise + nothing is written.""" + first = parts[inputs[0]] + s = { + "copy": lambda: first.copy(), + "cosebis": lambda: derived_sacc("cosebis"), + "pure_eb": lambda: derived_sacc("pure_eb"), + "plaintext": lambda: two_bin_sacc(xi_tags=("reporting",), cl=False), + "unruled": lambda: two_bin_sacc( + xi_tags=(), cl=False, unruled=next(iter(UNRULED)) + ), + }[content]() + stamp = blinds[inputs[0]].stamp + blinded = stamp["blinding"] == "blinded" + allowed = ( + all(blinds[v].stamp == stamp for v in inputs) + and (declared is None or blinds[declared].stamp == stamp) + and not (blinded and content in ("plaintext", "unruled")) ) - assert cu.read_stamp(written.metadata).stamp == blinds["TOY"].stamp + path = tmp_path_factory.mktemp("derived") / "d.sacc" + custody = blinds[declared] if declared else None + derive = [parts[v] for v in inputs] + if not allowed: + with pytest.raises(ValueError): + sio.save(s, path, derived_from=derive, custody=custody) + assert not path.exists() + return + sio.save(s, path, derived_from=derive, custody=custody) + loaded = sio.load(path) + assert cu.read_stamp(loaded.metadata).stamp == stamp + assert np.array_equal(_values(loaded), _values(s)) # --------------------------------------------------------------------------- # @@ -669,85 +593,65 @@ def _variants(config): ) -def test_the_digest_changes_with_every_field(): +def test_the_digest_changes_with_every_field_and_no_literal(): config = bd.BlindingConfig() digests = [v.digest() for v in _variants(config)] assert len(digests) == len(set(digests)) and config.digest() not in digests assert len(digests) > len(dataclasses.fields(TheoryConfig)) - -def test_the_digest_ignores_int_versus_float(): - a = bd.BlindingConfig(envelope={"S8": 0.075, "Omega_m": 0.1}) - b = bd.BlindingConfig( - envelope={"S8": 0.075, "Omega_m": 0.1}, - theory=TheoryConfig(w0=-1, wa=0, ia_bias=0), - ) - assert a.digest() == b.digest() - assert bd._recorded_config("toy", json.loads(json.dumps(a.record()))) == a + ints = bd.BlindingConfig(theory=TheoryConfig(w0=-1, wa=0, ia_bias=0)) + assert ints.digest() == config.digest() + assert bd._recorded_config("toy", json.loads(json.dumps(config.record()))) == config -def test_the_commitment_is_the_forks(): +@settings(max_examples=50) +@given(seed=st.text(min_size=1)) +@example(seed="my_secret_seed") +def test_the_commitment_is_the_forks_and_hides_its_rng_seed(seed): + """The fork seeds its RNG from the undomained sha256 of the seed; the + domain prefix keeps the commitment from publishing that RNG seed in its + first 16 hex characters, from which anyone could redraw the hidden point.""" import smokescreen - - assert cu.COMMITMENT_DOMAIN == smokescreen.COMMITMENT_DOMAIN - for seed in ("the-secret", "2112"): - assert cu.seed_commitment(seed) == smokescreen.seed_commitment(seed) - - -def test_the_commitment_does_not_embed_the_rng_seed(): - """The fork seeds its RNG from the undomained sha256 of the seed. - - A commitment over the bare seed would publish that RNG seed in its first - 16 hex characters, and with the public config anyone could redraw the - hidden cosmology; the domain prefix breaks that identity. - """ - import secrets - from smokescreen.param_shifts import _normalize_seed - for seed in ("my_secret_seed", "the-secret", secrets.token_hex(16)): - assert int(cu.seed_commitment(seed)[:16], 16) != _normalize_seed(seed) + assert cu.COMMITMENT_DOMAIN == smokescreen.COMMITMENT_DOMAIN + assert cu.seed_commitment(seed) == smokescreen.seed_commitment(seed) + assert int(cu.seed_commitment(seed)[:16], 16) != _normalize_seed(seed) # --------------------------------------------------------------------------- # -# I9: the seed never touches disk +# I9 and blind-drawn-once: init draws once, and the seed never touches disk # --------------------------------------------------------------------------- # SEED = "5eed" * 8 -def _files_containing(root, needle): - return [ - p for p in root.rglob("*") if p.is_file() and needle.encode() in p.read_bytes() - ] - - @pytest.mark.parametrize("fault", [None, "after_key", "encrypt"]) def test_init_never_writes_the_seed(tmp_path, monkeypatch, fault): - """A normal init, or one interrupted anywhere, leaves no file holding the seed.""" + """A normal init, or one interrupted anywhere, leaves no file holding the + seed; a completed record is read-only.""" from cryptography import fernet - _cat_config(tmp_path) + _registry(tmp_path) monkeypatch.setattr(bd.secrets, "token_hex", lambda n: SEED) if fault == "after_key": - - def replace(src, dst): - raise OSError("interrupted after the key was written") - - monkeypatch.setattr(bd.os, "replace", replace) + monkeypatch.setattr(bd.os, "replace", lambda src, dst: 1 / 0) elif fault == "encrypt": - - def encrypt(self, data): - raise RuntimeError("interrupted while encrypting") - - monkeypatch.setattr(fernet.Fernet, "encrypt", encrypt) + monkeypatch.setattr(fernet.Fernet, "encrypt", lambda self, data: 1 / 0) if fault is None: _init(tmp_path, "toy", "TOY") - assert (tmp_path / "blinds" / "toy" / "key").exists() + for name in ("commitment.json", "seed.fernet", "key"): + mode = os.stat(tmp_path / "blinds" / "toy" / name).st_mode + assert not mode & (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) else: - with pytest.raises((OSError, RuntimeError)): + with pytest.raises(ZeroDivisionError): _init(tmp_path, "toy", "TOY") - assert _files_containing(tmp_path, SEED) == [] + hits = [ + p + for p in tmp_path.rglob("*") + if p.is_file() and SEED.encode() in p.read_bytes() + ] + assert hits == [] def test_init_refuses_existing_state(fresh): @@ -762,14 +666,6 @@ def test_init_refuses_existing_state(fresh): _init(fresh, "later", "OTHER") -def test_the_record_is_read_only(fresh): - record = fresh / "blinds" / "toy" - for name in ("commitment.json", "seed.fernet", "key"): - assert not os.stat(record / name).st_mode & ( - stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH - ) - - @pytest.mark.parametrize( "config, named", [ @@ -780,16 +676,14 @@ def test_the_record_is_read_only(fresh): ids=["envelope_key", "theory_key", "unbuildable"], ) def test_init_refuses_a_config_no_blind_conceals_under(tmp_path, config, named): - """A config whose blind could not conceal a row is refused before any - record exists, so the base stays free for a blind drawn under a good one.""" - _cat_config(tmp_path) + """Refused before any record exists, so the base stays free for a blind + drawn under a good config.""" + _registry(tmp_path) (tmp_path / "bad.json").write_text(json.dumps(config)) args = ["init", "toy", "TOY", "--cat-config", str(tmp_path / "cat_config.yaml")] with pytest.raises(cu.CustodyError, match=named): bd.main([*args, "--config", str(tmp_path / "bad.json")]) - assert not (tmp_path / "blinds").exists() or not any( - (tmp_path / "blinds").iterdir() - ) + assert not any((tmp_path / "blinds").glob("*")) _init(tmp_path, "toy", "TOY") @@ -805,20 +699,12 @@ def test_share_adds_a_base_to_a_concealed_blind(fresh): }.items(): with pytest.raises(cu.CustodyError, match=refusal): bd.share("toy", base, cat_config=cat_config) - with pytest.raises(cu.CustodyError, match="no blind"): - _custody(fresh, "OTHER") assert bd.main(["share", "toy", "OTHER", "--cat-config", cat_config]) == 0 toy, other = _custody(fresh, "TOY"), _custody(fresh, "OTHER") - assert (other.status, other.blind, other.commitment) == ( - "blinded", - "toy", - toy.commitment, - ) + assert (other.blind, other.commitment) == ("toy", toy.commitment) - (fresh / "blinds" / "toy" / "revealed.json").write_text( - json.dumps({"seed": bd.open_blind(toy).seed}) - ) + _publish(fresh, bd.open_blind(toy).seed) with pytest.raises(cu.CustodyError, match="revealed"): bd.share("toy", "LATER", cat_config=cat_config) @@ -826,8 +712,8 @@ def test_share_adds_a_base_to_a_concealed_blind(fresh): def test_a_blind_is_drawn_for_every_catalogue_reading_its_file(tmp_path): """init and share refuse a catalogue whose shear file another catalogue reads under another custody, before anything is written; drawn for both, - the blind conceals both.""" - _cat_config(tmp_path) + the blind covers both.""" + _registry(tmp_path) toy, later = ({"shear": {"path": str(tmp_path / f)}} for f in ("a.fits", "b.fits")) _declare( tmp_path, @@ -842,7 +728,6 @@ def test_a_blind_is_drawn_for_every_catalogue_reading_its_file(tmp_path): _declare(tmp_path, TOY_TWIN=toy) _init(tmp_path, "toy", "TOY", "TOY_TWIN") - assert _custody(tmp_path, "TOY_TWIN").blind == "toy" with pytest.raises(cu.CustodyError, match="LATER_TWIN"): bd.share("toy", "LATER", cat_config=str(tmp_path / "cat_config.yaml")) bases = (tmp_path / "blinds" / "toy" / "bases").read_text().split() @@ -852,48 +737,79 @@ def test_a_blind_is_drawn_for_every_catalogue_reading_its_file(tmp_path): # --------------------------------------------------------------------------- # # I10: an opened blind is the one committed # --------------------------------------------------------------------------- # -def _edit(path, edit): - os.chmod(path, 0o644) - record = json.loads(path.read_text()) - edit(record) - path.write_text(json.dumps(record)) +@pytest.fixture(scope="module") +def tamperable(tmp_path_factory): + """Blind `toy`'s record, restored after each edit, and TOY's custody.""" + root = _registry(tmp_path_factory.mktemp("tamperable"), ("toy", "TOY")) + return root / "blinds" / "toy" / "commitment.json", _custody(root, "TOY") -def test_open_blind_verifies_the_record(fresh): - blind = bd.open_blind(_custody(fresh, "TOY")) - assert blind.name == "toy" and blind.seed not in repr(blind) +def _leaves(record, path=()): + for key, value in record.items(): + if isinstance(value, dict): + yield from _leaves(value, (*path, key)) + else: + yield (*path, key) -@pytest.mark.parametrize( - "tamper", - ["wrong_key", "config", "config_and_digest", "scheme", "name", "custody"], +# The fields of a blind's record the sealed seed binds: its name, commitment, +# draw scheme, config digest and every field of its config. +BOUND = sorted( + [("blind",), ("seed_commitment",), ("config_digest",), ("draw_scheme",)] + + [("config", *leaf) for leaf in _leaves(bd.BlindingConfig().record())] ) -def test_open_blind_refuses_a_tampered_record(fresh, tamper): + + +def _edited(value): + if isinstance(value, bool): + return not value + return value + ("x" if isinstance(value, str) else 1) + + +@settings(max_examples=15, deadline=None) +@given(leaf=st.sampled_from(BOUND), redigest=st.booleans()) +@example(leaf=("config", "envelope", "S8"), redigest=True) +@example(leaf=("config", "theory", "transfer_function"), redigest=False) +def test_open_blind_refuses_any_edit_of_the_record(tamperable, leaf, redigest): + """Edit any field the sealed seed binds, a config field with its digest + recomputed or not: opening the blind is refused.""" + path, custody = tamperable + original = path.read_bytes() + record = json.loads(original) + *parents, key = leaf + node = record + for parent in parents: + node = node[parent] + node[key] = _edited(node[key]) + if redigest and leaf != ("config_digest",): + record["config_digest"] = bd.record_digest(record["config"]) + os.chmod(path, 0o644) + try: + path.write_text(json.dumps(record)) + bd._open.cache_clear() + with pytest.raises(cu.CustodyError): + bd.open_blind(custody) + finally: + path.write_bytes(original) + bd._open.cache_clear() + assert bd.open_blind(custody).seed not in repr(bd.open_blind(custody)) + + +@pytest.mark.parametrize("tamper", ["wrong_key", "renamed", "custody"]) +def test_open_blind_refuses_another_key_name_or_commitment(fresh, tamper): from cryptography.fernet import Fernet record = fresh / "blinds" / "toy" - commitment = record / "commitment.json" if tamper == "wrong_key": os.chmod(record / "key", 0o644) (record / "key").write_bytes(Fernet.generate_key()) - elif tamper == "config": - _edit(commitment, lambda r: r["config"]["envelope"].update(S8=0.3)) - elif tamper == "config_and_digest": - - def edit(r): - r["config"]["envelope"]["S8"] = 0.3 - r["config_digest"] = bd.record_digest(r["config"]) - - _edit(commitment, edit) - elif tamper == "scheme": - _edit(commitment, lambda r: r.update(draw_scheme=r["draw_scheme"] + 1)) - elif tamper == "name": - os.rename(record, fresh / "blinds" / "toy2") - _edit( - fresh / "blinds" / "toy2" / "commitment.json", - lambda r: r.update(blind="toy2"), + elif tamper == "renamed": + os.rename(record, record.with_name("toy2")) + commitment = record.with_name("toy2") / "commitment.json" + os.chmod(commitment, 0o644) + commitment.write_text( + json.dumps({**json.loads(commitment.read_text()), "blind": "toy2"}) ) - (fresh / "blinds" / "toy2" / "bases").write_text("TOY\n") custody = _custody(fresh, "TOY") if tamper == "custody": custody = dataclasses.replace(custody, commitment="0" * 64) @@ -914,40 +830,41 @@ def _init_storing(root, edit): bd.init("toy", ["TOY"], cat_config=root / "cat_config.yaml", config=config) -def test_a_record_lacking_a_field_opens_to_the_shift_it_was_drawn_with( - tmp_path, blinds, monkeypatch +@pytest.mark.parametrize( + "edit, field", + [ + ( + lambda r: { + **r, + "theory": {k: v for k, v in r["theory"].items() if k != "ia_alphaz"}, + }, + "ia_alphaz", + ), + ( + lambda r: {**r, "theory": {**r["theory"], "baryon_boost": 0.0}}, + "baryon_boost", + ), + ], + ids=["lacks_a_field", "names_a_foreign_field"], +) +def test_a_record_of_another_schema_is_refused_by_the_field( + tmp_path, blinds, monkeypatch, edit, field ): - """A record that predates a TheoryConfig field opens once the field's - neutral value is declared, and conceals exactly as the blind that names it; - until then it is refused by the field's name, never as an edited record.""" - _cat_config(tmp_path) - - def without_alphaz(record): - del record["theory"]["ia_alphaz"] - return record - - _init_storing(tmp_path, without_alphaz) + """A record lacking a TheoryConfig field, or naming one this code lacks, + is refused by the field's name, never as an edited record. Once a lacking + field's neutral value is declared, it conceals as the blind that names it.""" + _registry(tmp_path) + _init_storing(tmp_path, edit) custody = _custody(tmp_path, "TOY") - with pytest.raises(cu.CustodyError, match="ia_alphaz") as refused: + with pytest.raises(cu.CustodyError, match=field) as refused: bd.open_blind(custody) assert "edited" not in str(refused.value) - - monkeypatch.setattr(bd, "NEUTRAL", {"ia_alphaz": 0.0}) - s = two_bin_sacc(cl=False) - assert np.array_equal( - np.asarray(sio.seal(s, custody).mean), - np.asarray(sio.seal(s, blinds["TOY"]).mean), - ) - - -def test_a_record_naming_a_field_this_code_lacks_is_refused_by_name(tmp_path): - _cat_config(tmp_path) - _init_storing( - tmp_path, lambda r: {**r, "theory": {**r["theory"], "baryon_boost": 0.0}} - ) - with pytest.raises(cu.CustodyError, match="baryon_boost") as refused: - bd.open_blind(_custody(tmp_path, "TOY")) - assert "edited" not in str(refused.value) + if field == "ia_alphaz": + monkeypatch.setattr(bd, "NEUTRAL", {"ia_alphaz": 0.0}) + s = two_bin_sacc(cl=False) + assert np.array_equal( + _values(sio.seal(s, custody)), _values(sio.seal(s, blinds["TOY"])) + ) def test_the_committed_blind_opens_under_this_code(): @@ -965,7 +882,7 @@ def test_the_committed_blind_opens_under_this_code(): def test_a_blind_drawn_under_another_draw_scheme_is_refused(tmp_path, monkeypatch): """Its seed and record agree, but this install's fork draws differently: opening it is refused, and a file stamped under it fails ``verify``.""" - _cat_config(tmp_path) + _registry(tmp_path) installed = bd.draw_scheme() with monkeypatch.context() as m: m.setattr(bd, "draw_scheme", lambda: installed + 1) @@ -983,41 +900,35 @@ def test_a_blind_drawn_under_another_draw_scheme_is_refused(tmp_path, monkeypatc # --------------------------------------------------------------------------- # # verify: a file's stamp against its catalogue's custody, seedless # --------------------------------------------------------------------------- # -def _verify(root, path, capsys): - code = bd.main(["verify", str(path), "--cat-config", str(root / "cat_config.yaml")]) - return code, capsys.readouterr().out - - def test_verify_names_what_disagrees_with_the_declaration(fresh, capsys): - part = fresh / "part.sacc" - sio.save(two_bin_sacc(cl=False), part, custody=_custody(fresh, "TOY")) - assert _verify(fresh, part, capsys)[0] == 0 + def verify(path): + code = bd.main( + ["verify", str(path), "--cat-config", str(fresh / "cat_config.yaml")] + ) + return code, capsys.readouterr().out + + part, forged = fresh / "part.sacc", fresh / "forged.sacc" + custody = _custody(fresh, "TOY") + sio.save(two_bin_sacc(cl=False), part, custody=custody) + assert verify(part)[0] == 0 - forged = fresh / "forged.sacc" _write( forged, two_bin_sacc(cl=False), - {**_custody(fresh, "TOY").stamp, "blinding_commitment": "0" * 64}, + {**custody.stamp, "blinding_commitment": "0" * 64}, ) - code, out = _verify(fresh, forged, capsys) + code, out = verify(forged) assert code == 1 and "['blinding_commitment']" in out - (fresh / "blinds" / "toy" / "revealed.json").write_text( - json.dumps({"seed": bd.open_blind(_custody(fresh, "TOY")).seed}) - ) + _publish(fresh, bd.open_blind(custody).seed) _declare(fresh, TOY={"blinding": "unblinded"}) - code, out = _verify(fresh, part, capsys) + code, out = verify(part) assert code == 1 and "'blinding'" in out and "unblinded:TOY" in out # --------------------------------------------------------------------------- # -# I11: the audit proves blinded − true = shift(seed) +# I11: reveal publishes and archives; the audit proves blinded − true = shift(seed) # --------------------------------------------------------------------------- # -def _concealed(root, s): - """``s`` sealed under blind toy, before its seed is published.""" - return sio.seal(s, _custody(root, "TOY")) - - def _write(path, s, stamp): """``s`` on disk under ``stamp`` as given, as the door would never write it.""" s = s.copy() @@ -1026,33 +937,8 @@ def _write(path, s, stamp): s.save_fits(str(path), overwrite=True) -def _audit(root, archived, live, *, centres=("a", "a"), seed=None, stamps=None): - """Publish toy's seed (or ``seed``), archive ``archived`` with ``live`` as its - re-measured twin, stamped as ``stamps`` (toy's and TOY unblinded), and audit - the pair.""" - blinded = _custody(root, "TOY") - seed = seed or bd.open_blind(blinded).seed +def _publish(root, seed): (root / "blinds" / "toy" / "revealed.json").write_text(json.dumps({"seed": seed})) - stamps = stamps or (blinded.stamp, cu.Custody("unblinded", "TOY").stamp) - for tree, s, stamp, centre in ( - ("archive", archived, stamps[0], centres[0]), - ("live", live, stamps[1], centres[1]), - ): - s = s.copy() - s.metadata["patch_centers_sha256"] = centre - _write(root / tree / "sub" / "part.sacc", s, stamp) - return bd.audit( - "toy", - archive=root / "archive", - true_root=root / "live", - cat_config=root / "cat_config.yaml", - ) - - -def _problems(report): - return report["problems"] + [ - problem for part in report["parts"].values() for problem in part["problems"] - ] def test_reveal_refuses_a_root_holding_nothing_concealed(fresh): @@ -1078,130 +964,178 @@ def test_reveal_refuses_a_root_holding_nothing_concealed(fresh): def test_reveal_refuses_a_record_publishing_another_seed(fresh): - root = fresh / "output" - part = root / "part.sacc" + part = fresh / "output" / "part.sacc" part.parent.mkdir() sio.save(two_bin_sacc(cl=False), part, custody=_custody(fresh, "TOY")) - revealed = fresh / "blinds" / "toy" / "revealed.json" - revealed.write_text(json.dumps({"seed": "0" * 32})) + _publish(fresh, "0" * 32) with pytest.raises(cu.CustodyError, match="another seed"): - bd.reveal("toy", root=root, cat_config=fresh / "cat_config.yaml") + bd.reveal("toy", root=part.parent, cat_config=fresh / "cat_config.yaml") assert part.exists() -def test_the_audit_of_an_empty_archive_says_so(fresh): - (fresh / "blinds" / "toy" / "revealed.json").write_text( - json.dumps({"seed": bd.open_blind(_custody(fresh, "TOY")).seed}) - ) - (fresh / "archive").mkdir() - report = bd.audit( - "toy", - archive=fresh / "archive", - true_root=fresh, - cat_config=fresh / "cat_config.yaml", - ) - assert not report["ok"] and report["problems"] == ["the archive holds no parts"] +@pytest.fixture(scope="module") +def revealed(tmp_path_factory): + """A registry whose blind `toy` has published its seed, TOY's custody + before the reveal, ``AUDITED`` sealed under it, and ``audit(archived, + live)``, which archives ``archived`` beside ``live`` as its re-measured + twin and audits the pair.""" + root = _registry(tmp_path_factory.mktemp("revealed"), ("toy", "TOY")) + blinded = _custody(root, "TOY") + _publish(root, bd.open_blind(blinded).seed) + true = cu.Custody("unblinded", "TOY") + def audit( + archived, live, *, stamps=(blinded.stamp, true.stamp), centres=("a", "a") + ): + for tree, s, stamp, centre in zip( + ("archive", "live"), (archived, live), stamps, centres + ): + s = s.copy() + s.metadata["patch_centers_sha256"] = centre + _write(root / tree / "sub" / "part.sacc", s, stamp) + return bd.audit( + "toy", + archive=root / "archive", + true_root=root / "live", + cat_config=root / "cat_config.yaml", + ) -def test_the_audit_passes_on_a_true_reveal(fresh): - s = two_bin_sacc(rho=True) - report = _audit(fresh, _concealed(fresh, s), s) - assert report["ok"], report - ((path, part),) = report["parts"].items() - assert path == "sub/part.sacc" and part["residual"] <= 1e-6 - assert ( - abs(report["shift"]["S8"]) <= 0.075 and abs(report["shift"]["Omega_m"]) <= 0.1 + fields = ["root", "blinded", "sealed", "audit"] + return dataclasses.make_dataclass("Revealed", fields)( + root, blinded, sio.seal(AUDITED, blinded), audit ) -def test_the_audit_fails_on_a_wrong_seed(fresh): - s = two_bin_sacc() - report = _audit(fresh, _concealed(fresh, s), s, seed="not-the-seed") - assert not report["ok"] and "published seed" in _problems(report)[0] +def _problems(report): + return report["problems"] + [ + p for part in report["parts"].values() for p in part["problems"] + ] -def test_the_audit_fails_on_other_patch_centres(fresh): - s = two_bin_sacc() - report = _audit(fresh, _concealed(fresh, s), s, centres=("a", "b")) - assert not report["ok"] and "patch centres" in _problems(report)[0] +AUDITED = two_bin_sacc(xi_tags=("reporting",), rho=True) +BLOCKS = [ + (types, sio._pair(pair)) + for pair in PAIRS + for types in ((sio.XI_PLUS, sio.XI_MINUS), (sio.CL_EE,)) +] -SPOILT_PAIRS = { +@settings(max_examples=4, deadline=None) +@given( + times=st.lists( + st.sampled_from([1.0, 0.0, 2.0, 1 + 1e-4, -1.0]), min_size=6, max_size=6 + ) +) +@example(times=[1.0] * 6) +@example(times=[1.0] * 5 + [0.0]) # one block left true +@example(times=[1.0] * 3 + [2.0] + [1.0] * 2) # one block shifted twice +@example(times=[1 + 1e-4] + [1.0] * 5) +def test_the_audit_accepts_exactly_the_shift(revealed, times): + """An archived part whose ξ± and Cℓ_EE blocks carry ``times`` × the blind's + shift passes the audit exactly when every block carries it once.""" + true = AUDITED + shift = _values(revealed.sealed) - _values(true) + archived = true.copy() + for k, (types, tracers) in zip(times, BLOCKS): + for i, dp in enumerate(true.data): + if dp.data_type in types and dp.tracers == tracers: + archived.data[i].value = dp.value + k * shift[i] + report = revealed.audit(archived, true) + assert report["ok"] == all(k == 1.0 for k in times), report + if report["ok"]: + (part,) = report["parts"].values() + assert part["residual"] <= 1e-6 + assert ( + abs(report["shift"]["S8"]) <= 0.075 + and abs(report["shift"]["Omega_m"]) <= 0.1 + ) + else: + assert any("≠ shift(seed)" in problem for problem in _problems(report)) + + +SPOILT = { "covariance": "covariances differ", "rows": "rows, tags or tracers differ", "live_mock": "the live file is stamped mock:TOY", "live_other_catalogue": "the live file is stamped unblinded:OTHER", "archive_other_blind": "not concealed under this blind", + "patch_centres": "patch centres differ: ['a', 'b']", + "wrong_seed": "the published seed is not the committed one", } -@pytest.mark.parametrize("spoilt", SPOILT_PAIRS) -def test_the_audit_names_a_pair_that_is_not_concealed_and_true(fresh, spoilt): - """An archived part is audited only under this blind, against the same - catalogue re-measured unblinded with the same rows and covariance.""" - s = two_bin_sacc() - live = two_bin_sacc(cl=False) if spoilt == "rows" else s.copy() +@pytest.mark.parametrize("spoilt", SPOILT) +def test_the_audit_names_a_pair_that_is_not_concealed_and_true(revealed, spoilt): + """An archived part is audited only under the published committed seed, + against the same catalogue re-measured unblinded with the same rows, + covariance and patch centres.""" + true = AUDITED + live = ( + two_bin_sacc(xi_tags=("reporting",), cl=False) + if spoilt == "rows" + else true.copy() + ) if spoilt == "covariance": - live.add_covariance(1.01 * np.asarray(s.covariance.dense), overwrite=True) - archived_stamp = _custody(fresh, "TOY").stamp + live.add_covariance(1.01 * np.asarray(true.covariance.dense), overwrite=True) + archived_stamp = revealed.blinded.stamp if spoilt == "archive_other_blind": archived_stamp = {**archived_stamp, "blinding_commitment": "0" * 64} live_custody = { "live_mock": cu.Custody("mock", "TOY"), "live_other_catalogue": cu.Custody("unblinded", "OTHER"), }.get(spoilt, cu.Custody("unblinded", "TOY")) - stamps = (archived_stamp, live_custody.stamp) - report = _audit(fresh, _concealed(fresh, s), live, stamps=stamps) - assert not report["ok"] and _problems(report) == [SPOILT_PAIRS[spoilt]] + published = revealed.root / "blinds" / "toy" / "revealed.json" + seed = published.read_text() + try: + if spoilt == "wrong_seed": + _publish(revealed.root, "not-the-seed") + report = revealed.audit( + revealed.sealed, + live, + stamps=(archived_stamp, live_custody.stamp), + centres=("a", "b" if spoilt == "patch_centres" else "a"), + ) + finally: + published.write_text(seed) + assert not report["ok"] and _problems(report) == [SPOILT[spoilt]] -@pytest.mark.parametrize( - "times", [0, 2, 1 + 1e-4], ids=["left_true", "shifted_twice", "off_by_1e-4"] -) -def test_the_audit_fails_on_a_mixed_file(fresh, times): - """One ξ± pair under the blinded stamp carries 0×, 2× or 1.0001× the shift.""" - s = two_bin_sacc(rho=True) - archived = _concealed(fresh, s) - for i in _rows(s, sio.XI_PLUS, sio.XI_MINUS): - if s.data[i].tracers == sio._pair((1, 1)): - shift = archived.data[i].value - s.data[i].value - archived.data[i].value = s.data[i].value + times * shift - report = _audit(fresh, archived, s) - assert not report["ok"] - assert any("≠ shift(seed)" in problem for problem in _problems(report)) +def test_the_audit_of_an_empty_archive_says_so(revealed, tmp_path): + report = bd.audit( + "toy", + archive=tmp_path, + true_root=tmp_path, + cat_config=revealed.root / "cat_config.yaml", + ) + assert not report["ok"] and report["problems"] == ["the archive holds no parts"] -@pytest.mark.parametrize( - "make", [rho_sacc, two_bin_sacc], ids=["alone", "beside_signal"] -) -def test_the_audit_passes_rho_tau_remeasured_at_run_noise(fresh, make): +@pytest.mark.parametrize("beside_signal", [False, True], ids=["alone", "beside_signal"]) +def test_the_audit_passes_rho_tau_remeasured_at_run_noise(revealed, beside_signal): """ρ/τ carries no signal, and a re-run does not reproduce it (TreeCorr's k-means patches move θ by ~1e-3 and the values by O(1)): its part is judged by its stamp, and beside signal rows only the signal is compared.""" - true = make(rho=True) if make is two_bin_sacc else make() - if true.covariance is None: - true.add_covariance(np.full(len(true.mean), 1e-14)) - archived = _concealed(fresh, true) + true = AUDITED if beside_signal else rho_sacc() rerun = true.copy() - rho = [i for i, dp in enumerate(true.data) if not sio.is_signal(dp.data_type)] cov = np.array(true.covariance.dense) - for i in rho: - rerun.data[i].value = 1.5 * true.data[i].value + 1e-7 - rerun.data[i].tags["theta"] *= 1 + 1.7e-3 - cov[i, i] *= 1.036 + for i, dp in enumerate(true.data): + if not sio.is_signal(dp.data_type): + rerun.data[i].value = 1.5 * dp.value + 1e-7 + rerun.data[i].tags["theta"] *= 1 + 1.7e-3 + cov[i, i] *= 1.036 rerun.add_covariance(cov, overwrite=True) - report = _audit(fresh, archived, rerun) + archived = revealed.sealed if beside_signal else sio.seal(true, revealed.blinded) + report = revealed.audit(archived, rerun) assert report["ok"], report @pytest.mark.parametrize("kind", [sio.CL_BB, sio.CL_EB]) -def test_the_audit_fails_when_an_unshifted_row_moved(fresh, kind): - s = two_bin_sacc(rho=True) - archived = _concealed(fresh, s) - i = _rows(s, kind)[-1] - archived.data[i].value += 1e-6 * abs(s.data[i].value) - report = _audit(fresh, archived, s) - assert not report["ok"] +def test_the_audit_fails_when_an_unshifted_row_moved(revealed, kind): + true = AUDITED + archived = revealed.sealed.copy() + i = _rows(true, kind)[-1] + archived.data[i].value += 1e-6 * abs(true.data[i].value) + report = revealed.audit(archived, true) assert _problems(report) == [f"{kind} moved, but the blind leaves it unshifted"] @@ -1214,16 +1148,16 @@ def test_the_audit_fails_when_an_unshifted_row_moved(fresh, kind): ids=["cosebis", "pure_eb"], ) @pytest.mark.parametrize("b_sigma", [0.0, 1.0]) -def test_the_audit_bounds_derived_b_modes(fresh, e_kind, b_kind, b_sigma): +def test_the_audit_bounds_derived_b_modes(revealed, e_kind, b_kind, b_sigma): """A derived part's E rows move with the blind; its B rows by ≤ ``B_SIGMA``.""" - true = cosebis_sacc() if e_kind == sio.COSEBI_EE else pure_eb_sacc() - true.add_covariance(np.full(len(true.mean), 0.01)) + true = derived_sacc("cosebis" if e_kind == sio.COSEBI_EE else "pure_eb") + true.add_covariance(np.full(len(true.mean), 0.01), overwrite=True) archived = true.copy() for i in _rows(true, e_kind): archived.data[i].value += 0.3 for i in _rows(true, b_kind): archived.data[i].value += b_sigma * 0.1 - report = _audit(fresh, archived, true) + report = revealed.audit(archived, true) assert report["ok"] == (b_sigma == 0.0), report (part,) = report["parts"].values() assert part["shift_over_sigma"][e_kind] == pytest.approx(3.0) diff --git a/src/sp_validation/tests/test_custody.py b/src/sp_validation/tests/test_custody.py index 86671144..6e88f633 100644 --- a/src/sp_validation/tests/test_custody.py +++ b/src/sp_validation/tests/test_custody.py @@ -13,6 +13,8 @@ import pytest import yaml +from hypothesis import example, given, settings +from hypothesis import strategies as st from sp_validation import custody as cu @@ -61,14 +63,79 @@ def _catalogues(**declarations): # --------------------------------------------------------------------------- # # I12: the custody table # --------------------------------------------------------------------------- # -def test_undeclared_catalogue_without_a_blind_fails_with_the_command(tmp_path): - cats = _catalogues(SP_v9=None) - with pytest.raises(cu.CustodyError) as err: - cu.custody_of(cats, "SP_v9", registry=tmp_path / "blinds") - message = str(err.value) - assert "declares no custody for it, so it is blinded" in message - assert "python -m sp_validation.blinding init" in message - assert "share" in message +NO_BLIND = [ + "declares no custody for it, so it is blinded", + "blinding init", + "blinding share", +] +Y3 = {"y3": (["SP_v9"], None)} # a blind over SP_v9, concealed +PUBLIC = {"y3": (["SP_v9"], "seed-of-y3")} # the same, its seed published +V = "SP_v9_ecut07" +# case: (declarations, blinds {name: (bases, published seed)}, expected, version). +# expected is the custody's token, or the fragments of the refusal. +# fmt: off +TABLE = { + "undeclared": ({"SP_v9": None}, {}, NO_BLIND), + "blinded": ({"SP_v9": "blinded"}, {}, ["no blind covers it"]), + "undeclared_covered": ({"SP_v9": None}, Y3, "blinded:SP_v9:y3:{y3}"), + "blinded_covered": ({"SP_v9": "blinded"}, Y3, "blinded:SP_v9:y3:{y3}"), + "blinded_revealed": ({"SP_v9": None}, PUBLIC, ["declare `blinding: unblinded`"]), + "unblinded": ({"SP_v9": "unblinded"}, {}, "unblinded:SP_v9"), + "unblinded_concealed": ({"SP_v9": "unblinded"}, Y3, ["blinding reveal y3"]), + "unblinded_revealed": ({"SP_v9": "unblinded"}, PUBLIC, "unblinded:SP_v9"), + "unblinded_wrong_seed": ( + {"SP_v9": "unblinded"}, {"y3": (["SP_v9"], "not-the-seed")}, ["its commitment"] + ), + "mock": ({"SP_v9": "mock"}, {}, "mock:SP_v9"), + "mock_covered": ({"SP_v9": "mock"}, Y3, ["a mock is never blinded"]), + "two_blinds": ({"SP_v9": None}, {**Y3, "b": (["SP_v9"], None)}, ["blinds: b, y3"]), + "unknown": ({"SP_v9": "open"}, {}, ["blinded, unblinded or mock"]), + "variant_declares": ( + {"SP_v9": "unblinded", V: {"base": "SP_v9", "blinding": "unblinded"}}, {}, + ["declare custody on SP_v9"], V, + ), + "alias_repeats": ( + {"SP_v9": "unblinded", "SP_v9_leak_corr": "unblinded"}, {}, "unblinded:SP_v9", + "SP_v9_leak_corr", + ), + "alias_repeats_base": ( + {"SP_v9": "unblinded", V: {"base": "SP_v9"}, f"{V}_seed7": {"base": "SP_v9"}}, + {}, "unblinded:SP_v9", f"{V}_seed7", + ), + "alias_declares": ( + {"SP_v9": "unblinded", "SP_v9_leak_corr": "blinded"}, {}, + ["declare custody on SP_v9"], "SP_v9_leak_corr", + ), + "alias_names_another_base": ( + {"SP_v9": "unblinded", "TOY": None, "SP_v9_seed7": {"base": "TOY"}}, + {"y3": (["TOY"], None)}, ["declare custody on SP_v9"], "SP_v9_seed7", + ), +} +# fmt: on + + +@pytest.mark.parametrize("case", TABLE) +def test_the_custody_table(tmp_path, case): + """Every row of custody_of's table: the declaration on the base catalogue + against the blinds covering it. An entry named as a variant of another + (``_leak_corr``, ``_seed``) may repeat that entry's + custody, never declare another.""" + declarations, blinds, expected, *version = TABLE[case] + version = version[0] if version else "SP_v9" + registry = tmp_path / "blinds" + commitments = { + name: cu.seed_commitment(_write_blind(registry, name, bases, revealed=seed)) + for name, (bases, seed) in blinds.items() + } + cats = _catalogues(**declarations) + if isinstance(expected, str): + custody = cu.custody_of(cats, version, registry=registry) + assert custody.token == expected.format(**commitments) + return + with pytest.raises(cu.CustodyError) as refused: + cu.custody_of(cats, version, registry=registry) + for fragment in expected: + assert fragment in str(refused.value) def test_the_printed_commands_name_the_config_as_it_was_given(tmp_path): @@ -86,129 +153,26 @@ def test_the_printed_commands_name_the_config_as_it_was_given(tmp_path): assert f"--cat-config {checkout}/cosmo_val/cat_config.yaml" in message -def test_declared_blinded_without_a_blind_fails(tmp_path): - cats = _catalogues(SP_v9="blinded") - with pytest.raises(cu.CustodyError, match="no blind covers it"): - cu.custody_of(cats, "SP_v9", registry=tmp_path / "blinds") - +SUFFIXES = st.lists( + st.one_of(st.just("_leak_corr"), st.integers(0, 99999).map("_seed{:05d}".format)), + max_size=3, +) -@pytest.mark.parametrize("declaration", [None, "blinded"]) -def test_blinded_under_the_covering_blind(tmp_path, declaration): - registry = tmp_path / "blinds" - seed = _write_blind(registry, "y3", ["SP_v9"]) - c = cu.custody_of(_catalogues(SP_v9=declaration), "SP_v9", registry=registry) - assert c.status == "blinded" and c.catalogue == "SP_v9" and c.blind == "y3" - assert c.commitment == cu.seed_commitment(seed) - assert c.token == f"blinded:SP_v9:y3:{c.commitment}" - assert c.stamp["blinding"] == "blinded" - assert c.stamp["blinding_commitment"] == c.commitment - - -def test_blinded_under_a_revealed_blind_fails(tmp_path): - registry = tmp_path / "blinds" - seed = _write_blind(registry, "y3", ["SP_v9"]) - (registry / "y3" / "revealed.json").write_text(json.dumps({"seed": seed})) - with pytest.raises(cu.CustodyError, match="declare `blinding: unblinded`"): - cu.custody_of(_catalogues(SP_v9=None), "SP_v9", registry=registry) - -def test_unblinded_without_a_blind(tmp_path): - c = cu.custody_of( - _catalogues(SP_v9="unblinded"), "SP_v9", registry=tmp_path / "blinds" - ) - assert (c.status, c.token) == ("unblinded", "unblinded:SP_v9") - assert c.stamp == {"blinding": "unblinded", "blinding_catalogue": "SP_v9"} - - -def test_unblinding_a_concealed_catalogue_is_the_reveal(tmp_path): - registry = tmp_path / "blinds" - _write_blind(registry, "y3", ["SP_v9"]) - with pytest.raises(cu.CustodyError, match="blinding reveal y3"): - cu.custody_of(_catalogues(SP_v9="unblinded"), "SP_v9", registry=registry) - - -def test_unblinded_after_a_reveal_whose_seed_matches(tmp_path): - registry = tmp_path / "blinds" - seed = _write_blind(registry, "y3", ["SP_v9"]) - (registry / "y3" / "revealed.json").write_text(json.dumps({"seed": seed})) - c = cu.custody_of(_catalogues(SP_v9="unblinded"), "SP_v9", registry=registry) - assert c.token == "unblinded:SP_v9" - - -def test_a_published_seed_that_misses_the_commitment_fails(tmp_path): - registry = tmp_path / "blinds" - _write_blind(registry, "y3", ["SP_v9"], revealed="not-the-seed") - with pytest.raises(cu.CustodyError, match="commitment"): - cu.custody_of(_catalogues(SP_v9="unblinded"), "SP_v9", registry=registry) - - -def test_mock(tmp_path): - registry = tmp_path / "blinds" - c = cu.custody_of(_catalogues(SP_v9="mock"), "SP_v9", registry=registry) - assert (c.status, c.token) == ("mock", "mock:SP_v9") - _write_blind(registry, "y3", ["SP_v9"]) - with pytest.raises(cu.CustodyError, match="mock"): - cu.custody_of(_catalogues(SP_v9="mock"), "SP_v9", registry=registry) - - -def test_two_covering_blinds_fail(tmp_path): - registry = tmp_path / "blinds" - _write_blind(registry, "a", ["SP_v9"]) - _write_blind(registry, "b", ["SP_v9"]) - with pytest.raises(cu.CustodyError, match="a, b"): - cu.custody_of(_catalogues(SP_v9=None), "SP_v9", registry=registry) - - -def test_a_variant_entry_may_not_declare_custody(tmp_path): - cats = _catalogues( - SP_v9="unblinded", SP_v9_ecut07={"base": "SP_v9", "blinding": "unblinded"} - ) - with pytest.raises(cu.CustodyError, match="declare custody on SP_v9"): - cu.custody_of(cats, "SP_v9_ecut07", registry=tmp_path / "blinds") - - -def test_an_unknown_declaration_fails(tmp_path): - with pytest.raises(cu.CustodyError, match="blinded, unblinded or mock"): - cu.custody_of(_catalogues(SP_v9="open"), "SP_v9", registry=tmp_path / "blinds") - - -def test_variants_share_their_base(tmp_path): - """`_leak_corr`, `_seed` and `base:` entries resolve to the base.""" - registry = tmp_path / "blinds" +@settings(max_examples=30, deadline=None) +@given(entry=st.sampled_from(["SP_v9", "SP_v9_ecut07"]), suffixes=SUFFIXES) +@example(entry="SP_v9", suffixes=["_leak_corr"]) +@example(entry="SP_v9_ecut07", suffixes=["_seed00042", "_leak_corr"]) +def test_every_variant_shares_its_base(tmp_path_factory, entry, suffixes): + """``_leak_corr`` and ``_seed`` versions, in any order, of a catalogue + or of an entry naming its parent with ``base:``, resolve to the base's + custody and blind.""" + registry = tmp_path_factory.mktemp("variants") / "blinds" _write_blind(registry, "y3", ["SP_v9"]) cats = _catalogues(SP_v9=None, SP_v9_ecut07={"base": "SP_v9"}) + version = entry + "".join(suffixes) base = cu.custody_of(cats, "SP_v9", registry=registry) - for version in ( - "SP_v9_leak_corr", - "SP_v9_seed00042", - "SP_v9_seed00042_leak_corr", - "SP_v9_leak_corr_seed00042", - "SP_v9_ecut07", - "SP_v9_ecut07_leak_corr", - ): - assert cu.custody_of(cats, version, registry=registry) == base, version - - -def test_an_entry_named_as_a_variant_shares_its_parents_custody(tmp_path): - """An entry named `_leak_corr` or `_seed` resolves through - that entry, as the variant it is named for does. It may repeat the entry's - `blinding` or `base`, as the copies `overwrite_config` writes do, but a - declaration of its own is refused, never ignored.""" - registry = tmp_path / "blinds" - _write_blind(registry, "y3", ["TOY"]) - cats = _catalogues(PARENT="unblinded", TOY=None, EC={"base": "PARENT"}) - copies = dict( - cats, PARENT_leak_corr=dict(cats["PARENT"]), EC_seed7=dict(cats["EC"]) - ) - for version in ("PARENT_leak_corr", "EC_seed7"): - custody = cu.custody_of(copies, version, registry=registry) - assert custody.token == "unblinded:PARENT", version - for alias, claim in { - "PARENT_leak_corr": {"blinding": "blinded"}, - "PARENT_seed7": {"base": "TOY"}, - }.items(): - with pytest.raises(cu.CustodyError, match="declare custody on PARENT"): - cu.custody_of({**cats, alias: claim}, alias, registry=registry) + assert cu.custody_of(cats, version, registry=registry) == base def _reading(cats, path, *names): diff --git a/uv.lock b/uv.lock index eb69507c..f0fec619 100644 --- a/uv.lock +++ b/uv.lock @@ -1372,6 +1372,60 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, ] +[[package]] +name = "hypothesis" +version = "6.168.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/10/5795cc413d2ba5ff9c81e5d1b27ba3c652ac6dfac6b1d67072052d338391/hypothesis-6.168.2.tar.gz", hash = "sha256:df61fcada928a3fb3fba4602be5b2d569eb50c36fddae0d2e2a5f51419d0ce9b", size = 510987, upload-time = "2026-09-27T05:22:19.959Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/79/989e20cebec9fef907c43e34dae9c0938b49e2d3bb8537a79990692471c0/hypothesis-6.168.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8268bcc733f2b93e77f64e54d6fa61db79af91806e2d26b950f5d1cca47b4bfa", size = 1124033, upload-time = "2026-09-27T05:20:17.522Z" }, + { url = "https://files.pythonhosted.org/packages/ec/44/3b5edc918128eccea902eb6ee91f0c58ad575274b99c565107fc575483a9/hypothesis-6.168.2-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c138d9848c210b8a1e511ab0884f16bc71470400b11255e8b595df320809afc", size = 1147868, upload-time = "2026-09-27T05:20:44.835Z" }, + { url = "https://files.pythonhosted.org/packages/2a/54/04d30544681626fb7fa25cc1041c12ebfad79aa4995438f6f3e577541111/hypothesis-6.168.2-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:420479d419d73dd7bb2ed0cfb57f35b1dc0bc246c30b250904c3987c5cae9463", size = 1149472, upload-time = "2026-09-27T05:21:49.503Z" }, + { url = "https://files.pythonhosted.org/packages/14/86/a7abe9660b4b997305537aa860fb932f97223125713655a7a1e9a302f629/hypothesis-6.168.2-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:01f74944abe6988a6213dd036bd2dbf1e4f0513af0cf66b7780773fdf0c6f5e2", size = 1191885, upload-time = "2026-09-27T05:22:15.685Z" }, + { url = "https://files.pythonhosted.org/packages/31/d0/24c77c25ec62941022edcbb98a052b98c57b60fee7d289cabea73523a727/hypothesis-6.168.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe84b8218a4fc33f79e64840d23fe9c627e7acb7cad1f70bf6c0f163c7f4f01f", size = 1169899, upload-time = "2026-09-27T05:20:13.379Z" }, + { url = "https://files.pythonhosted.org/packages/cb/23/dd129e570f44051de911761cada83db5695bbbe3db450bc905d449ea8aa2/hypothesis-6.168.2-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:ce74728a05d5e5dd4d28bb4fbaa0c1f5e809ebf6be54d343128d970ee1025697", size = 1129354, upload-time = "2026-09-27T05:20:10.768Z" }, + { url = "https://files.pythonhosted.org/packages/71/65/6d911e5b270c7f9560929b07304869a1f77e2440e22db107600788975bbb/hypothesis-6.168.2-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:29e12bcfcef363a9dbd0d3d62fd9608853268d15319ae3ecaf802555497392a9", size = 1160386, upload-time = "2026-09-27T05:22:05.475Z" }, + { url = "https://files.pythonhosted.org/packages/26/87/5cb26f7a2323ecdb4881f06c8eec19c4087e04b23cc7998b74f2b565a66c/hypothesis-6.168.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3cbe12b2427a86d769fdc93242ee3207d3a1523798b856630f16b674ebae17b1", size = 1299894, upload-time = "2026-09-27T05:20:02.55Z" }, + { url = "https://files.pythonhosted.org/packages/d7/f3/eb40e5c554b78b1b500b93172829b4b16c677acadd3f10f979117a82f851/hypothesis-6.168.2-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:3b62ce43973f4425530e590751439bacceec19921af0edf9fae3a63e173ea146", size = 1425524, upload-time = "2026-09-27T05:20:58.58Z" }, + { url = "https://files.pythonhosted.org/packages/2d/1e/03819636fa86ade2db91ad9953851b49f885a3481a22f0b1361d0c5dfc8d/hypothesis-6.168.2-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:51df0068106dd2db4fc5a1e56801adf5fc1d354792584289a9cc8ac09273d376", size = 1377070, upload-time = "2026-09-27T05:21:04.963Z" }, + { url = "https://files.pythonhosted.org/packages/5b/80/505146f225bbd24759651a16c17cd44aa6f082fa4a14761646e8ec37f374/hypothesis-6.168.2-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:e54638a0f66c608840af13f3ad665710e706bda5c52f706351bb83b5744bec63", size = 1281230, upload-time = "2026-09-27T05:20:04.007Z" }, + { url = "https://files.pythonhosted.org/packages/ff/7b/3c9f0f1cf0733b6d692dd5dca92037f6b707082bfb4c29712f2d0258049e/hypothesis-6.168.2-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:84c5b9891f979e851d140618783e9c3624352760dd78cba4759b263686cdad16", size = 1300413, upload-time = "2026-09-27T05:20:14.695Z" }, + { url = "https://files.pythonhosted.org/packages/81/dc/56c1626009545b86fefa71ee71d273fe518d1374beca40bc9b784818a58e/hypothesis-6.168.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6a6ac0836daba2da486a4fefc22fcdd794dac350bd787a1bb896748cdc47e05e", size = 1336267, upload-time = "2026-09-27T05:21:23.381Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c4/853cc464739bfd076e0f6bac55c4fcff362effdcfc10873bd1a119e80994/hypothesis-6.168.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e28cb8912d7811dcec91b2389d9dcf3b34523c043cfb7454f9c62c8e5a40b2a", size = 1123030, upload-time = "2026-09-27T05:20:38.642Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6a/506d3c0e45bf8cfb6bce2842b0642df37db07f2d5695f4614fa410e854cc/hypothesis-6.168.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3630d664a343354701618a409d3b9d563cb97cfcf749c7a37c5fb32cb3c7084c", size = 1169073, upload-time = "2026-09-27T05:20:55.361Z" }, + { url = "https://files.pythonhosted.org/packages/82/52/244eff24dbaa1f9520dcb42164fb3769e6c08a2e252455af2e8bd23305e5/hypothesis-6.168.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a0df395d27012b07eeed98b2e3f2a17f693236b0518d8a086679dc657801aead", size = 1298741, upload-time = "2026-09-27T05:20:32.609Z" }, + { url = "https://files.pythonhosted.org/packages/2e/99/cdcb4fa1b738ef07827464bdf5ff83897f8303e8eca569207cc52f5be2b2/hypothesis-6.168.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:778f043b8af618baabe79cdb15f6f7855ccf950c6911eea84acabca5b7637cde", size = 1335252, upload-time = "2026-09-27T05:20:09.229Z" }, + { url = "https://files.pythonhosted.org/packages/a5/49/63f0a6b5b709e2bdd0d1d0041c48c6d4614323e27c677a0d44d9748dd4cb/hypothesis-6.168.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97437cf297feb9645c23fd187734c137989c503303542a4034eb43d07f8f151e", size = 1123008, upload-time = "2026-09-27T05:20:01.27Z" }, + { url = "https://files.pythonhosted.org/packages/a5/21/284046912c05973f91d41bd7ed79e3bf1917b1332b70e677c3cf2872cea8/hypothesis-6.168.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5bd1b6334eaccde40ab56ffb96788f2af993869b08aebc4f1b3ff28f3edcda9", size = 1168965, upload-time = "2026-09-27T05:20:18.818Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b7/afe659dccd780393ca6beceabc3a418f11ff2539a83bc4ce9e6c8a0ab589/hypothesis-6.168.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac134d9fe4a1623b57b2a1681e3f51c2c5b324e690c1264dc01a3c57c445b08a", size = 1298607, upload-time = "2026-09-27T05:20:21.424Z" }, + { url = "https://files.pythonhosted.org/packages/e3/e0/d6300216c75fe878da9bc8bfc5e266baa1d350ea145001272d20b3ff3843/hypothesis-6.168.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:03ba2a0767955fd6491cb93e6917db1d5b718a89f9a4de33b0827b7fce1df289", size = 1335166, upload-time = "2026-09-27T05:20:53.745Z" }, + { url = "https://files.pythonhosted.org/packages/5e/2a/11e427ed3f09f0e1d548428144e97fb9bfbdb2f41b6871880625acf41411/hypothesis-6.168.2-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db13338672ba689203f405c6258ef62b0590e3d193714b30809e53e8bfc66e84", size = 1123318, upload-time = "2026-09-27T05:21:30.573Z" }, + { url = "https://files.pythonhosted.org/packages/89/03/80911aca76e411cb1d1e7a5deaea0a49b47321ccd85092ceba971100b921/hypothesis-6.168.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:170a6d2d6d9d209992db1e671d0c2ad17367c8046482ebff2cbc9e14a9698be7", size = 1169146, upload-time = "2026-09-27T05:20:20.159Z" }, + { url = "https://files.pythonhosted.org/packages/b3/19/243f18213e39924db4be21b53e3999c183d12954745254e717a639fc3490/hypothesis-6.168.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9310d3f997f3d5e8e31d5c5f9946bcd3ccf6eb62c92a1b728f2dd57fd62fda05", size = 1299106, upload-time = "2026-09-27T05:21:06.885Z" }, + { url = "https://files.pythonhosted.org/packages/6a/47/abc5881cf0cdce97663b877f486ff064d834850ea19171ba556c7dd835ae/hypothesis-6.168.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d66e68a0dc9e216f9bdda2c9f066c09e36849cbcb49fbcb1860087560709bb8d", size = 1335374, upload-time = "2026-09-27T05:20:46.633Z" }, + { url = "https://files.pythonhosted.org/packages/d1/f9/08d4bba84eab0404b42971af4ee21c4f9f2064444dc7c8040a14f722ffea/hypothesis-6.168.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f98f7581f68acc659510f8786c39c5cf140dbc3dacba6cd0ed059b0e296a158b", size = 624273, upload-time = "2026-09-27T05:20:16.14Z" }, + { url = "https://files.pythonhosted.org/packages/bb/e0/92fc641a751af530859023ea6ae6f73c2294587240b78884aa155fccbe5a/hypothesis-6.168.2-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:798ae9a91ab4cd8cfadd7991f46dc9c12fd43d2997f29744b7eae755ff36027a", size = 1121651, upload-time = "2026-09-27T05:21:40.104Z" }, + { url = "https://files.pythonhosted.org/packages/a9/1a/405c4c9b888721fc3b650ffc3c86d8507450e1ea105e88c97e2e5f277aa6/hypothesis-6.168.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00f8728aaef10c91688dab34df8a59aed83c76ade869b1812f01ac80808bc5cc", size = 1167748, upload-time = "2026-09-27T05:21:41.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/70/2f7559b159f98e14d950914a0aeb25b0e2ed473f35705192de4a2c20b3e8/hypothesis-6.168.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:63b183d5436688e54389f54b66d1611aef349f36345b6dc0a79ccef0dbc736c8", size = 1297293, upload-time = "2026-09-27T05:21:03.359Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c3/b197f8353c3026b76da82d49a119849cd9eac7b0be96239414d1d748156c/hypothesis-6.168.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:28dfe52bcf6e6621153cfbb5574223d117a3e66b24ffeacb4c9bc8f751190dc6", size = 1334246, upload-time = "2026-09-27T05:21:27.001Z" }, + { url = "https://files.pythonhosted.org/packages/63/98/55f83a1a9cb643866c27211a991cf56a012126a6d5548a02877df1334ab0/hypothesis-6.168.2-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3001d9e6013ebb82d9d7daa97efd75a383e628bf2b5d754512b56191898e377", size = 1121134, upload-time = "2026-09-27T05:21:32.576Z" }, + { url = "https://files.pythonhosted.org/packages/87/dd/2a642db35341893ee604d8cd0b8e6232ddc848e9fe04d6a17fd30cdbc586/hypothesis-6.168.2-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:44b7c3d431cca39342881761dfe3e34b39c9b5e7b0ad5c8c4cb7d92514f3b94c", size = 1144046, upload-time = "2026-09-27T05:20:06.616Z" }, + { url = "https://files.pythonhosted.org/packages/61/13/78e7b3192d4ccd19bbf36ebc0a678a8291f40093baee47fdded4240d2ca7/hypothesis-6.168.2-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e78ce758211e2fd049c358c979c0c1312fc0f6e13a829cf2a4828a9ebb1423f8", size = 1146678, upload-time = "2026-09-27T05:21:34.447Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6a/daa33a3e9eb999da96bd7315d1f5446c5e28866191e2007aad9b1fb6616f/hypothesis-6.168.2-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9aec441b5d7156e64285817fda72b0aad37aa3dd262693708e825858930b7b48", size = 1189433, upload-time = "2026-09-27T05:21:55.422Z" }, + { url = "https://files.pythonhosted.org/packages/28/49/138fa2c61ec12ea7b17a1693a17ce0567ede4e61dac65d3304664493641b/hypothesis-6.168.2-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3510ca51dfd5aa7eb24f39d65714f6a91297fa5382866124b931bf56acd207fd", size = 1167045, upload-time = "2026-09-27T05:22:13.561Z" }, + { url = "https://files.pythonhosted.org/packages/e5/04/38474d9c6bc119700b6eb6fdb73ff8775202af79d5a7ee3bd06619d1b5a0/hypothesis-6.168.2-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:8d7943a4c8dcca314bfd918247acea4a36fd3476167372b19d8db3070cdefa4f", size = 1126814, upload-time = "2026-09-27T05:21:43.835Z" }, + { url = "https://files.pythonhosted.org/packages/99/be/048888e2f1bfe412c46788443079bb03b6c299c0da85f11fdb536f27dfa4/hypothesis-6.168.2-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:903cc76f9ebd36d555039aac270f5652cf0a93cfbbac344915ee3529e917efa3", size = 1155839, upload-time = "2026-09-27T05:20:22.76Z" }, + { url = "https://files.pythonhosted.org/packages/3a/67/0a607ac12fedbd8b2be26b360013df3b064e6b80d7313db635bfb5cc75de/hypothesis-6.168.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:d6f060f4bcf10ac087c34da08fc9f3be979dcfb70d45a238a6fe6298f1cc1ae9", size = 1296640, upload-time = "2026-09-27T05:21:28.848Z" }, + { url = "https://files.pythonhosted.org/packages/d0/f8/f59636b8b9b8c6de5f02b752faba3c7592a13dd767ddc96bf582b8870720/hypothesis-6.168.2-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:8574573eea6693792966eb6c8da079fe63d8dafb884c613c9857b3a13052df45", size = 1422025, upload-time = "2026-09-27T05:20:07.72Z" }, + { url = "https://files.pythonhosted.org/packages/a0/f4/e36c4a95b7997e213cd85c3be1232c8ac19e1ecf20d99512ea279eade321/hypothesis-6.168.2-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:5714ac1e72ba5faa645f593de353af34a8b1de8bcff811f1d3da4c1b8628e1c3", size = 1374166, upload-time = "2026-09-27T05:21:01.762Z" }, + { url = "https://files.pythonhosted.org/packages/18/9e/b9c3e282e9833e75875bc9163f80c49f8153d0f96ce5ba034a7997f13850/hypothesis-6.168.2-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:dffa4c97f0b473f1da456807ff57b348fb675f7728a15faab4eea87dffb6c77a", size = 1278387, upload-time = "2026-09-27T05:20:05.125Z" }, + { url = "https://files.pythonhosted.org/packages/af/96/84ed26fd0855ad8d6d67413893c835e90a77294c1255e3e78c13fe082f0f/hypothesis-6.168.2-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:29c1d761ef45e063eba2f8287e0572976f6b758b6832d75d7e03d02a610d3a91", size = 1297784, upload-time = "2026-09-27T05:20:28.101Z" }, + { url = "https://files.pythonhosted.org/packages/94/19/c94f1bce7e72950f8f3c5ee9d34ba98a7da70548453a96befa86281bfd94/hypothesis-6.168.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c7ab9ecf409143ef26a0d20a5f373d2add8871be58028cdbd3a515b0ef43ac45", size = 1333959, upload-time = "2026-09-27T05:21:16.771Z" }, +] + [[package]] name = "idna" version = "3.18" @@ -3813,6 +3867,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl", hash = "sha256:7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752", size = 104164, upload-time = "2026-06-03T00:56:38.614Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "soupsieve" version = "2.8.4" @@ -3872,6 +3935,7 @@ dependencies = [ [package.optional-dependencies] develop = [ + { name = "hypothesis" }, { name = "myst-parser" }, { name = "numpydoc" }, { name = "pytest" }, @@ -3895,6 +3959,7 @@ glass = [ { name = "glass-ext-camb" }, ] test = [ + { name = "hypothesis" }, { name = "pytest" }, { name = "pytest-cov" }, { name = "ruff" }, @@ -3927,6 +3992,7 @@ requires-dist = [ { name = "h5py" }, { name = "healpy" }, { name = "healsparse" }, + { name = "hypothesis", marker = "extra == 'test'" }, { name = "importlib-metadata" }, { name = "joblib", specifier = ">=0.13" }, { name = "jupyter" }, @@ -3951,7 +4017,7 @@ requires-dist = [ { name = "regions" }, { name = "reproject" }, { name = "ruff", marker = "extra == 'test'" }, - { name = "sacc", specifier = ">=0.12" }, + { name = "sacc", specifier = ">=2.4,<3" }, { name = "scipy", specifier = "<1.19" }, { name = "seaborn" }, { name = "shear-psf-leakage", git = "https://github.com/CosmoStat/shear_psf_leakage.git?rev=develop" }, From e62ebc24b46d7484760cb405f8a0680b4361ab1e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 03:53:10 +0200 Subject: [PATCH 113/160] tests: DAG properties read one forced listing; e2e drops unit-guarded checks test_dag parses `snakemake -F -n all` once for P1, P2, P9, P10, P12 and P13, and the launches the DAG refuses (no blind, unrevealed blind, campaign type, missing patch centres) are one table. The end-to-end test no longer repeats the laundering refusal and two-bin seal, which the blinding properties guard. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- src/sp_validation/tests/test_custody_e2e.py | 19 +- workflow/tests/test_dag.py | 312 +++++++++----------- 2 files changed, 143 insertions(+), 188 deletions(-) diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py index 57394c9e..8c7e4e6b 100644 --- a/src/sp_validation/tests/test_custody_e2e.py +++ b/src/sp_validation/tests/test_custody_e2e.py @@ -4,7 +4,7 @@ unblinded, ``TOY_MOCK`` mock). A blind is drawn for ``TOY`` with ``blinding init``; the rule scripts run as Snakemake runs them (``runpy`` with a ``snakemake`` object) for ``TOY`` and ``TOY_leak_corr``, on patch centres drawn -by their command: both ξ± grids, the ξ± figures, a two-bin writer, pseudo-Cℓ on +by their command: both ξ± grids, the ξ± figures, pseudo-Cℓ on an nside-32 NaMaster workspace, ρ/τ, COSEBIs, pure-E/B and assembly. The blind is then revealed, the declaration flipped, the chain re-run, and the audit must prove blinded − true = shift(seed) on every part the reveal archived. Neither @@ -416,7 +416,7 @@ def test_a_blinded_catalogue_from_birth_to_audit(toy, monkeypatch): == 0 ) - # A two-bin writer conceals every pair by passing the custody. + # A two-bin part, for the interruption below. two_bin = sio.new_sacc( { 0: sio.get_nz(sio.load(out / "TOY_xi_reporting.sacc"), 0), @@ -426,8 +426,6 @@ def test_a_blinded_catalogue_from_birth_to_audit(toy, monkeypatch): theta = np.geomspace(5.0, 60.0, 6) for pair in ((0, 0), (0, 1), (1, 1)): sio.add_xi(two_bin, pair, theta, 1e-5 / theta, 1e-6 / theta, grid="tomo") - written = sio.save(two_bin, out / "two_bin.sacc", custody=blinded) - assert np.all(np.asarray(written.mean) != np.asarray(two_bin.mean)) # The true vector of TOY is TOY_OPEN's, the same galaxies declared unblinded. open_parts = run_chain( @@ -468,17 +466,6 @@ def test_a_blinded_catalogue_from_birth_to_audit(toy, monkeypatch): ) assert not list((toy.root / "mixed").iterdir()) - plain = sio.load(open_parts["reporting"]) - for key in cu.STAMP_KEYS: - plain.metadata.pop(key, None) - with pytest.raises(ValueError, match="not copies"): - sio.save( - plain, - out / "laundered.sacc", - derived_from=[sio.load(out / "TOY_xi_reporting.sacc")], - ) - assert not (out / "laundered.sacc").exists() - # --- an interrupted birth leaves no part --------------------------------- def interrupted(*args, **kwargs): raise RuntimeError("interrupted while sealing") @@ -513,7 +500,6 @@ def second_block_fails(*args, **kwargs): assert not (toy.root / "interrupted" / "two_bin.sacc").exists() # --- the reveal: archive, flip the declaration, re-measure, audit -------- - (out / "two_bin.sacc").unlink() blinded_run = { str(p.relative_to(toy.root)): p.read_bytes() for p in out.rglob("*") @@ -542,7 +528,6 @@ def second_block_fails(*args, **kwargs): assert all(s == true.stamp for s in stamps(out).values()) report = bd.audit("toy", archive=archive, true_root=out, cat_config=toy.cat_config) - print(json.dumps(report, indent=1, default=str)) assert report["ok"], json.dumps(report, indent=1, default=str) assert set(report["parts"]) == set(born) assert ( diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 2119e442..400a71fd 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -21,21 +21,31 @@ ) -def test_assemble_resolves(toy): - """Each terminal file gathers every part and the analytic covariances. - - The ξ± block takes the CosmoCov covariance on the reporting grid, and the - harmonic block is the part on the fiducial harmonic binning with the - NaMaster covariance of that same binning. - """ - result = toy.snakemake("-n", "assemble_sacc_all") +@pytest.fixture(scope="module") +def forced(toy): + """``snakemake -F -n all`` on the toy: its output and every job it schedules.""" + result = toy.snakemake("-F", "-n", "all") assert result.returncode == 0, result.stdout - jobs = [j for j in parse_jobs(result.stdout) if j.rule == "assemble_sacc"] - grids = toy.common.xi_grids(toy.config, toy.config["fiducial"]) + return result.stdout, parse_jobs(result.stdout) + + +@pytest.fixture(scope="module") +def grids(toy): + return toy.common.xi_grids(toy.config, toy.config["fiducial"]) + + +def test_assembly_gathers_every_part_and_its_covariances(toy, forced, grids): + """[P1] Each terminal file gathers every part and the analytic covariances: + ξ± and ρ/τ on the reporting grid with CosmoCov's covariance there, the + pseudo-Cℓ part and NaMaster covariance on one harmonic binning, and + COSEBIs and pure-E/B, which read the one integration-grid part and its + CosmoCov covariance.""" + jobs = forced[1] reporting = toy.common.grid_binning(grids["reporting"]) harmonic = toy.common.pseudo_cl_tag(toy.config) - assert sorted(j.wildcards["version"] for j in jobs) == sorted(VERSIONS) - for job in jobs: + assembled = [j for j in jobs if j.rule == "assemble_sacc"] + assert sorted(j.wildcards["version"] for j in assembled) == sorted(VERSIONS) + for job in assembled: version = job.wildcards["version"] assert [Path(o).name for o in job.output] == [f"{version}.sacc"] assert {Path(f).name for f in job.input} == { @@ -48,49 +58,30 @@ def test_assemble_resolves(toy): f"rho_tau_{version}_{reporting}.sacc", }, job.input - -def test_one_integration_grid(toy): - """ξ± is measured on two grids, and both B-mode statistics share one. - - COSEBIs and pure-E/B read the same integration-grid part and the same - CosmoCov covariance on that grid; no other binning is measured. - """ - result = toy.snakemake("-n", "assemble_sacc_all") - assert result.returncode == 0, result.stdout - jobs = parse_jobs(result.stdout) - grids = toy.common.xi_grids(toy.config, toy.config["fiducial"]) - - measured = { + measured = [ (j.wildcards["version"], toy.common.grid_of(grids, j.wildcards)) for j in jobs if j.rule == "xi" - } - assert len([j for j in jobs if j.rule == "xi"]) == len(measured), result.stdout - assert measured == {(v, g) for v in VERSIONS for g in grids}, measured - assert set(grids) == {"reporting", "integration"} - - tag = toy.common.grid_binning(grids["integration"]) + ] + assert sorted(measured) == sorted((v, g) for v in VERSIONS for g in grids) + integration = toy.common.grid_binning(grids["integration"]) for version in VERSIONS: by_rule = { j.rule: set(j.input) for j in jobs if j.wildcards.get("version") == version } - part = str(toy.cosmo_val / f"{version}_xi_{tag}.sacc") - covariance = str(toy.covariances[version, "g"]) - assert by_rule["cv_cosebis"] == {part, covariance}, by_rule["cv_cosebis"] - assert {part, covariance} <= by_rule["cv_pure_eb"], by_rule["cv_pure_eb"] + part = str(toy.cosmo_val / f"{version}_xi_{integration}.sacc") + pair = {part, str(toy.covariances[version, "g"])} + assert by_rule["cv_cosebis"] == pair, by_rule["cv_cosebis"] + assert pair <= by_rule["cv_pure_eb"], by_rule["cv_pure_eb"] -def test_xi_leaves_a_measurement_only_as_a_part(toy): - """No job reads or writes a ξ± text dump; the ξ± figures draw the parts.""" - result = toy.snakemake("-n", "all") - assert result.returncode == 0, result.stdout - jobs = parse_jobs(result.stdout) +def test_xi_leaves_a_measurement_only_as_a_part(toy, forced, grids): + """[P12] No job reads or writes a ξ± text dump; the ξ± figures draw the parts.""" + jobs = forced[1] dumps = [ f for j in jobs for f in j.input + j.output if re.search(r"_xi_.*\.txt$", f) ] assert not dumps, dumps - - grids = toy.common.xi_grids(toy.config, toy.config["fiducial"]) reporting = toy.common.grid_binning(grids["reporting"]) for rule in ("cv_plot_2pcf", "cv_ratio_xi_sys_xi"): (job,) = [j for j in jobs if j.rule == rule] @@ -99,21 +90,14 @@ def test_xi_leaves_a_measurement_only_as_a_part(toy): }, job.input -def test_patch_centres_are_an_input_no_rule_draws(toy): - """Every patched ξ± job splits at its base catalogue's centres, and even a - forced run writes none: a re-draw cannot be reproduced.""" - result = toy.snakemake("-F", "-n", "assemble_sacc_all") - assert result.returncode == 0, result.stdout - jobs = parse_jobs(result.stdout) - npatch = toy.common.xi_grids(toy.config, toy.config["fiducial"])["reporting"][ - "npatch" - ] +def test_patch_centres_are_an_input_no_rule_draws(toy, forced, grids): + """[P13] Every patched ξ± job, the variant's too, splits at its base + catalogue's centres, and even a forced run writes none.""" + jobs = forced[1] + npatch = grids["reporting"]["npatch"] assert npatch > 1 centres = str(toy.cosmo_val / "patches" / f"{VERSIONS[0]}_npatch={npatch}.dat") - - xi = [j for j in jobs if j.rule == "xi"] - assert {j.wildcards["version"] for j in xi} == set(VERSIONS) - for job in xi: + for job in [j for j in jobs if j.rule == "xi"]: patched = int(job.wildcards["npatch"]) > 1 assert [f for f in job.input if "/patches/" in f] == ( [centres] if patched else [] @@ -122,40 +106,99 @@ def test_patch_centres_are_an_input_no_rule_draws(toy): assert not drawn, drawn -def test_a_tree_without_centres_stops_the_launch(toy, tmp_path): - """A launch into a tree without the centres names the command that draws - them, for the base catalogue.""" +def test_custody_is_the_checkouts_and_no_rule_touches_a_blind(toy, forced, tmp_path): + """[P9, P10, P8] A catalogue and its variant share one custody line; even a + forced run schedules nothing that reads or writes the registry; and a + config file declaring the catalogue unblinded changes nothing.""" + output, jobs = forced + line = f"[custody] {VERSIONS[0]} (+ {VERSIONS[1]}): blinded under toy" + assert [x for x in output.splitlines() if x.startswith("[custody]")] == [line] + registry = (toy.root / "cosmo_val" / "blinds").resolve() + assert not [j.rule for j in jobs if "blind" in j.rule] + touched = [ + f + for j in jobs + for f in j.input + j.output + if (toy.rundir / f).resolve().is_relative_to(registry) + ] + assert not touched, touched + + override = tmp_path / "override.yaml" + override.write_text(yaml.safe_dump({VERSIONS[0]: {"blinding": "unblinded"}})) + result = toy.snakemake("-n", "assemble_sacc_all", "--configfile", str(override)) + assert result.returncode == 0, result.stdout + assert line in result.stdout + + +def _launch_refusals(toy, tmp_path): + """Launches the DAG cannot honour: ``(args, Toy.snakemake kwargs, message)``.""" + campaign = tmp_path / "campaign.yaml" + config = dict(toy.config, cosmo_val=dict(toy.config["cosmo_val"], type="mock")) + campaign.write_text(yaml.safe_dump(config)) npatch = toy.common.xi_grids(toy.config, toy.config["fiducial"])["reporting"][ "npatch" ] - result = toy.snakemake( - "-n", "assemble_sacc_all", env={**toy.env, "COSMO_VAL": str(tmp_path)} - ) plain = toy.common._plain + return { + # [P6] a blinded catalogue no blind covers: pull, then draw or share + "no_blind": ( + [], + dict(config=[f'versions=["{UNCOVERED}"]']), + ["git pull first", "python -m sp_validation.blinding init", "share"], + ), + # [P11] unblinded under a concealed blind, and the removed campaign switch + "unrevealed": ( + [], + dict(config=[f'versions=["{STALE}"]']), + ["blinding reveal stale"], + ), + "campaign_type": ( + ["--configfile", str(campaign)], + {}, + ["custody is declared per catalogue in cosmo_val/cat_config.yaml"], + ), + # a tree without its patch centres: the command that draws them + "no_centres": ( + [], + dict(env={**toy.env, "COSMO_VAL": str(tmp_path)}), + [ + f"python -m sp_validation.cosmo_val.patch_centers {VERSIONS[0]} {npatch} " + f"--cat-config {plain(toy.root / 'cosmo_val' / 'cat_config.yaml')} " + f"--output-dir {plain(tmp_path)}" + ], + ), + } + + +@pytest.mark.parametrize( + "case", ["no_blind", "unrevealed", "campaign_type", "no_centres"] +) +def test_a_launch_the_dag_cannot_honour_stops_with_the_fix(toy, tmp_path, case): + args, launch, message = _launch_refusals(toy, tmp_path)[case] + result = toy.snakemake("-n", "assemble_sacc_all", *args, **launch) assert result.returncode != 0, result.stdout - assert ( - f"python -m sp_validation.cosmo_val.patch_centers {VERSIONS[0]} {npatch} " - f"--cat-config {plain(toy.root / 'cosmo_val' / 'cat_config.yaml')} " - f"--output-dir {plain(tmp_path)}" - ) in result.stdout, result.stdout + for fragment in message: + assert fragment in result.stdout, result.stdout + assert "rule assemble_sacc" not in result.stdout @pytest.mark.parametrize("named", [True, False], ids=["named", "unnamed"]) -def test_outputs_stay_in_the_output_roots(toy, named): - """Nothing the suite declares lands outside the configured output roots. - - A launch that names no COSMO_VAL writes into its own checkout's - cosmo_val/output. - """ - env = toy.env if named else {k: v for k, v in toy.env.items() if k != "COSMO_VAL"} - cosmo_val = toy.cosmo_val if named else toy.root / "cosmo_val" / "output" - result = toy.snakemake("-n", "all", env=env) - assert result.returncode == 0, result.stdout +def test_outputs_stay_in_the_output_roots(toy, forced, named): + """[P2] Nothing the suite declares lands outside the configured output + roots; a launch that names no COSMO_VAL writes into its own checkout's + cosmo_val/output.""" + if named: + cosmo_val, jobs = toy.cosmo_val, forced[1] + else: + env = {k: v for k, v in toy.env.items() if k != "COSMO_VAL"} + result = toy.snakemake("-n", "all", env=env) + assert result.returncode == 0, result.stdout + cosmo_val, jobs = toy.root / "cosmo_val" / "output", parse_jobs(result.stdout) roots = [ r.resolve() for r in (cosmo_val, toy.cosmo_inference, toy.rundir / "results") ] - outputs = [Path(o) for j in parse_jobs(result.stdout) for o in j.output] - assert outputs, result.stdout + outputs = [Path(o) for j in jobs for o in j.output] + assert outputs strays = [ o for o in outputs @@ -178,7 +221,7 @@ def test_every_spelling_of_an_output_root_declares_the_same_paths(toy, tmp_path, assert not [f for f in declared if f.startswith(f"{link}/")], declared -def test_output_roots_take_the_plain_spelling(toy): +def test_output_roots_take_the_plain_spelling(toy, grids): """A root given as /automnt//... is declared as //..., the one spelling every node has, on any host: a job step re-derives the launch's paths on its own node, and the node that owns a disk has neither @@ -196,7 +239,6 @@ def test_output_roots_take_the_plain_spelling(toy): tree / "val", tree / "inference", ) - grids = toy.common.xi_grids(toy.config, toy.config["fiducial"]) integration = toy.common.grid_binning(grids["integration"]) target = tree / "val" / f"{VERSIONS[0]}_xi_{integration}.sacc" result = toy.snakemake("-n", str(target), env=env) @@ -207,33 +249,31 @@ def test_output_roots_take_the_plain_spelling(toy): @pytest.mark.parametrize( - "python, snakemake_version", - [("3.13.1", None), (None, "9.0.0")], - ids=["python-minor", "snakemake"], + "image, refused", + [ + (dict(python="3.13.1"), "3.13"), + (dict(snakemake_version="9.0.0"), HOST_PYTHON.rsplit(".", 1)[0]), + ("docker://example.org/image:tag", None), + ], + ids=["python-minor", "snakemake", "registry-tag"], ) -def test_image_parity_is_checked_at_launch(toy, tmp_path, python, snakemake_version): - """An image whose Python minor or Snakemake differs stops the launch.""" - image = fake_image( - tmp_path / "image", - **({"python": python} if python else {}), - **({"snakemake_version": snakemake_version} if snakemake_version else {}), - ) +def test_image_parity_is_checked_at_launch(toy, tmp_path, image, refused): + """[P3] An image whose Python minor or Snakemake differs stops the launch + with the reinstall line; a registry tag cannot be inspected, and the + launch says so and goes on.""" + if isinstance(image, dict): + image = fake_image(tmp_path / "image", **image) result = toy.snakemake("-n", "assemble_sacc_all", container=image) + if refused is None: + assert result.returncode == 0 and "parity unchecked" in result.stdout, ( + result.stdout + ) + return assert result.returncode != 0, result.stdout - minor = ".".join((python or HOST_PYTHON).split(".")[:2]) - assert f"uv tool install --force --python {minor} snakemake==" in result.stdout + assert f"uv tool install --force --python {refused} snakemake==" in result.stdout assert "rule assemble_sacc" not in result.stdout -def test_unreadable_image_is_named_not_fatal(toy): - """A registry tag cannot be inspected; the launch says so and goes on.""" - result = toy.snakemake( - "-n", "assemble_sacc_all", container="docker://example.org/image:tag" - ) - assert result.returncode == 0, result.stdout - assert "parity unchecked" in result.stdout - - def test_launch_reads_the_image_under_home(toy, tmp_path): """The launch finds your image under ~/.cache, whatever XDG_CACHE_HOME says. @@ -373,76 +413,6 @@ def test_image_sims_checks_parity_at_launch(toy, tmp_path): assert "uv tool install --force --python 3.13 snakemake==" in result.stdout -def _custody_lines(output): - return [line for line in output.splitlines() if line.startswith("[custody]")] - - -def test_a_catalogue_without_a_blind_stops_the_launch(toy): - """A blinded catalogue with no blind fails at parse: pull, then draw or share.""" - result = toy.snakemake( - "-n", "assemble_sacc_all", config=[f'versions=["{UNCOVERED}"]'] - ) - assert result.returncode != 0, result.stdout - assert "git pull first" in result.stdout - assert "python -m sp_validation.blinding init" in result.stdout - assert "share" in result.stdout - assert "rule assemble_sacc" not in result.stdout - - -def test_no_config_line_unblinds_a_catalogue(toy, tmp_path): - """Custody is read from the checkout's cat_config, never from the merged config.""" - override = tmp_path / "override.yaml" - override.write_text(yaml.safe_dump({VERSIONS[0]: {"blinding": "unblinded"}})) - result = toy.snakemake("-n", "assemble_sacc_all", "--configfile", str(override)) - assert result.returncode == 0, result.stdout - assert f"[custody] {VERSIONS[0]} (+ {VERSIONS[1]}): blinded under toy" in ( - _custody_lines(result.stdout) - ) - - -def test_a_catalogue_and_its_variant_share_one_custody(toy): - result = toy.snakemake("-n", "assemble_sacc_all") - assert result.returncode == 0, result.stdout - assert _custody_lines(result.stdout) == [ - f"[custody] {VERSIONS[0]} (+ {VERSIONS[1]}): blinded under toy" - ] - - -def test_no_rule_draws_or_touches_a_blind(toy): - """Even a forced run schedules nothing that reads or writes the registry.""" - result = toy.snakemake("-F", "-n", "all") - assert result.returncode == 0, result.stdout - jobs = parse_jobs(result.stdout) - registry = (toy.root / "cosmo_val" / "blinds").resolve() - assert jobs - assert not [j.rule for j in jobs if "blind" in j.rule] - touched = [ - f - for j in jobs - for f in j.input + j.output - if (toy.rundir / f).resolve().is_relative_to(registry) - ] - assert not touched, touched - - -def test_the_campaign_type_switch_is_refused(toy, tmp_path): - """A config carrying cosmo_val.type stops the launch with the pointer.""" - config = dict(toy.config, cosmo_val=dict(toy.config["cosmo_val"], type="mock")) - path = tmp_path / "config.yaml" - path.write_text(yaml.safe_dump(config)) - result = toy.snakemake("-n", "assemble_sacc_all", "--configfile", str(path)) - assert result.returncode != 0, result.stdout - assert "custody is declared per catalogue in cosmo_val/cat_config.yaml" in ( - result.stdout - ) - - -def test_unblinding_a_concealed_catalogue_needs_the_reveal(toy): - result = toy.snakemake("-n", "assemble_sacc_all", config=[f'versions=["{STALE}"]']) - assert result.returncode != 0, result.stdout - assert "blinding reveal stale" in result.stdout - - def _stand_in_centres(paper, cosmo_val): """Touch the centres ``paper``'s patched ξ± grids read; a dry-run reads none.""" common = _load_module( From 9a224bb12eaca9d7ff4deddad9b92555013f51b4 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 03:54:08 +0200 Subject: [PATCH 114/160] =?UTF-8?q?rho=5Ftau=5Fstats:=206=20h=20wall=20clo?= =?UTF-8?q?ck;=20the=20jackknife=20=CF=81/=CF=84=20outruns=20the=2060=20mi?= =?UTF-8?q?n=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- workflow/rules/twopoint.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 65973c75..852c8722 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -66,6 +66,7 @@ rule rho_tau_stats: resources: mem_mb=30000, disk_mb=20000, + runtime=360, script: "../scripts/run_rho_tau.py" From ff9ece2e963488723fc796472f215a2ca1e30043 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 04:12:19 +0200 Subject: [PATCH 115/160] tests: keep the custody contracts, drop precision and bound tests What remains guards the custody contracts, each by one small property or example: blinded data is never written or leaked in plaintext (seal conceals, laundering is refused, load refuses unstamped files, no xi text dump, and the end-to-end byte scan of a blinded run), derivations inherit their inputs' one stamp, a blind is drawn once and no rule touches the registry, and the shift lands on xi and Cl_EE rows only. The end-to-end run keeps its blinded chain, reveal and audit, and custody keeps its core table, variant property and host/job agreement. Dropped: the independent CCL shift reference, KS uniformity, the B-mode leakage bounds, CAMB/CCL precision (the halofit lineage check stays, as test_blinding_recipe.py), the pure-E/B transform pins, audit tolerance tables, record-tampering and schema-evolution tests, the import-boundary AST checks (the one-door check stays), and the launch-environment DAG tests (root spellings, launch cache, profile job bound, image-sims parity). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- src/sp_validation/tests/test_architecture.py | 172 +-- src/sp_validation/tests/test_b_modes.py | 77 +- src/sp_validation/tests/test_blinding.py | 1139 +++-------------- .../tests/test_blinding_recipe.py | 24 + .../tests/test_camb_ccl_crosscheck.py | 175 --- src/sp_validation/tests/test_cosmo_val.py | 49 +- src/sp_validation/tests/test_custody.py | 149 +-- src/sp_validation/tests/test_custody_e2e.py | 186 +-- workflow/tests/test_dag.py | 199 +-- 9 files changed, 236 insertions(+), 1934 deletions(-) create mode 100644 src/sp_validation/tests/test_blinding_recipe.py delete mode 100644 src/sp_validation/tests/test_camb_ccl_crosscheck.py diff --git a/src/sp_validation/tests/test_architecture.py b/src/sp_validation/tests/test_architecture.py index c76ed9f7..c3be2605 100644 --- a/src/sp_validation/tests/test_architecture.py +++ b/src/sp_validation/tests/test_architecture.py @@ -1,170 +1,48 @@ -"""Architecture on the resolved code tree: each assertion names its contract. +"""Architecture on the resolved code tree: every SACC file passes one door. -Import boundaries are read from the ``CONTRACTS`` files (the rule lines the -scientific-software-development skill's ``check-imports`` also reads); the -rest are AST queries over every Python file outside the tests. +``sacc_io.save`` seals and stamps what it writes and ``sacc_io.load`` refuses +what it did not, so no other module may write or read a SACC file directly. """ import ast -import fnmatch -import re -import sys from pathlib import Path import pytest -from sp_validation import custody - REPO = Path(__file__).resolve().parents[3] TESTS = Path(__file__).resolve().parent -HEADER = re.compile(r"^\s*@sc(?:\s+\[[^\]]*\])?\s+([\w][\w.-]*)\s*$") -ONLY = re.compile(r"^\s*only:\s*(\S+)\s+may import\s+(.+?)\s*$") -ONLY_IMPORTER = re.compile(r"^\s*only:\s*(\S+)\s+may be imported by\s+(.+?)\s*$") - def _sources(): - """``{module name: path}`` for every Python file outside the tests.""" - found = {} - for root, package in ((REPO / "src" / "sp_validation", "sp_validation"),): - for path in root.rglob("*.py"): - if TESTS in path.parents: - continue - parts = path.relative_to(root).with_suffix("").parts - parts = parts[:-1] if parts[-1] == "__init__" else parts - found[".".join((package, *parts))] = path - for top in ("workflow", "scripts", "papers", "cosmo_inference"): + """Every Python file of the package and the workflow, outside the tests.""" + for top in ( + "src/sp_validation", + "workflow", + "scripts", + "papers", + "cosmo_inference", + ): for path in (REPO / top).rglob("*.py"): - if "tests" in path.relative_to(REPO).parts or ".snakemake" in path.parts: - continue - found[".".join(path.relative_to(REPO).with_suffix("").parts)] = path - return found - - -SOURCES = _sources() - - -def _imports(module, path): - """Absolute names of every module ``path`` imports.""" - names = [] - for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): - if isinstance(node, ast.Import): - names += [alias.name for alias in node.names] - elif isinstance(node, ast.ImportFrom): - if node.level: - package = module.split(".") - package = package if path.name == "__init__.py" else package[:-1] - base = ".".join(package[: len(package) - node.level + 1]) - target = f"{base}.{node.module}" if node.module else base - names += ( - [target] - if node.module - else [f"{target}.{alias.name}" for alias in node.names] - ) - else: - names.append(node.module) - return names - - -def _matches(name, pattern): - if pattern == "stdlib": - return name.split(".")[0] in sys.stdlib_module_names | {"__future__"} - return fnmatch.fnmatchcase(name, pattern) - - -def _rules(): - """``(contract, kind, subject, patterns)`` from every CONTRACTS file.""" - rules = [] - for contracts in sorted(REPO.rglob("CONTRACTS")): - if ".snakemake" in contracts.parts: - continue - ident = None - for line in contracts.read_text().splitlines(): - if header := HEADER.match(line): - ident = header[1] - elif rule := ONLY_IMPORTER.match(line): - rules.append((ident, "importers", rule[1], rule[2].split(", "))) - elif rule := ONLY.match(line): - rules.append((ident, "imports", rule[1], rule[2].split(", "))) - return rules - - -RULES = _rules() - - -def test_the_contracts_carry_the_boundaries(): - assert {r[0] for r in RULES} >= { - "custody-is-stdlib", - "container-is-stdlib", - "blinding-owns-smokescreen", - "blinding-owns-the-seed-cipher", - "host-importable", - } - - -@pytest.mark.parametrize( - "contract, kind, subject, patterns", RULES, ids=[r[0] for r in RULES] -) -def test_import_boundaries(contract, kind, subject, patterns): - if kind == "imports": - assert subject in SOURCES, f"[{contract}] no module {subject}" - stray = [ - name - for name in _imports(subject, SOURCES[subject]) - if not any(_matches(name, p) for p in patterns) - ] - assert not stray, f"[{contract}] {subject} imports {stray}" - else: - stray = sorted( - module - for module, path in SOURCES.items() - if not any(_matches(module, p) for p in patterns) - and any(fnmatch.fnmatchcase(n, subject) for n in _imports(module, path)) - ) - assert not stray, f"[{contract}] {subject} is imported by {stray}" - - -def _nodes(kind): - for module, path in SOURCES.items(): - for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): - if isinstance(node, kind): - yield module, node - - -def test_stamp_keys_are_spelt_only_in_custody(): - """[stamped-or-refused] Stamps are read and minted through custody.stamp and read_stamp.""" - spelt = sorted( - (module, node.value) - for module, node in _nodes(ast.Constant) - if isinstance(node.value, str) - and node.value in custody.STAMP_KEYS - and module != "sp_validation.custody" - ) - assert not spelt, spelt - - -def test_custody_is_constructed_only_where_it_is_declared(): - """[custody-is-declared] Only custody_of builds a Custody.""" - built = sorted( - module - for module, node in _nodes(ast.Call) - if ( - (isinstance(node.func, ast.Name) and node.func.id == "Custody") - or (isinstance(node.func, ast.Attribute) and node.func.attr == "Custody") - ) - and module != "sp_validation.custody" - ) - assert not built, built + parts = path.relative_to(REPO).parts + if ( + TESTS not in path.parents + and "tests" not in parts + and ".snakemake" not in parts + ): + yield path @pytest.mark.parametrize("method", ["save_fits", "load_fits"]) def test_sacc_files_pass_one_door(method): """[one-door] Only sacc_io writes or reads a SACC file.""" + door = REPO / "src" / "sp_validation" / "sacc_io.py" callers = sorted( - module - for module, node in _nodes(ast.Call) - if isinstance(node.func, ast.Attribute) + str(path.relative_to(REPO)) + for path in _sources() + if path != door + for node in ast.walk(ast.parse(path.read_text(), filename=str(path))) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) and node.func.attr == method - and module != "sp_validation.sacc_io" ) assert not callers, callers diff --git a/src/sp_validation/tests/test_b_modes.py b/src/sp_validation/tests/test_b_modes.py index 20a9aa67..67c64376 100644 --- a/src/sp_validation/tests/test_b_modes.py +++ b/src/sp_validation/tests/test_b_modes.py @@ -1,9 +1,8 @@ """VALUE-DRIFT CHARACTERIZATION TESTS FOR THE B-MODE ESTIMATORS. This module pins the numeric behavior of the pure E/B-mode helpers in -``sp_validation.b_modes`` against fixed, deterministic inputs (seeded RNG, -hand-built arrays and one committed ξ± fixture — no cluster data, no catalogue -files). +``sp_validation.b_modes`` against fixed, deterministic, in-memory inputs +(seeded RNG and hand-built arrays — no cluster data, no catalogue files). Every pinned literal was produced by an actual run of the estimator inside the container; a future refactor that changes the numbers must fail. @@ -294,78 +293,6 @@ def test_calculate_eb_statistics_has_teeth(): assert loud_pte < 0.05 # louder B-modes are clearly rejected -# --------------------------------------------------------------------------- -# 5. pure_eb_from_xi on committed ξ± (the transform pin) -# --------------------------------------------------------------------------- - -# pure_eb_from_xi(**fixture); regenerated only when the transform is meant to move. -_PURE_EB_PINS = { - "xip_E": [ - -2.9831529669542025e-06, - -1.5008524620265777e-05, - 3.221623968725757e-07, - 1.1797672310858565e-05, - 5.715510692557323e-06, - 8.825804523824443e-07, - ], - "xim_E": [ - -4.737558091773235e-05, - -0.00010853189443993388, - -9.094825175032069e-05, - -5.826599101284694e-05, - -4.646405415748759e-05, - -1.9978028925333273e-05, - ], - "xip_B": [ - 1.7069121242262332e-05, - 3.059889782373755e-05, - -4.8805399253844115e-06, - -6.999262696335271e-06, - -1.2672006989728095e-05, - -1.214149138979614e-06, - ], - "xim_B": [ - -0.00011478091634539627, - -5.445112002141066e-05, - -3.100806652947907e-05, - -1.0940424256759085e-05, - -5.755185146643215e-06, - -1.628217762504557e-06, - ], - "xip_amb": [ - 0.00014017621792612224, - 0.0001378482153667787, - 0.0001339573019551001, - 0.00012745126271361765, - 0.00011662385105911986, - 9.851844704443032e-05, - ], - "xim_amb": [ - -4.389203999135455e-05, - 5.279277664928643e-05, - 5.800339397836051e-05, - 4.4242350610114584e-05, - 2.9902912946755567e-05, - 1.912262132836568e-05, - ], -} - - -def test_pure_eb_from_xi_reproduces_pins_on_committed_xi(pure_eb_xi): - """The pure-E/B transform of the committed ξ± reproduces its pins. - - With ξ± frozen, these pins move only when the transform does. rtol=1e-6 is - far above the 1e-12 reduction-order noise across thread counts. - """ - modes = b_modes.pure_eb_from_xi(**pure_eb_xi) - for key in b_modes._EB_KEYS: - npt.assert_allclose(modes[key], _PURE_EB_PINS[key], rtol=1e-6, err_msg=key) - - # Teeth: widening the integration interval by 1% leaves the pins. - moved = b_modes.pure_eb_from_xi(**{**pure_eb_xi, "tmax": 1.01 * pure_eb_xi["tmax"]}) - assert not np.allclose(moved["xip_E"], _PURE_EB_PINS["xip_E"], rtol=1e-6, atol=0) - - # --------------------------------------------------------------------------- # 6. Grid edges and the COSEBIs covariance seam # --------------------------------------------------------------------------- diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 6e57135a..3e98ead3 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -1,40 +1,28 @@ -"""The blind and the file door: invariants I1–I11. +"""The blind and the file door. A blind is drawn once (``blinding init``) into a registry beside a catalogue config; ``sacc_io.save`` is the only writer, and conceals a blinded catalogue's ξ± and Cℓ_EE rows in memory before the file exists. Blinds here -use the fast Eisenstein–Hu theory, so the whole module runs in the fast suite; -``test_camb_ccl_crosscheck.py`` covers the production CAMB recipe. +use the fast Eisenstein–Hu theory. """ -import dataclasses import json -import os -import stat -import string from pathlib import Path import numpy as np import pytest import yaml -from hypothesis import assume, example, given, settings +from hypothesis import example, given, settings from hypothesis import strategies as st from sp_validation import blinding as bd from sp_validation import custody as cu from sp_validation import sacc_io as sio -from sp_validation.blinding_theory import TheoryConfig -FAST = {"theory": {"transfer_function": "eisenstein_hu"}} DATA = Path(__file__).parent / "data" -# The hidden (S8, Ωm) of the blind committed under DATA/blinds/committed. -COMMITTED_POINT = (0.8374952413635888, 0.32730573347638175) VERSIONS = ("TOY", "OTHER", "TOY_OPEN", "OTHER_OPEN", "TOY_MOCK") -# --------------------------------------------------------------------------- # -# Registries with real blinds, and custody under them -# --------------------------------------------------------------------------- # def _registry(root, *blinds): """A catalogue config declaring one catalogue per custody state, and ``blinds`` ((name, base) pairs) drawn for it, each from a fixed seed.""" @@ -44,10 +32,11 @@ def _registry(root, *blinds): "TOY_OPEN": {"blinding": "unblinded"}, "OTHER_OPEN": {"blinding": "unblinded"}, "TOY_MOCK": {"blinding": "mock"}, - "paths": {"output": str(root / "output")}, } (root / "cat_config.yaml").write_text(yaml.safe_dump(entries)) - (root / "fast.json").write_text(json.dumps(FAST)) + (root / "fast.json").write_text( + json.dumps({"theory": {"transfer_function": "eisenstein_hu"}}) + ) with pytest.MonkeyPatch.context() as m: for blind, base in blinds: m.setattr(bd.secrets, "token_hex", lambda n, b=blind: f"{b}-seed") @@ -67,12 +56,6 @@ def _custody(root, version): return bd.declared_custody(root / "cat_config.yaml", version) -def _declare(root, **entries): - """Set catalogue entries of ``root``'s catalogue config.""" - path = root / "cat_config.yaml" - path.write_text(yaml.safe_dump({**yaml.safe_load(path.read_text()), **entries})) - - @pytest.fixture(scope="module") def blinds(tmp_path_factory): """The five custodies: TOY under blind `toy`, OTHER under `other`.""" @@ -82,545 +65,172 @@ def blinds(tmp_path_factory): return {v: _custody(root, v) for v in VERSIONS} -@pytest.fixture -def fresh(tmp_path): - """A registry of its own, for tests that edit or reveal a blind.""" - return _registry(tmp_path, ("toy", "TOY")) - - -# --------------------------------------------------------------------------- # -# Synthetic parts -# --------------------------------------------------------------------------- # -def _gauss_nz(z0, sigma, n=200): - z = np.linspace(0.0, 3.0, n) - nz = np.exp(-0.5 * ((z - z0) / sigma) ** 2) - return z, nz / np.trapezoid(nz, z) - - -NZ2 = {0: _gauss_nz(0.5, 0.15), 1: _gauss_nz(0.9, 0.2)} -PAIRS = ((0, 0), (0, 1), (1, 1)) -# The ξ± tags a part may carry; blocks never depend on them (None: no tag). -XI_TAGS = ("reporting", "integration", "cosebis", "mystery", None) -# Data types no blind has a rule for, galaxy–galaxy lensing and CMB κ × shear, -# with the tag SACC requires of each. -UNRULED = { - "galaxy_shearDensity_xi_t": "theta", - "cmbGalaxy_convergenceShear_cl_e": "ell", -} - +def _nz(z0): + z = np.linspace(0.0, 3.0, 200) + return z, np.exp(-0.5 * ((z - z0) / 0.2) ** 2) -def _xi_template(theta, k=0): - return 1e-4 * (1 + 0.1 * k) * (theta / 10.0) ** -0.6, 0.5e-4 * (1 + 0.1 * k) * ( - theta / 10.0 - ) ** -0.9 - -def _add_xi_rows(s, pair, theta, tag, k=0): - xip, xim = _xi_template(theta, k) - if tag is not None: - sio.add_xi(s, pair, theta, xip, xim, grid=tag) - return - for dtype, values in ((sio.XI_PLUS, xip), (sio.XI_MINUS, xim)): - for th, v in zip(theta, values): - s.add_data_point(dtype, sio._pair(pair), float(v), theta=float(th)) - - -def _add_cl_rows(s, pair, k=0): - ell_eff = np.array([30.0, 80.0, 150.0, 280.0, 450.0]) +def part(*, xi_tags=("reporting",), cl=True, rho=False, derived=None, unruled=False): + """A two-bin part: ξ± under each of ``xi_tags`` (None: no tag) and + pseudo-Cℓ (EE, BB, EB) on pairs (0,0), (0,1), (1,1), and optionally ρ/τ, + a derived statistic's rows and rows of a type with no blinding rule.""" + s = sio.new_sacc({0: _nz(0.5), 1: _nz(0.9)}) + theta = np.geomspace(2.0, 200.0, 6) + ell = np.array([30.0, 80.0, 150.0, 280.0, 450.0]) w_ell = np.arange(2, 501).astype(float) - w_mat = np.exp(-0.5 * ((w_ell[:, None] - ell_eff[None, :]) / 40.0) ** 2) - ee = 1e-8 * (1 + 0.1 * k) * (ell_eff / 100.0) ** -1.2 - sio.add_pseudo_cl( - s, - pair, - ell_eff, - ee, - 0.01 * ee, - 0.02 * ee, - window_ells=w_ell, - window_weights=w_mat / w_mat.sum(axis=0), - ) - - -def _add_rho_rows(s): - theta = np.geomspace(5.0, 250.0, 6) - sio.add_rho(s, 0, theta, np.arange(6) * 1e-7, np.arange(6) * 2e-7) - sio.add_tau(s, (0,), 0, theta, np.arange(6) * 3e-7, np.arange(6) * 4e-7) - - -def two_bin_sacc( - *, - pairs=PAIRS, - xi_tags=XI_TAGS, - cl=True, - rho=False, - derived=(), - unruled=None, - covariance=True, -): - """A catalogue's part: ξ± under each of ``xi_tags`` and pseudo-Cℓ (EE, BB, - EB) over ``pairs`` of two bins, and optionally ρ/τ, COSEBIs and pure-E/B - rows and rows of a data type without a blinding rule.""" - s = sio.new_sacc(NZ2, metadata={"catalogue_version": "TOY"}) - for pair in pairs: - k = PAIRS.index(pair) - for n, tag in enumerate(xi_tags): - _add_xi_rows(s, pair, np.geomspace(2.0, 200.0, 6) * (1 + 0.013 * n), tag, k) + window = np.exp(-0.5 * ((w_ell[:, None] - ell[None, :]) / 40.0) ** 2) + for pair in ((0, 0), (0, 1), (1, 1)): + for tag in xi_tags: + xip, xim = 1e-4 * (theta / 10) ** -0.6, 0.5e-4 * (theta / 10) ** -0.9 + if tag is not None: + sio.add_xi(s, pair, theta, xip, xim, grid=tag) + else: + for dtype, values in ((sio.XI_PLUS, xip), (sio.XI_MINUS, xim)): + for th, v in zip(theta, values): + s.add_data_point(dtype, sio._pair(pair), v, theta=th) if cl: - _add_cl_rows(s, pair, k) + ee = 1e-8 * (ell / 100.0) ** -1.2 + sio.add_pseudo_cl( + s, + pair, + ell, + ee, + 0.01 * ee, + 0.02 * ee, + window_ells=w_ell, + window_weights=window / window.sum(axis=0), + ) if rho: - _add_rho_rows(s) - if "cosebis" in derived: + sio.add_rho(s, 0, theta, np.arange(1, 7) * 1e-7, np.arange(1, 7) * 2e-7) + if derived == "cosebis": sio.add_cosebis(s, (0, 0), np.arange(1, 6) * 1e-10, (12.0, 83.0), Bn=np.ones(5)) - if "pure_eb" in derived: - theta = np.geomspace(2.0, 200.0, 4) - sio.add_pure_eb(s, (0, 0), theta, **{k: np.ones(4) for k in sio.PURE_KEYS}) - if unruled is not None: + elif derived == "pure_eb": + sio.add_pure_eb(s, (0, 0), theta[:4], **{k: np.ones(4) for k in sio.PURE_KEYS}) + if unruled: for x in (5.0, 20.0, 80.0): s.add_data_point( - unruled, ("source_0", "source_0"), 1e-5, **{UNRULED[unruled]: x} + "galaxy_shearDensity_xi_t", ("source_0", "source_0"), 1e-5, theta=x ) - if covariance: - rng = np.random.default_rng(3) - s.add_covariance( - np.abs(np.asarray(s.mean)) ** 2 * rng.uniform(1, 2, len(s.mean)) - ) + s.add_covariance(np.abs(np.asarray(s.mean)) ** 2 + 1e-20) return s -def derived_sacc(kind): - """A COSEBIs or pure-E/B part alone, as its rule writes it.""" - return two_bin_sacc(pairs=((0, 0),), xi_tags=(), cl=False, derived=(kind,)) - - -def rho_sacc(): - return two_bin_sacc(pairs=(), rho=True) - - -def _rows(s, *types): - return np.array([i for i, dp in enumerate(s.data) if dp.data_type in types], int) - - def _values(s): return np.asarray(s.mean) # --------------------------------------------------------------------------- # -# An independent CCL reference for the shift -# --------------------------------------------------------------------------- # -def _reference_cosmologies(seed): - """The hidden and fiducial CCL cosmologies, the draw made by the fork directly.""" - import pyccl as ccl - from smokescreen.param_shifts import draw_param_shifts - - shift = draw_param_shifts({"S8": 0.075, "Omega_m": 0.1}, seed) - - def cosmology(s8, om): - h, ob, mnu = 0.70, 0.0469, 0.06 - return ccl.Cosmology( - Omega_c=om - ob - mnu / (93.14 * h**2), - Omega_b=ob, - h=h, - n_s=0.96, - sigma8=s8 / np.sqrt(om / 0.3), - m_nu=mnu, - mass_split="normal", - w0=-1.0, - wa=0.0, - Neff=3.046, - T_CMB=2.7255, - transfer_function="eisenstein_hu", - matter_power_spectrum="halofit", - ) - - return cosmology(0.80 + shift["S8"], 0.30 + shift["Omega_m"]), cosmology(0.80, 0.30) - - -def reference_shift(s, seed): - """t(hidden) − t(fiducial) on every ξ± and Cℓ_EE row of ``s``, from scratch.""" - import pyccl as ccl - - cosmologies = _reference_cosmologies(seed) - ell = np.unique(np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)])) - out = np.full(len(s.mean), np.nan) - for tracers in {dp.tracers for dp in s.data if dp.data_type in sio.SHIFTABLE}: - a, b = (s.tracers[t] for t in tracers) - - def cl(c, ells): - lens = [ccl.WeakLensingTracer(c, dndz=(t.z, t.nz)) for t in (a, b)] - return ccl.angular_cl(c, *lens, ells) - - for dtype, kind in ((sio.XI_PLUS, "GG+"), (sio.XI_MINUS, "GG-")): - rows = [ - i - for i, dp in enumerate(s.data) - if (dp.data_type, dp.tracers) == (dtype, tracers) - ] - theta = np.array([s.data[i].tags["theta"] for i in rows]) / 60.0 - hidden, fiducial = ( - ccl.correlation(c, ell=ell, C_ell=cl(c, ell), theta=theta, type=kind) - for c in cosmologies - ) - out[rows] = hidden - fiducial - for i, dp in enumerate(s.data): - if (dp.data_type, dp.tracers) == (sio.CL_EE, tracers): - window = dp.tags["window"] - ells = np.asarray(window.values, float) - w = np.asarray(window.weight, float)[:, dp.tags["window_ind"]] - out[i] = w @ (cl(cosmologies[0], ells) - cl(cosmologies[1], ells)) - return out - - -# --------------------------------------------------------------------------- # -# I1, I2, I5: seal is the seal table; a blinded seal shifts exactly the -# shiftable rows, by the theory difference, and nothing else +# Born sealed: a blind shifts exactly the ξ± and Cℓ_EE rows, and nothing else # --------------------------------------------------------------------------- # PARTS = st.fixed_dictionaries( { - "pairs": st.lists(st.sampled_from(PAIRS), min_size=1, max_size=3, unique=True), - "xi_tags": st.lists(st.sampled_from(XI_TAGS), max_size=3, unique=True), + "xi_tags": st.lists( + st.sampled_from(["reporting", "integration", "cosebis", None]), + max_size=2, + unique=True, + ), "cl": st.booleans(), "rho": st.booleans(), - "derived": st.sampled_from([(), ("cosebis",), ("pure_eb",)]), - "unruled": st.sampled_from([None, *UNRULED]), + "derived": st.sampled_from([None, "cosebis", "pure_eb"]), + "unruled": st.booleans(), } -).filter(lambda p: p["xi_tags"] or p["cl"] or p["rho"] or p["derived"] or p["unruled"]) -EVERY_SHIFTABLE_ROW = dict( - pairs=PAIRS, xi_tags=XI_TAGS, cl=True, rho=True, derived=(), unruled=None -) +).filter(lambda p: p["xi_tags"] or p["cl"] or p["rho"]) -@settings(max_examples=6, deadline=None) +@settings(max_examples=10, deadline=None) @given(content=PARTS, version=st.sampled_from(["TOY", "TOY_OPEN", "TOY_MOCK"])) -@example(content=EVERY_SHIFTABLE_ROW, version="TOY") -@example(content={**EVERY_SHIFTABLE_ROW, "xi_tags": [], "cl": False}, version="TOY") -def test_seal_is_the_seal_table(blinds, content, version): - """Under a blind, a birth with derived or unruled rows is refused; any - other has ``seal(s) − s`` equal to an independent CCL reference at (fid, - hidden) on every ξ± and Cℓ_EE row, within 1e-8 of each block's largest - shift, and every other value, tag, tracer and the covariance bitwise kept. - Unblinded and mock births keep every value. The blind opens only to shift - rows, and the input is never stamped in place. - - Blocks come from the data types and tracers: ξ± under every tag and none, - over the two bins' three pairs, whatever subset a part holds. - """ - s, custody = two_bin_sacc(**content), blinds[version] +@example( + content=dict( + xi_tags=["mystery", None], cl=True, rho=True, derived=None, unruled=False + ), + version="TOY", +) +def test_seal_shifts_only_the_signal_it_has_a_rule_for(blinds, content, version): + """Under a blind every ξ± and Cℓ_EE row moves, whatever its grid tag, and + every other value and the covariance stay bitwise; a birth carrying + derived rows or a type with no blinding rule is refused. Unblinded and + mock births keep their values. Each is stamped with its custody.""" + s, custody = part(**content), blinds[version] blinded = custody.status == "blinded" - opened, open_blind = [], bd.open_blind - with pytest.MonkeyPatch.context() as m: - m.setattr(bd, "open_blind", lambda c: opened.append(c) or open_blind(c)) - if blinded and (content["derived"] or content["unruled"]): - with pytest.raises(ValueError, match="derived_from|no blinding rule"): - sio.seal(s, custody) - return - sealed = sio.seal(s, custody) - - assert "blinding" not in s.metadata + if blinded and (content["derived"] or content["unruled"]): + with pytest.raises(ValueError, match="derived_from|no blinding rule"): + sio.seal(s, custody) + return + sealed = sio.seal(s, custody) assert cu.read_stamp(sealed.metadata).stamp == custody.stamp - moved = _rows(s, *sio.SHIFTABLE) if blinded else np.array([], int) - assert bool(opened) == bool(len(moved)) - kept = np.setdiff1d(np.arange(len(s.mean)), moved) - assert np.array_equal(_values(sealed)[kept], _values(s)[kept]) + shiftable = np.array([dp.data_type in sio.SHIFTABLE for dp in s.data]) + moved = _values(sealed) != _values(s) + assert np.array_equal(moved, shiftable & blinded) assert np.array_equal(sealed.covariance.dense, s.covariance.dense) - for a, b in zip(s.data, sealed.data): - assert (a.data_type, a.tracers) == (b.data_type, b.tracers) - assert {k: v for k, v in a.tags.items() if k != "window"} == { - k: v for k, v in b.tags.items() if k != "window" - } - for name, tracer in s.tracers.items(): - if hasattr(tracer, "nz"): - assert np.array_equal(sealed.tracers[name].nz, tracer.nz) - if not len(moved): - return - - shift = _values(sealed) - _values(s) - expected = reference_shift(s, open_blind(custody).seed) - for pair in content["pairs"]: - for types in ((sio.XI_PLUS, sio.XI_MINUS), (sio.CL_EE,)): - block = [ - i - for i in moved - if s.data[i].tracers == sio._pair(pair) and s.data[i].data_type in types - ] - if block: - scale = np.max(np.abs(expected[block])) - gap = np.max(np.abs(shift[block] - expected[block])) - assert scale > 0 and gap <= 1e-8 * scale, (pair, types, gap / scale) - - -# --------------------------------------------------------------------------- # -# I3: the hidden point is uniform in the physical (S8, Ωm) box -# --------------------------------------------------------------------------- # -def test_hidden_point_is_uniform_in_the_s8_om_box(): - from scipy import stats - - config = bd.BlindingConfig() - fid = config.theory - hidden = [bd.hidden_theory(f"seed-{i}", config) for i in range(5000)] - s8 = np.array([h.S8 for h in hidden]) - om = np.array([h.Omega_m for h in hidden]) - - assert np.all(np.abs(s8 - fid.S8) <= 0.075) - assert np.all(np.abs(om - fid.Omega_m) <= 0.1) - assert stats.kstest((s8 - fid.S8 + 0.075) / 0.15, "uniform").pvalue > 1e-3 - assert stats.kstest((om - fid.Omega_m + 0.1) / 0.2, "uniform").pvalue > 1e-3 - for h in hidden[:50]: - p = h.ccl_params() - assert p["sigma8"] == pytest.approx(h.S8 / np.sqrt(h.Omega_m / 0.3), rel=1e-12) - omega_nu = h.m_nu / (93.14 * h.h**2) - assert p["Omega_c"] == pytest.approx( - h.Omega_m - h.Omega_b - omega_nu, rel=1e-12 - ) - assert bd.hidden_theory("seed-1", config) == hidden[1] - - -# --------------------------------------------------------------------------- # -# I4: the shift leaks no B-modes -# --------------------------------------------------------------------------- # -def _shape_noise_variance(left, right): - """Var ξ± per bin from shape noise at UNIONS n_eff, σ_e and area.""" - n_eff, sigma_e, area = 4.96, 0.378, 2894.0 * 3600.0 # arcmin⁻², —, arcmin² - pairs = np.pi * area * n_eff**2 * (right**2 - left**2) / 2.0 - return 2.0 * sigma_e**4 / pairs - - -@pytest.mark.parametrize("ds8", [0.075, -0.075]) -@pytest.mark.parametrize("dom", [0.1, -0.1]) -def test_shift_leaks_no_b_modes(ds8, dom): - """B-modes a shift at the envelope's edge induces stay within the audit's bound. - - At each corner of the (S8, Ωm) envelope, on the production grids (the - 0.08–300′ 1000-bin integration grid, the 1–250′ 20-bin reporting grid), - single-bin n(z), σ from shape noise at UNIONS depth: the 20 COSEBIs B-modes - on [12, 83]′ and pure-mode ξ_B move by at most ``blinding.B_SIGMA``. - """ - from sp_validation import b_modes - - config = bd.BlindingConfig(theory=TheoryConfig(transfer_function="eisenstein_hu")) - fid = config.theory - hidden = dataclasses.replace( - fid, - S8=fid.S8 + np.sign(ds8) * config.envelope["S8"], - Omega_m=fid.Omega_m + np.sign(dom) * config.envelope["Omega_m"], - ) - grids = { - "integration": b_modes.log_bin_edges(0.08, 300.0, 1000), - "reporting": b_modes.log_bin_edges(1.0, 250.0, 20), - } - s = sio.new_sacc({0: NZ2[0]}, metadata={"catalogue_version": "TOY"}) - theta = {} - for grid, (left, right) in grids.items(): - theta[grid] = np.sqrt(left * right) - sio.add_xi(s, (0, 0), theta[grid], *_xi_template(theta[grid]), grid=grid) - shift = np.zeros(len(s.mean)) - for block, factor in bd.factors(s, fid, hidden): - shift[block.rows] = factor - - def delta(grid): - return tuple( - shift[s.indices(t, grid=grid)] for t in (sio.XI_PLUS, sio.XI_MINUS) - ) - - left, right = grids["integration"] - var = _shape_noise_variance(left, right) - (result,) = b_modes.cosebis_scan_from_xi( - theta["integration"], - *delta("integration"), - np.diag(np.concatenate([var, var])), - left, - right, - nmodes=20, - scale_cuts=[(12.0, 83.0)], - ).values() - sigma_b = np.sqrt(np.diag(result["cov"])[20:]) - assert np.max(np.abs(result["En"])) > 1e2 * np.max(np.abs(result["Bn"])) - assert np.max(np.abs(result["Bn"]) / sigma_b) <= bd.B_SIGMA[sio.COSEBI_BB] - - rep_left, rep_right = grids["reporting"] - modes = b_modes.pure_eb_from_xi( - theta["reporting"], - *delta("reporting"), - theta["integration"], - *delta("integration"), - rep_left[0], - rep_right[-1], - ) - sigma = np.sqrt(_shape_noise_variance(rep_left, rep_right)) - for key in ("xip_B", "xim_B"): - finite = np.isfinite(modes[key]) - assert finite.sum() > 10 - bound = bd.B_SIGMA[sio.PURE_TYPES[key]] - assert np.max(np.abs(modes[key][finite]) / sigma[finite]) <= bound, key -# --------------------------------------------------------------------------- # -# I6: the gates -# --------------------------------------------------------------------------- # -def test_nothing_is_written_unstamped_or_stamped_twice(blinds, tmp_path): +def test_nothing_is_written_unstamped(blinds, tmp_path): + """``save`` needs a custody or its inputs; ``load`` refuses a file born + outside the door.""" with pytest.raises(ValueError, match="custody"): - sio.save(two_bin_sacc(), tmp_path / "x.sacc") + sio.save(part(), tmp_path / "x.sacc") assert not (tmp_path / "x.sacc").exists() - sealed = sio.seal(two_bin_sacc(cl=False), blinds["TOY_OPEN"]) - with pytest.raises(ValueError, match="already stamped"): - sio.seal(sealed, blinds["TOY"]) - path = tmp_path / "part.sacc" - sio.save(two_bin_sacc(cl=False), path, custody=blinds["TOY_OPEN"]) - with pytest.raises(ValueError, match="already stamped"): - sio.save(sio.load(path), path, custody=blinds["TOY_OPEN"]) - - -@settings(max_examples=20, deadline=None) -@given( - version=st.sampled_from(["TOY", "TOY_OPEN", "TOY_MOCK"]), - spoil=st.sampled_from([None, "drop", "add", "status"]), - data=st.data(), -) -def test_load_opens_exactly_the_stamps_the_door_mints( - blinds, tmp_path_factory, version, spoil, data -): - """A file stamped as ``save`` stamps it loads with that stamp; one missing - a stamp key, carrying a blind's keys under another status, or naming an - unknown status is refused.""" - stamp = dict(blinds[version].stamp) - assume(not (spoil == "add" and version == "TOY")) - if spoil == "drop": - del stamp[data.draw(st.sampled_from(sorted(stamp)))] - elif spoil == "add": - key = data.draw(st.sampled_from(cu.STAMP_KEYS[2:])) - stamp[key] = blinds["TOY"].stamp[key] - elif spoil == "status": - status = st.text(string.ascii_letters, min_size=1, max_size=8) - stamp["blinding"] = data.draw(status.filter(lambda t: t not in cu.STATUSES)) - path = tmp_path_factory.mktemp("raw") / "part.sacc" - _write(path, two_bin_sacc(cl=False), stamp) - if spoil is None: - assert cu.read_stamp(sio.load(path).metadata).stamp == stamp - else: - with pytest.raises(ValueError, match="stamp"): - sio.load(path) + part().save_fits(str(tmp_path / "raw.sacc")) + with pytest.raises(ValueError, match="stamp"): + sio.load(tmp_path / "raw.sacc") # --------------------------------------------------------------------------- # -# I7: a derivation carries its inputs' one stamp; an assembly its declaration +# Derived: a derivation carries its inputs' one stamp, never plaintext # --------------------------------------------------------------------------- # @pytest.fixture(scope="module") def parts(blinds): """A ξ± part born under each custody.""" - return { - v: sio.seal(two_bin_sacc(xi_tags=("reporting",), cl=False), blinds[v]) - for v in VERSIONS - } - - -DERIVATIONS = ("copy", "cosebis", "pure_eb", "plaintext", "unruled") + return {v: sio.seal(part(cl=False), blinds[v]) for v in VERSIONS} @settings(max_examples=25, deadline=None) @given( inputs=st.lists(st.sampled_from(VERSIONS), min_size=1, max_size=3), declared=st.sampled_from([None, *VERSIONS]), - content=st.sampled_from(DERIVATIONS), + content=st.sampled_from(["copy", "cosebis", "plaintext"]), ) @example(inputs=["TOY"], declared=None, content="plaintext") # laundering -@example(inputs=["TOY"], declared="TOY_OPEN", content="copy") # stale after a reveal +@example(inputs=["TOY", "TOY"], declared="TOY", content="copy") # an assembly @example(inputs=["OTHER"], declared="TOY", content="copy") # another blind -@example(inputs=["TOY_MOCK"], declared="TOY_OPEN", content="copy") # mock into data @example(inputs=["TOY_OPEN"], declared="TOY", content="copy") # unblinded into blinded -@example(inputs=["TOY_OPEN", "TOY_MOCK"], declared=None, content="cosebis") @example(inputs=["TOY", "OTHER"], declared=None, content="cosebis") -@example(inputs=["TOY_OPEN", "OTHER_OPEN"], declared=None, content="cosebis") -@example(inputs=["TOY"], declared=None, content="unruled") -@example(inputs=["TOY", "TOY"], declared="TOY", content="copy") # an assembly -@example(inputs=["TOY_OPEN"], declared="TOY_OPEN", content="plaintext") -@example(inputs=["TOY_MOCK"], declared=None, content="pure_eb") def test_a_derivation_carries_its_inputs_one_stamp( blinds, parts, tmp_path_factory, inputs, declared, content ): - """``save(s, derived_from=inputs, custody=declared)`` writes ``s``'s values - under the inputs' stamp exactly when the inputs share one stamp, it is the - declared custody's (if any), and, under a blind, every ξ± row of ``s`` is a - copy of an input row and every data type has a blinding rule. Otherwise - nothing is written.""" - first = parts[inputs[0]] + """``save(s, derived_from=inputs, custody=declared)`` writes ``s`` under + the inputs' stamp exactly when they share one, it is the declared + custody's (if any), and, under a blind, ``s``'s ξ± rows are copies of + theirs. Otherwise nothing is written.""" s = { - "copy": lambda: first.copy(), - "cosebis": lambda: derived_sacc("cosebis"), - "pure_eb": lambda: derived_sacc("pure_eb"), - "plaintext": lambda: two_bin_sacc(xi_tags=("reporting",), cl=False), - "unruled": lambda: two_bin_sacc( - xi_tags=(), cl=False, unruled=next(iter(UNRULED)) - ), + "copy": lambda: parts[inputs[0]].copy(), + "cosebis": lambda: part(xi_tags=(), cl=False, derived="cosebis"), + "plaintext": lambda: part(cl=False), }[content]() stamp = blinds[inputs[0]].stamp - blinded = stamp["blinding"] == "blinded" allowed = ( all(blinds[v].stamp == stamp for v in inputs) and (declared is None or blinds[declared].stamp == stamp) - and not (blinded and content in ("plaintext", "unruled")) + and not (stamp["blinding"] == "blinded" and content == "plaintext") ) path = tmp_path_factory.mktemp("derived") / "d.sacc" - custody = blinds[declared] if declared else None - derive = [parts[v] for v in inputs] + kwargs = dict( + derived_from=[parts[v] for v in inputs], + custody=blinds[declared] if declared else None, + ) if not allowed: with pytest.raises(ValueError): - sio.save(s, path, derived_from=derive, custody=custody) + sio.save(s, path, **kwargs) assert not path.exists() return - sio.save(s, path, derived_from=derive, custody=custody) + sio.save(s, path, **kwargs) loaded = sio.load(path) assert cu.read_stamp(loaded.metadata).stamp == stamp assert np.array_equal(_values(loaded), _values(s)) # --------------------------------------------------------------------------- # -# I8: the digest binds the whole config; the commitment is the fork's -# --------------------------------------------------------------------------- # -def _variants(config): - """Configs differing from ``config`` in exactly one field, at any depth.""" - for key, value in config.envelope.items(): - yield dataclasses.replace( - config, envelope={**config.envelope, key: value + 0.01} - ) - rest = {k: v for k, v in config.envelope.items() if k != key} - yield dataclasses.replace(config, envelope={**rest, key + "_": value}) - yield dataclasses.replace(config, envelope={**config.envelope, "h": 0.01}) - for field in dataclasses.fields(TheoryConfig): - value = getattr(config.theory, field.name) - changed = value + "x" if isinstance(value, str) else value + 0.01 - yield dataclasses.replace( - config, theory=dataclasses.replace(config.theory, **{field.name: changed}) - ) - - -def test_the_digest_changes_with_every_field_and_no_literal(): - config = bd.BlindingConfig() - digests = [v.digest() for v in _variants(config)] - assert len(digests) == len(set(digests)) and config.digest() not in digests - assert len(digests) > len(dataclasses.fields(TheoryConfig)) - - ints = bd.BlindingConfig(theory=TheoryConfig(w0=-1, wa=0, ia_bias=0)) - assert ints.digest() == config.digest() - assert bd._recorded_config("toy", json.loads(json.dumps(config.record()))) == config - - -@settings(max_examples=50) -@given(seed=st.text(min_size=1)) -@example(seed="my_secret_seed") -def test_the_commitment_is_the_forks_and_hides_its_rng_seed(seed): - """The fork seeds its RNG from the undomained sha256 of the seed; the - domain prefix keeps the commitment from publishing that RNG seed in its - first 16 hex characters, from which anyone could redraw the hidden point.""" - import smokescreen - from smokescreen.param_shifts import _normalize_seed - - assert cu.COMMITMENT_DOMAIN == smokescreen.COMMITMENT_DOMAIN - assert cu.seed_commitment(seed) == smokescreen.seed_commitment(seed) - assert int(cu.seed_commitment(seed)[:16], 16) != _normalize_seed(seed) - - -# --------------------------------------------------------------------------- # -# I9 and blind-drawn-once: init draws once, and the seed never touches disk +# The blind: drawn once, its seed never on disk # --------------------------------------------------------------------------- # SEED = "5eed" * 8 @@ -628,7 +238,7 @@ def test_the_commitment_is_the_forks_and_hides_its_rng_seed(seed): @pytest.mark.parametrize("fault", [None, "after_key", "encrypt"]) def test_init_never_writes_the_seed(tmp_path, monkeypatch, fault): """A normal init, or one interrupted anywhere, leaves no file holding the - seed; a completed record is read-only.""" + seed.""" from cryptography import fernet _registry(tmp_path) @@ -637,529 +247,86 @@ def test_init_never_writes_the_seed(tmp_path, monkeypatch, fault): monkeypatch.setattr(bd.os, "replace", lambda src, dst: 1 / 0) elif fault == "encrypt": monkeypatch.setattr(fernet.Fernet, "encrypt", lambda self, data: 1 / 0) - if fault is None: _init(tmp_path, "toy", "TOY") - for name in ("commitment.json", "seed.fernet", "key"): - mode = os.stat(tmp_path / "blinds" / "toy" / name).st_mode - assert not mode & (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) else: with pytest.raises(ZeroDivisionError): _init(tmp_path, "toy", "TOY") - hits = [ - p - for p in tmp_path.rglob("*") - if p.is_file() and SEED.encode() in p.read_bytes() - ] - assert hits == [] + files = [p for p in tmp_path.rglob("*") if p.is_file()] + assert not [p for p in files if SEED.encode() in p.read_bytes()] -def test_init_refuses_existing_state(fresh): +def test_a_blind_is_drawn_once(tmp_path): + _registry(tmp_path, ("toy", "TOY")) with pytest.raises(cu.CustodyError, match="exists"): - _init(fresh, "toy", "OTHER") + _init(tmp_path, "toy", "OTHER") with pytest.raises(cu.CustodyError, match="already covered"): - _init(fresh, "again", "TOY") - with pytest.raises(cu.CustodyError, match="blinded first"): - _init(fresh, "open", "TOY_OPEN") - (fresh / "blinds" / ".later.tmp").mkdir() - with pytest.raises(cu.CustodyError, match=r"\.later\.tmp exists"): - _init(fresh, "later", "OTHER") - - -@pytest.mark.parametrize( - "config, named", - [ - ({"envelope": {"S8": 0.075, "Omega_M": 0.1}}, "Omega_M"), - ({"theory": {"transfer_function": "eisenstein_hu", "Omega_M": 0.3}}, "Omega_M"), - ({"theory": {"transfer_function": "eisenstein-hu"}}, "eisenstein-hu"), - ], - ids=["envelope_key", "theory_key", "unbuildable"], -) -def test_init_refuses_a_config_no_blind_conceals_under(tmp_path, config, named): - """Refused before any record exists, so the base stays free for a blind - drawn under a good config.""" - _registry(tmp_path) - (tmp_path / "bad.json").write_text(json.dumps(config)) - args = ["init", "toy", "TOY", "--cat-config", str(tmp_path / "cat_config.yaml")] - with pytest.raises(cu.CustodyError, match=named): - bd.main([*args, "--config", str(tmp_path / "bad.json")]) - assert not any((tmp_path / "blinds").glob("*")) - _init(tmp_path, "toy", "TOY") - - -def test_share_adds_a_base_to_a_concealed_blind(fresh): - _declare(fresh, TOY_V={"base": "TOY"}, LATER={}) - cat_config = str(fresh / "cat_config.yaml") - for base, refusal in { - "TOY": "already covered", - "TOY_V": "variant", - "TOY_leak_corr": "variant", - "TOY_OPEN": "blinded first", - "TOY_MOCK": "blinded first", - }.items(): - with pytest.raises(cu.CustodyError, match=refusal): - bd.share("toy", base, cat_config=cat_config) - - assert bd.main(["share", "toy", "OTHER", "--cat-config", cat_config]) == 0 - toy, other = _custody(fresh, "TOY"), _custody(fresh, "OTHER") - assert (other.blind, other.commitment) == ("toy", toy.commitment) - - _publish(fresh, bd.open_blind(toy).seed) - with pytest.raises(cu.CustodyError, match="revealed"): - bd.share("toy", "LATER", cat_config=cat_config) - - -def test_a_blind_is_drawn_for_every_catalogue_reading_its_file(tmp_path): - """init and share refuse a catalogue whose shear file another catalogue - reads under another custody, before anything is written; drawn for both, - the blind covers both.""" - _registry(tmp_path) - toy, later = ({"shear": {"path": str(tmp_path / f)}} for f in ("a.fits", "b.fits")) - _declare( - tmp_path, - TOY=toy, - TOY_TWIN={**toy, "blinding": "unblinded"}, - LATER=later, - LATER_TWIN={**later, "blinding": "unblinded"}, - ) - with pytest.raises(cu.CustodyError, match="TOY_TWIN"): - _init(tmp_path, "toy", "TOY") - assert not (tmp_path / "blinds").exists() - - _declare(tmp_path, TOY_TWIN=toy) - _init(tmp_path, "toy", "TOY", "TOY_TWIN") - with pytest.raises(cu.CustodyError, match="LATER_TWIN"): - bd.share("toy", "LATER", cat_config=str(tmp_path / "cat_config.yaml")) - bases = (tmp_path / "blinds" / "toy" / "bases").read_text().split() - assert bases == ["TOY", "TOY_TWIN"] - - -# --------------------------------------------------------------------------- # -# I10: an opened blind is the one committed -# --------------------------------------------------------------------------- # -@pytest.fixture(scope="module") -def tamperable(tmp_path_factory): - """Blind `toy`'s record, restored after each edit, and TOY's custody.""" - root = _registry(tmp_path_factory.mktemp("tamperable"), ("toy", "TOY")) - return root / "blinds" / "toy" / "commitment.json", _custody(root, "TOY") + _init(tmp_path, "again", "TOY") -def _leaves(record, path=()): - for key, value in record.items(): - if isinstance(value, dict): - yield from _leaves(value, (*path, key)) - else: - yield (*path, key) - - -# The fields of a blind's record the sealed seed binds: its name, commitment, -# draw scheme, config digest and every field of its config. -BOUND = sorted( - [("blind",), ("seed_commitment",), ("config_digest",), ("draw_scheme",)] - + [("config", *leaf) for leaf in _leaves(bd.BlindingConfig().record())] -) - - -def _edited(value): - if isinstance(value, bool): - return not value - return value + ("x" if isinstance(value, str) else 1) - - -@settings(max_examples=15, deadline=None) -@given(leaf=st.sampled_from(BOUND), redigest=st.booleans()) -@example(leaf=("config", "envelope", "S8"), redigest=True) -@example(leaf=("config", "theory", "transfer_function"), redigest=False) -def test_open_blind_refuses_any_edit_of_the_record(tamperable, leaf, redigest): - """Edit any field the sealed seed binds, a config field with its digest - recomputed or not: opening the blind is refused.""" - path, custody = tamperable - original = path.read_bytes() - record = json.loads(original) - *parents, key = leaf - node = record - for parent in parents: - node = node[parent] - node[key] = _edited(node[key]) - if redigest and leaf != ("config_digest",): - record["config_digest"] = bd.record_digest(record["config"]) - os.chmod(path, 0o644) - try: - path.write_text(json.dumps(record)) - bd._open.cache_clear() - with pytest.raises(cu.CustodyError): - bd.open_blind(custody) - finally: - path.write_bytes(original) - bd._open.cache_clear() - assert bd.open_blind(custody).seed not in repr(bd.open_blind(custody)) - - -@pytest.mark.parametrize("tamper", ["wrong_key", "renamed", "custody"]) -def test_open_blind_refuses_another_key_name_or_commitment(fresh, tamper): - from cryptography.fernet import Fernet - - record = fresh / "blinds" / "toy" - if tamper == "wrong_key": - os.chmod(record / "key", 0o644) - (record / "key").write_bytes(Fernet.generate_key()) - elif tamper == "renamed": - os.rename(record, record.with_name("toy2")) - commitment = record.with_name("toy2") / "commitment.json" - os.chmod(commitment, 0o644) - commitment.write_text( - json.dumps({**json.loads(commitment.read_text()), "blind": "toy2"}) - ) - custody = _custody(fresh, "TOY") - if tamper == "custody": - custody = dataclasses.replace(custody, commitment="0" * 64) - with pytest.raises(cu.CustodyError): - bd.open_blind(custody) - - -def _init_storing(root, edit): - """Draw blind toy for TOY (seed ``toy-seed``), its config stored as ``edit`` - makes it: the record a code with another TheoryConfig schema writes.""" - record = bd.BlindingConfig.record - config = bd.BlindingConfig(theory=TheoryConfig(**FAST["theory"])) - with pytest.MonkeyPatch.context() as m: - m.setattr(bd.BlindingConfig, "record", lambda self: edit(record(self))) - m.setattr(bd.secrets, "token_hex", lambda n: "toy-seed") - # That code's init opened the record under its own schema. - m.setattr(bd, "_conceal_one_row", lambda name, seed, record: None) - bd.init("toy", ["TOY"], cat_config=root / "cat_config.yaml", config=config) - +def test_the_commitment_hides_the_rng_seed(): + """The fork seeds its RNG from the undomained sha256 of the seed; the + domain prefix keeps the public commitment from publishing it.""" + from smokescreen.param_shifts import _normalize_seed -@pytest.mark.parametrize( - "edit, field", - [ - ( - lambda r: { - **r, - "theory": {k: v for k, v in r["theory"].items() if k != "ia_alphaz"}, - }, - "ia_alphaz", - ), - ( - lambda r: {**r, "theory": {**r["theory"], "baryon_boost": 0.0}}, - "baryon_boost", - ), - ], - ids=["lacks_a_field", "names_a_foreign_field"], -) -def test_a_record_of_another_schema_is_refused_by_the_field( - tmp_path, blinds, monkeypatch, edit, field -): - """A record lacking a TheoryConfig field, or naming one this code lacks, - is refused by the field's name, never as an edited record. Once a lacking - field's neutral value is declared, it conceals as the blind that names it.""" - _registry(tmp_path) - _init_storing(tmp_path, edit) - custody = _custody(tmp_path, "TOY") - with pytest.raises(cu.CustodyError, match=field) as refused: - bd.open_blind(custody) - assert "edited" not in str(refused.value) - if field == "ia_alphaz": - monkeypatch.setattr(bd, "NEUTRAL", {"ia_alphaz": 0.0}) - s = two_bin_sacc(cl=False) - assert np.array_equal( - _values(sio.seal(s, custody)), _values(sio.seal(s, blinds["TOY"])) - ) + seed = "the-secret" + assert int(cu.seed_commitment(seed)[:16], 16) != _normalize_seed(seed) def test_the_committed_blind_opens_under_this_code(): - """``tests/data/blinds/committed`` opens to the point it was drawn at. - - A TheoryConfig field added without its value in ``blinding.NEUTRAL`` would - strand every live blind; this record turns that red. - """ + """A TheoryConfig field added without its value in ``blinding.NEUTRAL`` + would strand every live blind; ``tests/data/blinds/committed`` turns that + red.""" blind = bd._open(DATA / "blinds", "committed") assert (blind.hidden.S8, blind.hidden.Omega_m) == pytest.approx( - COMMITTED_POINT, rel=1e-12 + (0.8374952413635888, 0.32730573347638175), rel=1e-12 ) -def test_a_blind_drawn_under_another_draw_scheme_is_refused(tmp_path, monkeypatch): - """Its seed and record agree, but this install's fork draws differently: - opening it is refused, and a file stamped under it fails ``verify``.""" - _registry(tmp_path) - installed = bd.draw_scheme() - with monkeypatch.context() as m: - m.setattr(bd, "draw_scheme", lambda: installed + 1) - _init(tmp_path, "toy", "TOY") - custody = _custody(tmp_path, "TOY") - with pytest.raises(cu.CustodyError, match="draw scheme"): - bd.open_blind(custody) - part = tmp_path / "rho_tau.sacc" - sio.save(rho_sacc(), part, custody=custody) # no signal: the blind stays shut - assert bd.verify(part, cat_config=tmp_path / "cat_config.yaml") == [ - "this install draws under another scheme" - ] - - -# --------------------------------------------------------------------------- # -# verify: a file's stamp against its catalogue's custody, seedless -# --------------------------------------------------------------------------- # -def test_verify_names_what_disagrees_with_the_declaration(fresh, capsys): - def verify(path): - code = bd.main( - ["verify", str(path), "--cat-config", str(fresh / "cat_config.yaml")] - ) - return code, capsys.readouterr().out - - part, forged = fresh / "part.sacc", fresh / "forged.sacc" - custody = _custody(fresh, "TOY") - sio.save(two_bin_sacc(cl=False), part, custody=custody) - assert verify(part)[0] == 0 - - _write( - forged, - two_bin_sacc(cl=False), - {**custody.stamp, "blinding_commitment": "0" * 64}, - ) - code, out = verify(forged) - assert code == 1 and "['blinding_commitment']" in out - - _publish(fresh, bd.open_blind(custody).seed) - _declare(fresh, TOY={"blinding": "unblinded"}) - code, out = verify(part) - assert code == 1 and "'blinding'" in out and "unblinded:TOY" in out - - # --------------------------------------------------------------------------- # -# I11: reveal publishes and archives; the audit proves blinded − true = shift(seed) +# The reveal: publish, then prove blinded − true = shift(seed) # --------------------------------------------------------------------------- # -def _write(path, s, stamp): - """``s`` on disk under ``stamp`` as given, as the door would never write it.""" - s = s.copy() - s.metadata.update(stamp) - path.parent.mkdir(parents=True, exist_ok=True) - s.save_fits(str(path), overwrite=True) - - -def _publish(root, seed): - (root / "blinds" / "toy" / "revealed.json").write_text(json.dumps({"seed": seed})) - - -def test_reveal_refuses_a_root_holding_nothing_concealed(fresh): - """A mistyped or unbound ``--root`` is refused before the seed is published; - a reveal interrupted after its first move runs again.""" - root = fresh / "output" - part = root / "sub" / "part.sacc" - part.parent.mkdir(parents=True) - sio.save(two_bin_sacc(cl=False), part, custody=_custody(fresh, "TOY")) - revealed = fresh / "blinds" / "toy" / "revealed.json" - cat_config = fresh / "cat_config.yaml" - (fresh / "empty").mkdir() - for wrong in (fresh / "outptu", fresh / "empty"): - with pytest.raises(cu.CustodyError, match="nothing concealed"): - bd.reveal("toy", root=wrong, cat_config=cat_config) - assert not revealed.exists() - - archive = bd.reveal("toy", root=root, cat_config=cat_config) - assert revealed.exists() and (archive / "sub" / "part.sacc").exists() - assert bd.reveal("toy", root=root, cat_config=cat_config) == archive +def test_reveal_refuses_a_root_holding_nothing_concealed(tmp_path): + """Publishing cannot be undone: a mistyped ``--root`` is refused before + the seed is published.""" + root = _registry(tmp_path, ("toy", "TOY")) + (root / "out").mkdir() + sio.save(part(cl=False), root / "out" / "part.sacc", custody=_custody(root, "TOY")) + cat_config = root / "cat_config.yaml" with pytest.raises(cu.CustodyError, match="nothing concealed"): - bd.reveal("toy", root=fresh / "empty", cat_config=cat_config) - - -def test_reveal_refuses_a_record_publishing_another_seed(fresh): - part = fresh / "output" / "part.sacc" - part.parent.mkdir() - sio.save(two_bin_sacc(cl=False), part, custody=_custody(fresh, "TOY")) - _publish(fresh, "0" * 32) - with pytest.raises(cu.CustodyError, match="another seed"): - bd.reveal("toy", root=part.parent, cat_config=fresh / "cat_config.yaml") - assert part.exists() + bd.reveal("toy", root=root / "outptu", cat_config=cat_config) + assert not (root / "blinds" / "toy" / "revealed.json").exists() + archive = bd.reveal("toy", root=root / "out", cat_config=cat_config) + assert (archive / "part.sacc").exists() -@pytest.fixture(scope="module") -def revealed(tmp_path_factory): - """A registry whose blind `toy` has published its seed, TOY's custody - before the reveal, ``AUDITED`` sealed under it, and ``audit(archived, - live)``, which archives ``archived`` beside ``live`` as its re-measured - twin and audits the pair.""" - root = _registry(tmp_path_factory.mktemp("revealed"), ("toy", "TOY")) +@pytest.mark.parametrize("left_true", [False, True], ids=["revealed", "mixed"]) +def test_the_audit_proves_the_shift(tmp_path, left_true): + """An archived part passes the audit against its re-measured twin only if + every shiftable row carries the shift: a pair left true fails.""" + root = _registry(tmp_path, ("toy", "TOY")) blinded = _custody(root, "TOY") - _publish(root, bd.open_blind(blinded).seed) - true = cu.Custody("unblinded", "TOY") - - def audit( - archived, live, *, stamps=(blinded.stamp, true.stamp), centres=("a", "a") - ): - for tree, s, stamp, centre in zip( - ("archive", "live"), (archived, live), stamps, centres - ): - s = s.copy() - s.metadata["patch_centers_sha256"] = centre - _write(root / tree / "sub" / "part.sacc", s, stamp) - return bd.audit( - "toy", - archive=root / "archive", - true_root=root / "live", - cat_config=root / "cat_config.yaml", - ) - - fields = ["root", "blinded", "sealed", "audit"] - return dataclasses.make_dataclass("Revealed", fields)( - root, blinded, sio.seal(AUDITED, blinded), audit - ) - - -def _problems(report): - return report["problems"] + [ - p for part in report["parts"].values() for p in part["problems"] - ] - - -AUDITED = two_bin_sacc(xi_tags=("reporting",), rho=True) -BLOCKS = [ - (types, sio._pair(pair)) - for pair in PAIRS - for types in ((sio.XI_PLUS, sio.XI_MINUS), (sio.CL_EE,)) -] - - -@settings(max_examples=4, deadline=None) -@given( - times=st.lists( - st.sampled_from([1.0, 0.0, 2.0, 1 + 1e-4, -1.0]), min_size=6, max_size=6 - ) -) -@example(times=[1.0] * 6) -@example(times=[1.0] * 5 + [0.0]) # one block left true -@example(times=[1.0] * 3 + [2.0] + [1.0] * 2) # one block shifted twice -@example(times=[1 + 1e-4] + [1.0] * 5) -def test_the_audit_accepts_exactly_the_shift(revealed, times): - """An archived part whose ξ± and Cℓ_EE blocks carry ``times`` × the blind's - shift passes the audit exactly when every block carries it once.""" - true = AUDITED - shift = _values(revealed.sealed) - _values(true) - archived = true.copy() - for k, (types, tracers) in zip(times, BLOCKS): + true = part() + archived = sio.seal(true, blinded) + if left_true: for i, dp in enumerate(true.data): - if dp.data_type in types and dp.tracers == tracers: - archived.data[i].value = dp.value + k * shift[i] - report = revealed.audit(archived, true) - assert report["ok"] == all(k == 1.0 for k in times), report - if report["ok"]: - (part,) = report["parts"].values() - assert part["residual"] <= 1e-6 - assert ( - abs(report["shift"]["S8"]) <= 0.075 - and abs(report["shift"]["Omega_m"]) <= 0.1 - ) - else: - assert any("≠ shift(seed)" in problem for problem in _problems(report)) - - -SPOILT = { - "covariance": "covariances differ", - "rows": "rows, tags or tracers differ", - "live_mock": "the live file is stamped mock:TOY", - "live_other_catalogue": "the live file is stamped unblinded:OTHER", - "archive_other_blind": "not concealed under this blind", - "patch_centres": "patch centres differ: ['a', 'b']", - "wrong_seed": "the published seed is not the committed one", -} - - -@pytest.mark.parametrize("spoilt", SPOILT) -def test_the_audit_names_a_pair_that_is_not_concealed_and_true(revealed, spoilt): - """An archived part is audited only under the published committed seed, - against the same catalogue re-measured unblinded with the same rows, - covariance and patch centres.""" - true = AUDITED - live = ( - two_bin_sacc(xi_tags=("reporting",), cl=False) - if spoilt == "rows" - else true.copy() + if dp.tracers == sio._pair((1, 1)) and dp.data_type in sio.SHIFTABLE: + archived.data[i].value = dp.value + (root / "blinds" / "toy" / "revealed.json").write_text( + json.dumps({"seed": bd.open_blind(blinded).seed}) ) - if spoilt == "covariance": - live.add_covariance(1.01 * np.asarray(true.covariance.dense), overwrite=True) - archived_stamp = revealed.blinded.stamp - if spoilt == "archive_other_blind": - archived_stamp = {**archived_stamp, "blinding_commitment": "0" * 64} - live_custody = { - "live_mock": cu.Custody("mock", "TOY"), - "live_other_catalogue": cu.Custody("unblinded", "OTHER"), - }.get(spoilt, cu.Custody("unblinded", "TOY")) - published = revealed.root / "blinds" / "toy" / "revealed.json" - seed = published.read_text() - try: - if spoilt == "wrong_seed": - _publish(revealed.root, "not-the-seed") - report = revealed.audit( - revealed.sealed, - live, - stamps=(archived_stamp, live_custody.stamp), - centres=("a", "b" if spoilt == "patch_centres" else "a"), - ) - finally: - published.write_text(seed) - assert not report["ok"] and _problems(report) == [SPOILT[spoilt]] - - -def test_the_audit_of_an_empty_archive_says_so(revealed, tmp_path): + for tree, s, stamp in ( + ("archive", archived, blinded.stamp), + ("live", true, cu.Custody("unblinded", "TOY").stamp), + ): + s = s.copy() + s.metadata.update(stamp) + (root / tree).mkdir() + s.save_fits(str(root / tree / "part.sacc")) report = bd.audit( "toy", - archive=tmp_path, - true_root=tmp_path, - cat_config=revealed.root / "cat_config.yaml", + archive=root / "archive", + true_root=root / "live", + cat_config=root / "cat_config.yaml", ) - assert not report["ok"] and report["problems"] == ["the archive holds no parts"] - - -@pytest.mark.parametrize("beside_signal", [False, True], ids=["alone", "beside_signal"]) -def test_the_audit_passes_rho_tau_remeasured_at_run_noise(revealed, beside_signal): - """ρ/τ carries no signal, and a re-run does not reproduce it (TreeCorr's - k-means patches move θ by ~1e-3 and the values by O(1)): its part is - judged by its stamp, and beside signal rows only the signal is compared.""" - true = AUDITED if beside_signal else rho_sacc() - rerun = true.copy() - cov = np.array(true.covariance.dense) - for i, dp in enumerate(true.data): - if not sio.is_signal(dp.data_type): - rerun.data[i].value = 1.5 * dp.value + 1e-7 - rerun.data[i].tags["theta"] *= 1 + 1.7e-3 - cov[i, i] *= 1.036 - rerun.add_covariance(cov, overwrite=True) - archived = revealed.sealed if beside_signal else sio.seal(true, revealed.blinded) - report = revealed.audit(archived, rerun) - assert report["ok"], report - - -@pytest.mark.parametrize("kind", [sio.CL_BB, sio.CL_EB]) -def test_the_audit_fails_when_an_unshifted_row_moved(revealed, kind): - true = AUDITED - archived = revealed.sealed.copy() - i = _rows(true, kind)[-1] - archived.data[i].value += 1e-6 * abs(true.data[i].value) - report = revealed.audit(archived, true) - assert _problems(report) == [f"{kind} moved, but the blind leaves it unshifted"] - - -@pytest.mark.parametrize( - "e_kind, b_kind", - [ - (sio.COSEBI_EE, sio.COSEBI_BB), - (sio.PURE_TYPES["xim_E"], sio.PURE_TYPES["xim_B"]), - ], - ids=["cosebis", "pure_eb"], -) -@pytest.mark.parametrize("b_sigma", [0.0, 1.0]) -def test_the_audit_bounds_derived_b_modes(revealed, e_kind, b_kind, b_sigma): - """A derived part's E rows move with the blind; its B rows by ≤ ``B_SIGMA``.""" - true = derived_sacc("cosebis" if e_kind == sio.COSEBI_EE else "pure_eb") - true.add_covariance(np.full(len(true.mean), 0.01), overwrite=True) - archived = true.copy() - for i in _rows(true, e_kind): - archived.data[i].value += 0.3 - for i in _rows(true, b_kind): - archived.data[i].value += b_sigma * 0.1 - report = revealed.audit(archived, true) - assert report["ok"] == (b_sigma == 0.0), report - (part,) = report["parts"].values() - assert part["shift_over_sigma"][e_kind] == pytest.approx(3.0) - if b_sigma: - assert _problems(report) == [f"{b_kind} moved by 1.00e+00σ under the blind"] + assert report["ok"] != left_true, report diff --git a/src/sp_validation/tests/test_blinding_recipe.py b/src/sp_validation/tests/test_blinding_recipe.py new file mode 100644 index 00000000..b04245bb --- /dev/null +++ b/src/sp_validation/tests/test_blinding_recipe.py @@ -0,0 +1,24 @@ +"""The blinding theory's nonlinear recipe is the inference pipeline's. + +The blinding shift is a difference of CCL theory vectors; inference runs CAMB +through CosmoSIS. The shift means what it is meant to only if both use one +recipe, so the blinding reads it from the CosmoSIS config it must match. +""" + +import pathlib +import re + +from sp_validation import blinding_theory as bt + +INI = ( + pathlib.Path(__file__).resolve().parents[3] + / "cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia_cell.ini" +) + + +def test_halofit_recipe_matches_inference_config(): + match = re.search(r"^halofit_version\s*=\s*(\S+)", INI.read_text(), re.MULTILINE) + assert match, f"no halofit_version in {INI}" + cfg = bt.TheoryConfig() + assert cfg.halofit_version == match.group(1) + assert cfg.transfer_function == "boltzmann_camb" diff --git a/src/sp_validation/tests/test_camb_ccl_crosscheck.py b/src/sp_validation/tests/test_camb_ccl_crosscheck.py deleted file mode 100644 index 02414f5d..00000000 --- a/src/sp_validation/tests/test_camb_ccl_crosscheck.py +++ /dev/null @@ -1,175 +0,0 @@ -"""The blinding theory against an independent CAMB oracle (AC10–13). - -The blinding shift is a difference of CCL theory vectors; inference runs CAMB -(CosmoSIS). The shift means what it is meant to only if CCL and CAMB predict the -same ξ± at a fixed cosmology on our θ grid. This module compares -:func:`sp_validation.blinding_theory.xi_ccl` (CCL's Boltzmann-CAMB HMCode2020 -P(k), projected by CCL's Limber + FFTLog) with an oracle written here: a direct -pycamb run of the HMCode2020 ``P(k, z)`` at a σ8-matched ``A_s``, wrapped in a -``ccl.Pk2D`` and projected by the same CCL machinery. - -Both paths route P(k) through CAMB and project through CCL, so a shared -projection bug cancels: the comparison validates the P(k) recipe and the σ8/A_s -amplitude convention. The fiducial fixes σ8 for CCL but A_s for CAMB; a nominal -``A_s = 2.1e-9`` leaves CAMB's σ8 ≈3% off target, enough to move ξ± by ~10%. -""" - -import dataclasses -import pathlib -import re - -import numpy as np - -from sp_validation import blinding_theory as bt - -XIP_RTOL = 0.005 -XIM_RTOL = 0.010 -# ξ− crosses zero on this grid: the relative bound applies only where |ξ−| -# exceeds this fraction of its peak. -XIM_FLOOR_FRAC = 0.05 - -THETA_ARCMIN = np.geomspace(5.0, 250.0, 12) -PK_ZMAX, PK_NZ = 3.0, 48 - - -def _gauss_nz(n=400): - z = np.linspace(0.01, 3.0, n) - nz = np.exp(-0.5 * ((z - 0.7) / 0.2) ** 2) - return z, nz / np.trapezoid(nz, z) - - -# --------------------------------------------------------------------------- # -# The CAMB oracle -# --------------------------------------------------------------------------- # -def camb_params(config, As, *, nonlinear, kmax=20.0): - """``CAMBparams`` at ``config``'s background, every field fed from one source.""" - import camb - - p = camb.CAMBparams() - p.set_cosmology( - H0=config.h * 100, - ombh2=config.Omega_b * config.h**2, - omch2=config.omega_c() * config.h**2, - mnu=config.m_nu, - num_massive_neutrinos=1, - neutrino_hierarchy=config.mass_split, - nnu=bt.NEFF, - TCMB=bt.T_CMB, - ) - p.set_dark_energy(w=config.w0, wa=config.wa, dark_energy_model="ppf") - p.InitPower.set_params(As=As, ns=config.n_s) - p.set_matter_power(redshifts=list(np.linspace(0.0, PK_ZMAX, PK_NZ)), kmax=kmax) - if nonlinear: - p.NonLinear = camb.model.NonLinear_both - p.NonLinearModel.set_params( - halofit_version=config.halofit_version, - HMCode_logT_AGN=config.hmcode_logT_AGN, - ) - else: - p.NonLinear = camb.model.NonLinear_none - return p - - -def camb_sigma8(config, As): - import camb - - return float( - camb.get_results(camb_params(config, As, nonlinear=False)).get_sigma8_0() - ) - - -def camb_As_for_sigma8(config, target, As_seed=2.1e-9): - """σ8² ∝ A_s exactly, so one evaluation and one rescale land on target.""" - return As_seed * (target / camb_sigma8(config, As_seed)) ** 2 - - -def xi_camb(config, nz, theta_arcmin, *, n_ell=300, ell_max=60000, kmax=20.0, n_k=400): - """ξ± from a direct CAMB P(k), projected by CCL; returns ``(xip, xim, As)``.""" - import camb - import pyccl as ccl - - As = camb_As_for_sigma8(config, config.sigma8()) - results = camb.get_results(camb_params(config, As, nonlinear=True, kmax=kmax)) - # CCL's native units already: k in 1/Mpc, P in Mpc³. - interp = results.get_matter_power_interpolator( - nonlinear=True, hubble_units=False, k_hunit=False - ) - k = np.geomspace(1e-4, kmax * config.h, n_k) - z = np.linspace(0.0, PK_ZMAX, PK_NZ) - a = 1.0 / (1.0 + z) - order = np.argsort(a) - pk2d = ccl.Pk2D( - a_arr=a[order], - lk_arr=np.log(k), - pk_arr=np.log(interp.P(z, k)[order]), - is_logp=True, - ) - cosmo = bt.ccl_cosmology(config.ccl_params(), config) - lens = ccl.WeakLensingTracer(cosmo, dndz=nz) - ells = np.unique(np.geomspace(2, ell_max, n_ell).astype(int)).astype(float) - cl = ccl.angular_cl(cosmo, lens, lens, ells, p_of_k_a=pk2d) - theta_deg = np.asarray(theta_arcmin) / 60.0 - xip = ccl.correlation(cosmo, ell=ells, C_ell=cl, theta=theta_deg, type="GG+") - xim = ccl.correlation(cosmo, ell=ells, C_ell=cl, theta=theta_deg, type="GG-") - return xip, xim, As - - -# --------------------------------------------------------------------------- # -# AC10–12 -# --------------------------------------------------------------------------- # -def _assert_agreement(config, label): - nz = _gauss_nz() - xip_a, xim_a = bt.xi_ccl(config.ccl_params(), config, nz, nz, THETA_ARCMIN) - xip_b, xim_b, _ = xi_camb(config, nz, THETA_ARCMIN) - assert np.all(xip_a > 0) and np.all(xip_b > 0) - rel_p = np.abs(xip_b - xip_a) / np.abs(xip_a) - assert rel_p.max() < XIP_RTOL, f"{label}: ξ+ max rel diff {rel_p.max():.3%}" - floor = XIM_FLOOR_FRAC * np.max(np.abs(xim_a)) - above = np.abs(xim_a) > floor - rel_m = np.abs(xim_b - xim_a)[above] / np.abs(xim_a)[above] - assert rel_m.max() < XIM_RTOL, f"{label}: ξ− max rel diff {rel_m.max():.3%}" - assert np.all(np.abs(xim_b - xim_a)[~above] < XIM_RTOL * floor), label - - -def test_ac10_sigma8_As_reconciliation(): - """Nominal A_s misses σ8 by >2%; the closed-form rescale lands within 1e-4.""" - cfg = bt.TheoryConfig() - assert abs(camb_sigma8(cfg, 2.1e-9) / cfg.sigma8() - 1) > 0.02 - assert ( - abs(camb_sigma8(cfg, camb_As_for_sigma8(cfg, cfg.sigma8())) - cfg.sigma8()) - < 1e-4 - ) - - -def test_ac11_xi_agreement_at_fiducial(): - _assert_agreement(bt.TheoryConfig(), "fiducial") - - -def test_ac12_xi_agreement_off_fiducial(): - """An in-envelope point: the shift must not inherit a stack disagreement.""" - cfg = dataclasses.replace(bt.TheoryConfig(), S8=0.80 + 0.075, Omega_m=0.30 - 0.05) - _assert_agreement(cfg, "off-fiducial") - - -# --------------------------------------------------------------------------- # -# AC13: the nonlinear recipe is the inference config's -# --------------------------------------------------------------------------- # -def test_ac13_halofit_token_matches_inference_config(): - """The blinding recipe is the CosmoSIS pipeline's, read from its config file. - - The CCL path and the CAMB oracle share the recipe by construction, so they - would agree while jointly diverging from inference; only this lineage check - catches that. - """ - ini = ( - pathlib.Path(__file__).resolve().parents[3] - / "cosmo_inference" - / "cosmosis_config" - / "templates" - / "cosmosis_pipeline_A_ia_cell.ini" - ) - match = re.search(r"^halofit_version\s*=\s*(\S+)", ini.read_text(), re.MULTILINE) - assert match, f"no halofit_version in {ini}" - cfg = bt.TheoryConfig() - assert cfg.halofit_version == match.group(1) - assert cfg.transfer_function == "boltzmann_camb" diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index f3d932e2..3c994989 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -310,50 +310,6 @@ def test_calculate_2pcf_runs_on_synthetic_catalog(self, tmp_path): # The additive-bias subtraction in the pipeline must have run. assert version in cv.c1 and version in cv.c2 - @pytest.mark.parametrize("npatch", [1, 4]) - def test_xi_part_carries_the_covariance_the_measurement_estimated( - self, tmp_path, npatch - ): - """run_2pcf's ξ± part carries TreeCorr's covariance, whoever calls it. - - The jackknife covariance (correlated bins) with patches, the shot-noise - diagonal without, so every part has variances whether the rule or the - CLI measured it. - """ - import importlib.util - - import sacc - - script = Path(__file__).resolve().parents[3] / "workflow/scripts/run_2pcf.py" - spec = importlib.util.spec_from_file_location("run_2pcf_part", script) - run_2pcf = importlib.util.module_from_spec(spec) - spec.loader.exec_module(run_2pcf) - - params, version = write_synthetic_catalogs(tmp_path) - if npatch > 1: - CosmologyValidation(versions=[version], **params).write_patch_centers( - version, npatch - ) - part = tmp_path / "part.sacc" - run_2pcf.run_2pcf( - ver=version, - min_sep=5.0, - max_sep=100.0, - nbins=6, - npatch=npatch, - cat_config=params["catalog_config"], - output_dir=params["output_dir"], - sacc_out=str(part), - ) - - cov = sacc_io.load(str(part)).covariance - assert np.all(np.diag(cov.dense) > 0) - if npatch > 1: - assert isinstance(cov, sacc.covariance.FullCovariance) - assert np.any(cov.dense[~np.eye(12, dtype=bool)] != 0) - else: - assert isinstance(cov, sacc.covariance.DiagonalCovariance) - def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): """calculate_2pcf's ξ± is the same on 4 and on 48 TreeCorr threads. @@ -462,9 +418,8 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( The ξ± it measures equal the committed ``pure_eb_xi``, its modes are ``pure_eb_from_xi`` of those ξ± and edges, and every reporting bin is - finite. ``test_b_modes`` pins the transform itself on the same ξ±, so a - failure names the step that moved: measurement, wiring or transform. - The jackknife covariance runs the transform once per realisation. + finite. The jackknife covariance runs the transform once per + realisation. Finiteness: the Schneider (2022) integrals are near-singular where a reporting bin meets the integration boundary, so the integration grid diff --git a/src/sp_validation/tests/test_custody.py b/src/sp_validation/tests/test_custody.py index 6e88f633..99568b13 100644 --- a/src/sp_validation/tests/test_custody.py +++ b/src/sp_validation/tests/test_custody.py @@ -1,10 +1,9 @@ -"""Custody is declared per base catalogue and resolved in one place (I12, I13). +"""Custody is declared per base catalogue and resolved in one place. ``sp_validation.custody.custody_of`` reads a catalogue's declaration and the -blind registry beside the catalogue config. These tests pin every row of its -table on hand-written registries (the host never decrypts, so a record needs -no real seed), and that the host Snakemake and a container job resolve the same -custody for every version. +blind registry beside the catalogue config, here hand-written (the host never +decrypts, so a record needs no real seed). The host Snakemake and a container +job must resolve the same custody for every version. """ import importlib.util @@ -61,67 +60,32 @@ def _catalogues(**declarations): # --------------------------------------------------------------------------- # -# I12: the custody table +# The custody table # --------------------------------------------------------------------------- # -NO_BLIND = [ - "declares no custody for it, so it is blinded", - "blinding init", - "blinding share", -] +NO_BLIND = ["declares no custody for it, so it is blinded", "blinding init"] Y3 = {"y3": (["SP_v9"], None)} # a blind over SP_v9, concealed PUBLIC = {"y3": (["SP_v9"], "seed-of-y3")} # the same, its seed published -V = "SP_v9_ecut07" -# case: (declarations, blinds {name: (bases, published seed)}, expected, version). +# case: (declarations, blinds {name: (bases, published seed)}, expected). # expected is the custody's token, or the fragments of the refusal. # fmt: off TABLE = { "undeclared": ({"SP_v9": None}, {}, NO_BLIND), - "blinded": ({"SP_v9": "blinded"}, {}, ["no blind covers it"]), "undeclared_covered": ({"SP_v9": None}, Y3, "blinded:SP_v9:y3:{y3}"), - "blinded_covered": ({"SP_v9": "blinded"}, Y3, "blinded:SP_v9:y3:{y3}"), "blinded_revealed": ({"SP_v9": None}, PUBLIC, ["declare `blinding: unblinded`"]), "unblinded": ({"SP_v9": "unblinded"}, {}, "unblinded:SP_v9"), "unblinded_concealed": ({"SP_v9": "unblinded"}, Y3, ["blinding reveal y3"]), "unblinded_revealed": ({"SP_v9": "unblinded"}, PUBLIC, "unblinded:SP_v9"), - "unblinded_wrong_seed": ( - {"SP_v9": "unblinded"}, {"y3": (["SP_v9"], "not-the-seed")}, ["its commitment"] - ), "mock": ({"SP_v9": "mock"}, {}, "mock:SP_v9"), "mock_covered": ({"SP_v9": "mock"}, Y3, ["a mock is never blinded"]), - "two_blinds": ({"SP_v9": None}, {**Y3, "b": (["SP_v9"], None)}, ["blinds: b, y3"]), - "unknown": ({"SP_v9": "open"}, {}, ["blinded, unblinded or mock"]), - "variant_declares": ( - {"SP_v9": "unblinded", V: {"base": "SP_v9", "blinding": "unblinded"}}, {}, - ["declare custody on SP_v9"], V, - ), - "alias_repeats": ( - {"SP_v9": "unblinded", "SP_v9_leak_corr": "unblinded"}, {}, "unblinded:SP_v9", - "SP_v9_leak_corr", - ), - "alias_repeats_base": ( - {"SP_v9": "unblinded", V: {"base": "SP_v9"}, f"{V}_seed7": {"base": "SP_v9"}}, - {}, "unblinded:SP_v9", f"{V}_seed7", - ), - "alias_declares": ( - {"SP_v9": "unblinded", "SP_v9_leak_corr": "blinded"}, {}, - ["declare custody on SP_v9"], "SP_v9_leak_corr", - ), - "alias_names_another_base": ( - {"SP_v9": "unblinded", "TOY": None, "SP_v9_seed7": {"base": "TOY"}}, - {"y3": (["TOY"], None)}, ["declare custody on SP_v9"], "SP_v9_seed7", - ), } # fmt: on @pytest.mark.parametrize("case", TABLE) def test_the_custody_table(tmp_path, case): - """Every row of custody_of's table: the declaration on the base catalogue - against the blinds covering it. An entry named as a variant of another - (``_leak_corr``, ``_seed``) may repeat that entry's - custody, never declare another.""" - declarations, blinds, expected, *version = TABLE[case] - version = version[0] if version else "SP_v9" + """custody_of: the declaration on the base catalogue against the blinds + covering it; absent means blinded.""" + declarations, blinds, expected = TABLE[case] registry = tmp_path / "blinds" commitments = { name: cu.seed_commitment(_write_blind(registry, name, bases, revealed=seed)) @@ -129,30 +93,15 @@ def test_the_custody_table(tmp_path, case): } cats = _catalogues(**declarations) if isinstance(expected, str): - custody = cu.custody_of(cats, version, registry=registry) + custody = cu.custody_of(cats, "SP_v9", registry=registry) assert custody.token == expected.format(**commitments) return with pytest.raises(cu.CustodyError) as refused: - cu.custody_of(cats, version, registry=registry) + cu.custody_of(cats, "SP_v9", registry=registry) for fragment in expected: assert fragment in str(refused.value) -def test_the_printed_commands_name_the_config_as_it_was_given(tmp_path): - """The operator pastes these commands, so they keep the launch's spelling - of the checkout: on candide the plain /nXXdataN, never the /automnt path it - resolves to.""" - (tmp_path / "real" / "cosmo_val").mkdir(parents=True) - checkout = tmp_path / "checkout" - checkout.symlink_to(tmp_path / "real", target_is_directory=True) - registry = cu.registry_of(checkout / "cosmo_val" / "cat_config.yaml") - with pytest.raises(cu.CustodyError) as err: - cu.custody_of(_catalogues(SP_v9=None), "SP_v9", registry=registry) - message = str(err.value) - assert f"APPTAINERENV_PYTHONPATH={checkout}/src " in message - assert f"--cat-config {checkout}/cosmo_val/cat_config.yaml" in message - - SUFFIXES = st.lists( st.one_of(st.just("_leak_corr"), st.integers(0, 99999).map("_seed{:05d}".format)), max_size=3, @@ -175,78 +124,6 @@ def test_every_variant_shares_its_base(tmp_path_factory, entry, suffixes): assert cu.custody_of(cats, version, registry=registry) == base -def _reading(cats, path, *names): - """Point ``names``' entries at one shear file, spelled two ways.""" - for i, name in enumerate(names): - cats[name]["shear"]["path"] = path.name if i % 2 else str(path) - cats[name]["subdir"] = str(path.parent) - return cats - - -def test_catalogues_reading_one_file_share_one_blind(tmp_path): - """A blind conceals a shear file, whatever entry reads it: a catalogue - reading a concealed catalogue's file under any other custody is refused, - on either side, until the blind covers it too.""" - registry = tmp_path / "blinds" - seed = _write_blind(registry, "y3", ["TOY"]) - cats = _catalogues(TOY=None, TOY_A="unblinded", TOY_V={"base": "TOY"}) - _reading(cats, tmp_path / "toy.fits", "TOY", "TOY_A", "TOY_V") - for version in ("TOY_A", "TOY", "TOY_leak_corr"): - with pytest.raises(cu.CustodyError, match="not concealed alike") as err: - cu.custody_of(cats, version, registry=registry) - assert "TOY: blind y3" in str(err.value), version - assert "TOY_A: public" in str(err.value), version - - two = _reading( - _catalogues(TOY=None, TOY_B=None), tmp_path / "toy.fits", "TOY", "TOY_B" - ) - _write_blind(tmp_path / "two", "y3", ["TOY"]) - _write_blind(tmp_path / "two", "y4", ["TOY_B"]) - with pytest.raises(cu.CustodyError, match="TOY: blind y3; TOY_B: blind y4"): - cu.custody_of(two, "TOY", registry=tmp_path / "two") - - mock = _catalogues(OPEN="unblinded", OPEN_MOCK="mock") - _reading(mock, tmp_path / "open.fits", "OPEN", "OPEN_MOCK") - assert cu.custody_of(mock, "OPEN", registry=registry).status == "unblinded" - - del cats["TOY_A"]["blinding"] - (registry / "y3" / "bases").write_text("TOY\nTOY_A\n") - for version in ("TOY", "TOY_A", "TOY_V"): - assert cu.custody_of(cats, version, registry=registry).blind == "y3" - - (registry / "y3" / "revealed.json").write_text(json.dumps({"seed": seed})) - cats["TOY"]["blinding"] = "unblinded" - cats["TOY_A"]["blinding"] = "unblinded" - assert cu.custody_of(cats, "TOY_A", registry=registry).status == "unblinded" - - -def test_base_links_are_followed_and_checked(): - cats = _catalogues( - A="unblinded", B={"base": "A"}, C={"base": "B"}, D={"base": "nowhere"} - ) - assert cu.base_catalogue(cats, "C_leak_corr") == "A" - with pytest.raises(cu.CustodyError, match="nowhere"): - cu.base_catalogue(cats, "D") - loop = _catalogues(E={"base": "F"}, F={"base": "E"}) - with pytest.raises(cu.CustodyError, match="cycle"): - cu.base_catalogue(loop, "E") - with pytest.raises(cu.CustodyError, match="not a catalogue"): - cu.base_catalogue(cats, "paths") - - -def test_summary_is_one_line_per_base(tmp_path): - registry = tmp_path / "blinds" - _write_blind(registry, "y3", ["SP_v9"]) - cats = _catalogues(SP_v9=None, SP_v8="unblinded") - lines = cu.summary( - cats, ["SP_v9", "SP_v9_leak_corr", "SP_v8_leak_corr"], registry=registry - ) - assert lines == [ - "[custody] SP_v9 (+ SP_v9_leak_corr): blinded under y3", - "[custody] SP_v8 (+ SP_v8_leak_corr): unblinded", - ] - - def test_every_catalogue_in_the_repository_resolves(): """Each entry of the committed cat_config has a custody, from the repo registry.""" path = REPO / "cosmo_val" / "cat_config.yaml" @@ -261,7 +138,7 @@ def test_every_catalogue_in_the_repository_resolves(): # --------------------------------------------------------------------------- # -# I13: the host and a job resolve the same custody +# The host and a job resolve the same custody # --------------------------------------------------------------------------- # def _toy_checkout(tmp_path): """A checkout-shaped tree: workflow/common.py, src/, cosmo_val/{cat_config,blinds}.""" diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py index 8c7e4e6b..a49a5f76 100644 --- a/src/sp_validation/tests/test_custody_e2e.py +++ b/src/sp_validation/tests/test_custody_e2e.py @@ -1,8 +1,7 @@ """End to end: one blinded catalogue through the rule scripts, then its reveal. -A synthetic catalogue is declared three ways (``TOY`` blinded, ``TOY_OPEN`` -unblinded, ``TOY_MOCK`` mock). A blind is drawn for ``TOY`` with ``blinding -init``; the rule scripts run as Snakemake runs them (``runpy`` with a +A synthetic catalogue ``TOY`` is declared blinded, and its blind drawn with +``blinding init``; the rule scripts run as Snakemake runs them (``runpy`` with a ``snakemake`` object) for ``TOY`` and ``TOY_leak_corr``, on patch centres drawn by their command: both ξ± grids, the ξ± figures, pseudo-Cℓ on an nside-32 NaMaster workspace, ρ/τ, COSEBIs, pure-E/B and assembly. The blind @@ -12,7 +11,6 @@ """ import json -import os import re import runpy import types @@ -153,7 +151,7 @@ def patch_centres(cat_config, version, out, npatch): return centres -def run_chain(cat_config, version, out, cov, *, grid_parts=None): +def run_chain(cat_config, version, out, cov): """The cosmo_val chain for one version, into ``out``; returns the part paths.""" out.mkdir(parents=True, exist_ok=True) token = bd.declared_custody(cat_config, version).token @@ -178,9 +176,6 @@ def run_chain(cat_config, version, out, cov, *, grid_parts=None): "custody": token, }, ) - if grid_parts == "only": - return xi - cv = CosmologyValidation( versions=[version], catalog_config=str(cat_config), output_dir=str(out) ) @@ -368,7 +363,7 @@ def toy(tmp_path, monkeypatch): n_gal=20000, coherent_shear=True, with_psf=True, - catalogues={"TOY": None, "TOY_OPEN": "unblinded", "TOY_MOCK": "mock"}, + catalogues={"TOY": None}, ) (tmp_path / "fast.json").write_text( json.dumps({"theory": {"transfer_function": "eisenstein_hu"}}) @@ -395,10 +390,9 @@ def toy(tmp_path, monkeypatch): def test_a_blinded_catalogue_from_birth_to_audit(toy, monkeypatch): out = toy.root / "cosmo_val" blinded = bd.declared_custody(toy.cat_config, "TOY") - assert blinded.status == "blinded" - - # --- a blinded run ------------------------------------------------------- versions = ("TOY", "TOY_leak_corr") + + # --- a blinded run: every part concealed under the one blind ------------- for version in versions: run_chain(toy.cat_config, version, out, toy.cov) with monkeypatch.context() as m: @@ -406,142 +400,29 @@ def test_a_blinded_catalogue_from_birth_to_audit(toy, monkeypatch): plot_2pcf( toy.cat_config, out, {v: out / f"{v}_xi_reporting.sacc" for v in versions} ) - concealed = sio.xi_correlation(sio.load(out / "TOY_xi_reporting.sacc")) - assert any(np.array_equal(a, concealed.xip) for a in drawn) # drawn from the part born = stamps(out) assert len(born) == 2 * (2 + len(PARTS)), sorted(born) - assert all(s == blinded.stamp for s in born.values()), born # one commitment - assert ( - bd.main(["verify", str(out / "TOY.sacc"), "--cat-config", str(toy.cat_config)]) - == 0 - ) - - # A two-bin part, for the interruption below. - two_bin = sio.new_sacc( - { - 0: sio.get_nz(sio.load(out / "TOY_xi_reporting.sacc"), 0), - 1: sio.get_nz(sio.load(out / "TOY_xi_reporting.sacc"), 0), - }, - ) - theta = np.geomspace(5.0, 60.0, 6) - for pair in ((0, 0), (0, 1), (1, 1)): - sio.add_xi(two_bin, pair, theta, 1e-5 / theta, 1e-6 / theta, grid="tomo") - - # The true vector of TOY is TOY_OPEN's, the same galaxies declared unblinded. - open_parts = run_chain( - toy.cat_config, "TOY_OPEN", toy.root / "open", toy.cov, grid_parts="only" - ) - with pytest.raises(ValueError, match="stamps"): - assemble( - toy.cat_config, - "TOY", - toy.root / "swap", - { - "xi_reporting": open_parts["reporting"], - **{ - k: out / Path(v).name - for k, v in { - "pseudo_cl": "pseudo_cl_TOY.sacc", - "cosebis": "TOY_cosebis.sacc", - "pure_eb": "TOY_pure_eb.sacc", - "rho_tau": "rho_tau_TOY.sacc", - }.items() - }, - }, - toy.cov, - blinded.token, - ) - # Pure-E/B from parts under two custodies writes nothing at all. - mixed = pure_eb_outputs("TOY", toy.root / "mixed") - (toy.root / "mixed").mkdir() - with pytest.raises(ValueError, match="stamps"): - pure_eb( - "TOY", - mixed, - { - "reporting": open_parts["reporting"], - "integration": out / "TOY_xi_integration.sacc", - }, - toy.cov, - ) - assert not list((toy.root / "mixed").iterdir()) - - # --- an interrupted birth leaves no part --------------------------------- - def interrupted(*args, **kwargs): - raise RuntimeError("interrupted while sealing") - - with monkeypatch.context() as m: - m.setattr(bd, "conceal", interrupted) - with pytest.raises(RuntimeError, match="while sealing"): - run_chain( - toy.cat_config, - "TOY", - toy.root / "interrupted", - toy.cov, - grid_parts="only", - ) - assert not list((toy.root / "interrupted").glob("*.sacc")) - - calls = [] - real_factor = bd._factor - - def second_block_fails(*args, **kwargs): - calls.append(1) - if len(calls) == 2: - raise RuntimeError("interrupted inside conceal") - return real_factor(*args, **kwargs) - - with monkeypatch.context() as m: - m.setattr(bd, "_factor", second_block_fails) - with pytest.raises(RuntimeError, match="inside conceal"): - sio.save( - two_bin, toy.root / "interrupted" / "two_bin.sacc", custody=blinded - ) - assert not (toy.root / "interrupted" / "two_bin.sacc").exists() - - # --- the reveal: archive, flip the declaration, re-measure, audit -------- + assert all(s == blinded.stamp for s in born.values()), born blinded_run = { str(p.relative_to(toy.root)): p.read_bytes() for p in out.rglob("*") if p.is_file() } + + # --- the reveal: archive, flip the declaration, re-measure, audit -------- + cat_config = str(toy.cat_config) assert ( - bd.main( - ["reveal", "toy", "--root", str(out), "--cat-config", str(toy.cat_config)] - ) - == 0 + bd.main(["reveal", "toy", "--root", str(out), "--cat-config", cat_config]) == 0 ) archive = out / "revealed" / "toy" - assert sorted( - str(p.relative_to(archive)) for p in archive.rglob("*.sacc") - ) == sorted(born) - assert not list(out.glob("*.sacc")) - config = yaml.safe_load(toy.cat_config.read_text()) config["TOY"]["blinding"] = "unblinded" toy.cat_config.write_text(yaml.safe_dump(config, sort_keys=False)) - true = bd.declared_custody(toy.cat_config, "TOY") - assert true.token == "unblinded:TOY" - - for version in ("TOY", "TOY_leak_corr"): + for version in versions: run_chain(toy.cat_config, version, out, toy.cov) - assert all(s == true.stamp for s in stamps(out).values()) - report = bd.audit("toy", archive=archive, true_root=out, cat_config=toy.cat_config) assert report["ok"], json.dumps(report, indent=1, default=str) assert set(report["parts"]) == set(born) - assert ( - abs(report["shift"]["S8"]) <= 0.075 and abs(report["shift"]["Omega_m"]) <= 0.1 - ) - - revealed = toy.cat_config.parent / "blinds" / "toy" / "revealed.json" - published = revealed.read_text() - os.chmod(revealed, 0o644) - revealed.write_text(json.dumps({"seed": "not-the-seed"})) - assert not bd.audit( - "toy", archive=archive, true_root=out, cat_config=toy.cat_config - )["ok"] - revealed.write_text(published) # --- no true ξ± value was ever written or drawn by the blinded run ------- true_xi = np.concatenate( @@ -555,46 +436,3 @@ def second_block_fails(*args, **kwargs): figures = {"figures": np.concatenate(drawn).tobytes()} assert not plaintext({**blinded_run, **figures}, true_xi) assert not list(toy.root.rglob("*_xi_*.txt")) - - -def test_a_mock_never_opens_a_blind(toy, monkeypatch): - def refuse(custody): - raise AssertionError(f"a mock opened a blind: {custody}") - - monkeypatch.setattr(bd, "open_blind", refuse) - out = toy.root / "mock" - run_chain(toy.cat_config, "TOY_MOCK", out, toy.cov) - mock = bd.declared_custody(toy.cat_config, "TOY_MOCK") - found = stamps(out) - assert len(found) == 2 + len(PARTS) - assert all(s == mock.stamp for s in found.values()), found - - -@pytest.mark.parametrize( - "script", ["run_2pcf.py", "run_rho_tau.py", "assemble_sacc.py"] -) -def test_a_job_refuses_a_custody_other_than_its_params(toy, script): - """Snakemake resolved TOY unblinded; declared blinded when the job resolves - it, the job refuses before it measures or writes anything.""" - out = toy.root / "stale" - outputs = { - "sacc": out / "part.sacc", - "rho_stats": out / "rho.fits", - "tau_stats": out / "tau.fits", - "rho_tau": out / "rho_tau.sacc", - } - out.mkdir() - with pytest.raises(cu.CustodyError, match="differs between the job"): - run_rule( - script, - output={k: str(v) for k, v in outputs.items()}, - params={ - "ver": "TOY", - "version": "TOY", - **GRIDS["reporting"], - "cat_config": str(toy.cat_config), - "output_dir": str(out), - "custody": "unblinded:TOY", - }, - ) - assert not list(out.iterdir()) diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 400a71fd..64e51342 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -182,22 +182,13 @@ def test_a_launch_the_dag_cannot_honour_stops_with_the_fix(toy, tmp_path, case): assert "rule assemble_sacc" not in result.stdout -@pytest.mark.parametrize("named", [True, False], ids=["named", "unnamed"]) -def test_outputs_stay_in_the_output_roots(toy, forced, named): - """[P2] Nothing the suite declares lands outside the configured output - roots; a launch that names no COSMO_VAL writes into its own checkout's - cosmo_val/output.""" - if named: - cosmo_val, jobs = toy.cosmo_val, forced[1] - else: - env = {k: v for k, v in toy.env.items() if k != "COSMO_VAL"} - result = toy.snakemake("-n", "all", env=env) - assert result.returncode == 0, result.stdout - cosmo_val, jobs = toy.root / "cosmo_val" / "output", parse_jobs(result.stdout) +def test_outputs_stay_in_the_output_roots(toy, forced): + """[P2] Nothing the suite declares lands outside the configured output roots.""" roots = [ - r.resolve() for r in (cosmo_val, toy.cosmo_inference, toy.rundir / "results") + r.resolve() + for r in (toy.cosmo_val, toy.cosmo_inference, toy.rundir / "results") ] - outputs = [Path(o) for j in jobs for o in j.output] + outputs = [Path(o) for j in forced[1] for o in j.output] assert outputs strays = [ o @@ -207,47 +198,6 @@ def test_outputs_stay_in_the_output_roots(toy, forced, named): assert not strays, strays -@pytest.mark.parametrize("root", ["COSMO_VAL", "COSMO_INFERENCE"]) -def test_every_spelling_of_an_output_root_declares_the_same_paths(toy, tmp_path, root): - """Snakemake keys its persistence records by path string, so a symlinked - spelling of an output root declares the tree's resolved paths.""" - tree = Path(toy.env[root]).resolve() - link = tmp_path / "link" - link.symlink_to(tree, target_is_directory=True) - result = toy.snakemake("-n", "all", env={**toy.env, root: str(link)}) - assert result.returncode == 0, result.stdout - declared = [f for j in parse_jobs(result.stdout) for f in j.input + j.output] - assert any(f.startswith(f"{tree}/") for f in declared), declared - assert not [f for f in declared if f.startswith(f"{link}/")], declared - - -def test_output_roots_take_the_plain_spelling(toy, grids): - """A root given as /automnt//... is declared as //..., the one - spelling every node has, on any host: a job step re-derives the launch's - paths on its own node, and the node that owns a disk has neither - /automnt/ nor a / link to it, like the disk no host has here. - A file target named in the plain spelling resolves, and no declared path - lies under /automnt.""" - tree = Path("/n00data0/spv-dag-toy") # a dry-run creates nothing - env = { - **toy.env, - "COSMO_VAL": f"/automnt{tree}/val", - "COSMO_INFERENCE": f"/automnt{tree}/inference", - } - common = _load_module(toy.root / "workflow" / "common.py", "plain_common", env) - assert (common.COSMO_VAL, common.COSMO_INFERENCE) == ( - tree / "val", - tree / "inference", - ) - integration = toy.common.grid_binning(grids["integration"]) - target = tree / "val" / f"{VERSIONS[0]}_xi_{integration}.sacc" - result = toy.snakemake("-n", str(target), env=env) - assert result.returncode == 0, result.stdout - declared = [f for j in parse_jobs(result.stdout) for f in j.input + j.output] - assert str(target) in declared, declared - assert not [f for f in declared if f.startswith("/automnt/")], declared - - @pytest.mark.parametrize( "image, refused", [ @@ -274,145 +224,6 @@ def test_image_parity_is_checked_at_launch(toy, tmp_path, image, refused): assert "rule assemble_sacc" not in result.stdout -def test_launch_reads_the_image_under_home(toy, tmp_path): - """The launch finds your image under ~/.cache, whatever XDG_CACHE_HOME says. - - Jobs on other nodes run the image from the path the launching host - resolved, and a cluster's XDG_CACHE_HOME is often node-local. The image here - reports another Python, so reading it stops the launch. - """ - home = tmp_path / "home" - fake_image(home / ".cache" / "sp_validation" / "sandbox", python="3.13.1") - env = {k: v for k, v in toy.env.items() if not k.startswith("SPV_")} - env.update(HOME=str(home), XDG_CACHE_HOME=str(tmp_path / "node-local")) - result = toy.snakemake("-n", "assemble_sacc_all", container=False, env=env) - assert result.returncode != 0, result.stdout - assert "uv tool install --force --python 3.13 snakemake==" in result.stdout - - -def _apptainer_stub(tmp_path): - """A PATH entry that answers where apptainer is not installed. - - The candide profile deploys with apptainer, whose version Snakemake reads - even when it runs no job. - """ - apptainer = tmp_path / "bin" / "apptainer" - apptainer.parent.mkdir() - apptainer.write_text("#!/bin/sh\necho apptainer version 1.3.4\n") - apptainer.chmod(0o755) - return apptainer.parent - - -def test_a_job_needs_no_launch_cache(toy, tmp_path): - """A job starts on a node where the launch's XDG cache cannot exist. - - Jobs inherit the launching shell's environment, whose XDG_CACHE_HOME may be - node-local. Under the candide profile, a Snakemake that cannot create that - cache still starts, and a job's environment carries no XDG_CACHE_HOME for - the Snakemake its job step starts. - """ - blocker = tmp_path / "a-file" - blocker.touch() - candide = toy.root / "workflow" / "profiles" / "candide" - result = toy.snakemake( - "-n", - "--profile", - str(candide), - "assemble_sacc_all", - env=toy.env - | { - "XDG_CACHE_HOME": str(blocker / "cache"), - "PATH": f"{_apptainer_stub(tmp_path)}{os.pathsep}{toy.env['PATH']}", - }, - ) - assert result.returncode == 0, result.stdout - - snakefile = tmp_path / "Snakefile" - snakefile.write_text( - f"import sys\nsys.path.insert(0, {str(toy.root / 'workflow')!r})\n" - "import common\n\n" - 'rule job:\n output: "env.txt"\n' - ' shell: "printenv XDG_CACHE_HOME > {output} || true"\n' - ) - result = toy.snakemake( - "-s", - str(snakefile), - "--directory", - str(tmp_path), - container=False, - env=toy.env | {"XDG_CACHE_HOME": str(tmp_path / "launch-cache")}, - ) - assert result.returncode == 0, result.stdout - assert (tmp_path / "env.txt").read_text() == "" - - -def test_the_candide_profile_bounds_its_jobs(tmp_path): - """A real launch through the candide profile needs no --jobs. - - Snakemake refuses a real run on a remote executor without a job bound. The - target is up to date, so the launch submits nothing and runs on any host; - the same launch through the profile stripped of its bound shows the refusal. - """ - (tmp_path / "Snakefile").write_text( - 'rule done:\n output: "done.txt"\n shell: "touch {output}"\n' - ) - (tmp_path / "done.txt").touch() - env = {k: v for k, v in os.environ.items() if k != "SNAKEMAKE_PROFILE"} - env["PATH"] = f"{_apptainer_stub(tmp_path)}{os.pathsep}{env['PATH']}" - candide = REPO / "workflow" / "profiles" / "candide" - unbounded = tmp_path / "unbounded" - unbounded.mkdir() - profile = yaml.safe_load((candide / "config.yaml").read_text()) - profile.pop("jobs", None) - (unbounded / "config.yaml").write_text(yaml.safe_dump(profile)) - - def launch(profile_dir): - return subprocess.run( - [sys.executable, "-m", "snakemake", "--profile", str(profile_dir)], - cwd=tmp_path, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=300, - check=False, - ) - - result = launch(candide) - assert result.returncode == 0, result.stdout - assert "Nothing to be done" in result.stdout, result.stdout - refused = launch(unbounded) - assert refused.returncode != 0 and "--jobs" in refused.stdout, refused.stdout - - -def test_image_sims_checks_parity_at_launch(toy, tmp_path): - """The standalone image-sims workflow stops on a mismatched image too.""" - run = { - "image_sims": { - "sif": str(fake_image(tmp_path / "image", python="3.13.1")), - "grids_base": str(tmp_path / "grids"), - "mask_config": "mask.yaml", - "match_radius_deg": 0.0002, - "w_cols": ["none"], - "pair_match": True, - "n_bootstrap": 1, - "bootstrap_seed": 0, - } - } - (tmp_path / "run.yaml").write_text(yaml.safe_dump(run)) - result = toy.snakemake( - "-n", - "-s", - "workflow/image_sims/Snakefile", - "--configfile", - str(tmp_path / "run.yaml"), - container=False, - cwd=toy.root, - ) - assert result.returncode != 0, result.stdout - assert "uv tool install --force --python 3.13 snakemake==" in result.stdout - - def _stand_in_centres(paper, cosmo_val): """Touch the centres ``paper``'s patched ξ± grids read; a dry-run reads none.""" common = _load_module( From 9092b1fa056aec8b12f683fcea829aad9fca224d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 04:21:01 +0200 Subject: [PATCH 116/160] tests: cut to the custody core, one blinded pass end to end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The end-to-end test runs one blinded pass through the rule scripts (init, measure, assemble, verify, a no-plaintext byte scan, reveal, audit) on the reporting grid alone. Blinding keeps the seal-table and derivation-stamp properties, the unstamped-file refusal, the seed never on disk, one digest test, one wrong-key refusal, and one passing and one wrong-seed audit. Custody keeps its table, the variant property, one twins case and host/job agreement; the DAG keeps no-blind-rule, no ξ± text, patch centres, custody refusals, output roots and parity. Removed: the CAMB/CCL recipe check, the committed-blind fixture, the candide-only toy run and sweep-driver image test, the thread-count and persisted-centres cosmo_val tests, the DAG assembly-inputs test and the missing-centres launch case. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- .../tests/data/blinds/committed/bases | 1 - .../data/blinds/committed/commitment.json | 35 -- .../tests/data/blinds/committed/key | 1 - .../tests/data/blinds/committed/seed.fernet | 1 - src/sp_validation/tests/test_blinding.py | 56 +-- .../tests/test_blinding_recipe.py | 24 - src/sp_validation/tests/test_cosmo_val.py | 66 --- src/sp_validation/tests/test_custody.py | 16 + src/sp_validation/tests/test_custody_e2e.py | 431 +++--------------- workflow/tests/conftest.py | 17 +- workflow/tests/test_dag.py | 59 +-- workflow/tests/test_image_resolution.py | 36 -- workflow/tests/test_toy_run.py | 204 --------- 13 files changed, 115 insertions(+), 832 deletions(-) delete mode 100644 src/sp_validation/tests/data/blinds/committed/bases delete mode 100644 src/sp_validation/tests/data/blinds/committed/commitment.json delete mode 100644 src/sp_validation/tests/data/blinds/committed/key delete mode 100644 src/sp_validation/tests/data/blinds/committed/seed.fernet delete mode 100644 src/sp_validation/tests/test_blinding_recipe.py delete mode 100644 workflow/tests/test_image_resolution.py delete mode 100644 workflow/tests/test_toy_run.py diff --git a/src/sp_validation/tests/data/blinds/committed/bases b/src/sp_validation/tests/data/blinds/committed/bases deleted file mode 100644 index e20be6bb..00000000 --- a/src/sp_validation/tests/data/blinds/committed/bases +++ /dev/null @@ -1 +0,0 @@ -COMMITTED diff --git a/src/sp_validation/tests/data/blinds/committed/commitment.json b/src/sp_validation/tests/data/blinds/committed/commitment.json deleted file mode 100644 index 35b1f7ad..00000000 --- a/src/sp_validation/tests/data/blinds/committed/commitment.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "blind": "committed", - "seed_commitment": "810515ec5aa95a3a2445c949f8084f1280e25a21579ed7071562682fd87b8068", - "config": { - "envelope": { - "S8": 0.075, - "Omega_m": 0.1 - }, - "theory": { - "S8": 0.8, - "Omega_m": 0.3, - "Omega_b": 0.0469, - "h": 0.7, - "n_s": 0.96, - "m_nu": 0.06, - "w0": -1.0, - "wa": 0.0, - "mass_split": "normal", - "transfer_function": "boltzmann_camb", - "halofit_version": "mead2020_feedback", - "hmcode_logT_AGN": 7.5, - "ia_bias": 0.0, - "ia_z_piv": 0.62, - "ia_alphaz": 0.0 - } - }, - "config_digest": "9c0de8766e8947da70d4bf0a936436acae3bdb4c6b5ee304cd3373d1a7a214de", - "draw_scheme": 2, - "theory_stack": { - "pyccl": "3.3.4", - "camb": "1.6.6", - "smokescreen": "1.5.6" - }, - "created": "2026-09-26T09:16:50+00:00" -} diff --git a/src/sp_validation/tests/data/blinds/committed/key b/src/sp_validation/tests/data/blinds/committed/key deleted file mode 100644 index 43fc5df6..00000000 --- a/src/sp_validation/tests/data/blinds/committed/key +++ /dev/null @@ -1 +0,0 @@ -C7iIarmGreIIF8_Xs2kxQ9ofySIhSQn2UfYUI30KvII= \ No newline at end of file diff --git a/src/sp_validation/tests/data/blinds/committed/seed.fernet b/src/sp_validation/tests/data/blinds/committed/seed.fernet deleted file mode 100644 index 81f5403b..00000000 --- a/src/sp_validation/tests/data/blinds/committed/seed.fernet +++ /dev/null @@ -1 +0,0 @@ -gAAAAABqt42C0vXv4_tLg74hQ_3vQRitvL_ZzRGOZ_SxTJBbJk7PfIH10iKeNTocfyIblDU162039lrm-m_Tcv0vLvOpjhRFMq7lyc-OnAM1S4BaeRjv7M84r3DZJqvpO_Cwk85CjLlqxtXqjOormHMfGX7tZS93eH00yKgLZ-8S5pGjre9dJ8hN5AlQHAiyAmB_bDzaMQVRBR0DCK-G8WYSizlg-QZk7pgKNfxVyWy6OZk8K9xn_dxAXkSPQK_ROyJgZtDr3zf65rx-Xt3O-3z9kpAXzPSXEdUeYEotmqOxXYKiSanACAk= \ No newline at end of file diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 3e98ead3..327890a6 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -7,7 +7,6 @@ """ import json -from pathlib import Path import numpy as np import pytest @@ -19,7 +18,6 @@ from sp_validation import custody as cu from sp_validation import sacc_io as sio -DATA = Path(__file__).parent / "data" VERSIONS = ("TOY", "OTHER", "TOY_OPEN", "OTHER_OPEN", "TOY_MOCK") @@ -273,47 +271,37 @@ def test_the_commitment_hides_the_rng_seed(): assert int(cu.seed_commitment(seed)[:16], 16) != _normalize_seed(seed) -def test_the_committed_blind_opens_under_this_code(): - """A TheoryConfig field added without its value in ``blinding.NEUTRAL`` - would strand every live blind; ``tests/data/blinds/committed`` turns that - red.""" - blind = bd._open(DATA / "blinds", "committed") - assert (blind.hidden.S8, blind.hidden.Omega_m) == pytest.approx( - (0.8374952413635888, 0.32730573347638175), rel=1e-12 - ) +def test_the_digest_binds_the_config(): + config = bd.BlindingConfig() + moved = bd.BlindingConfig(envelope={**config.envelope, "S8": 0.08}) + assert moved.digest() != config.digest() + + +def test_a_blind_opens_only_with_its_key(tmp_path): + from cryptography.fernet import Fernet + + root = _registry(tmp_path, ("toy", "TOY")) + key = root / "blinds" / "toy" / "key" + key.chmod(0o644) + key.write_bytes(Fernet.generate_key()) + with pytest.raises(cu.CustodyError, match="does not decrypt"): + bd.open_blind(_custody(root, "TOY")) # --------------------------------------------------------------------------- # # The reveal: publish, then prove blinded − true = shift(seed) # --------------------------------------------------------------------------- # -def test_reveal_refuses_a_root_holding_nothing_concealed(tmp_path): - """Publishing cannot be undone: a mistyped ``--root`` is refused before - the seed is published.""" - root = _registry(tmp_path, ("toy", "TOY")) - (root / "out").mkdir() - sio.save(part(cl=False), root / "out" / "part.sacc", custody=_custody(root, "TOY")) - cat_config = root / "cat_config.yaml" - with pytest.raises(cu.CustodyError, match="nothing concealed"): - bd.reveal("toy", root=root / "outptu", cat_config=cat_config) - assert not (root / "blinds" / "toy" / "revealed.json").exists() - archive = bd.reveal("toy", root=root / "out", cat_config=cat_config) - assert (archive / "part.sacc").exists() - - -@pytest.mark.parametrize("left_true", [False, True], ids=["revealed", "mixed"]) -def test_the_audit_proves_the_shift(tmp_path, left_true): - """An archived part passes the audit against its re-measured twin only if - every shiftable row carries the shift: a pair left true fails.""" +@pytest.mark.parametrize("seed", ["committed", "wrong"]) +def test_the_audit_proves_the_shift(tmp_path, seed): + """A concealed part passes the audit against its re-measured twin under + the published committed seed, and fails under any other.""" root = _registry(tmp_path, ("toy", "TOY")) blinded = _custody(root, "TOY") true = part() archived = sio.seal(true, blinded) - if left_true: - for i, dp in enumerate(true.data): - if dp.tracers == sio._pair((1, 1)) and dp.data_type in sio.SHIFTABLE: - archived.data[i].value = dp.value + published = bd.open_blind(blinded).seed if seed == "committed" else "not-it" (root / "blinds" / "toy" / "revealed.json").write_text( - json.dumps({"seed": bd.open_blind(blinded).seed}) + json.dumps({"seed": published}) ) for tree, s, stamp in ( ("archive", archived, blinded.stamp), @@ -329,4 +317,4 @@ def test_the_audit_proves_the_shift(tmp_path, left_true): true_root=root / "live", cat_config=root / "cat_config.yaml", ) - assert report["ok"] != left_true, report + assert report["ok"] == (seed == "committed"), report diff --git a/src/sp_validation/tests/test_blinding_recipe.py b/src/sp_validation/tests/test_blinding_recipe.py deleted file mode 100644 index b04245bb..00000000 --- a/src/sp_validation/tests/test_blinding_recipe.py +++ /dev/null @@ -1,24 +0,0 @@ -"""The blinding theory's nonlinear recipe is the inference pipeline's. - -The blinding shift is a difference of CCL theory vectors; inference runs CAMB -through CosmoSIS. The shift means what it is meant to only if both use one -recipe, so the blinding reads it from the CosmoSIS config it must match. -""" - -import pathlib -import re - -from sp_validation import blinding_theory as bt - -INI = ( - pathlib.Path(__file__).resolve().parents[3] - / "cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia_cell.ini" -) - - -def test_halofit_recipe_matches_inference_config(): - match = re.search(r"^halofit_version\s*=\s*(\S+)", INI.read_text(), re.MULTILINE) - assert match, f"no halofit_version in {INI}" - cfg = bt.TheoryConfig() - assert cfg.halofit_version == match.group(1) - assert cfg.transfer_function == "boltzmann_camb" diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 3c994989..500999b3 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -310,72 +310,6 @@ def test_calculate_2pcf_runs_on_synthetic_catalog(self, tmp_path): # The additive-bias subtraction in the pipeline must have run. assert version in cv.c1 and version in cv.c2 - def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): - """calculate_2pcf's ξ± is the same on 4 and on 48 TreeCorr threads. - - Production binning (default bin_slop/angle_slop), both runs on the - persisted patch centres, each from a fresh Catalog; they must agree to - far below the jackknife σ. - """ - import treecorr - - params, version = write_synthetic_catalogs( - tmp_path, n_gal=4000, coherent_shear=True - ) - cv = CosmologyValidation( - versions=[version], - npatch=8, - theta_min=15.0, - theta_max=70.0, - nbins=6, - **params, - ) - cv.write_patch_centers(version, 8) - - xi = {} - for n_threads in (4, 48): - gg = sacc_io.xi_correlation( - cv.calculate_2pcf(version, num_threads=n_threads) - ) - assert treecorr.get_omp_threads() == n_threads # the count took effect - xi[n_threads] = np.concatenate([gg.xip, gg.xim]) - sigma = np.sqrt(np.concatenate([gg.varxip, gg.varxim])) - - shift = np.max(np.abs(xi[48] - xi[4]) / sigma) - assert shift < 1e-6, f"ξ± moves by {shift:.3g}σ between 4 and 48 threads" - - def test_a_patched_measurement_splits_at_the_persisted_centres(self, tmp_path): - """[patch-centres-are-inputs] With patches, calculate_2pcf splits at the - base catalogue's centres file and names it in the part. Without the - file it refuses, naming the command that draws it, which works from an - object holding only a variant and never replaces the file.""" - import hashlib - import shlex - - from sp_validation.cosmo_val import patch_centers - - params, base = write_synthetic_catalogs(tmp_path) - version = f"{base}_leak_corr" - cv = CosmologyValidation(versions=[version], npatch=4, **params) - with pytest.raises(FileNotFoundError) as refused: - cv.calculate_2pcf(version) - command = "python -m sp_validation.cosmo_val.patch_centers " - assert command in str(refused.value), refused.value - argv = shlex.split(str(refused.value).split(command, 1)[1]) - - patch_centers.main(argv) - centres = Path(cv.patch_centers_path(version, 4)) - drawn = centres.read_bytes() - with pytest.raises(FileExistsError): - patch_centers.main(argv) - assert centres.read_bytes() == drawn - - part = cv.calculate_2pcf(version) - assert part.metadata["npatch"] == 4 - assert ( - part.metadata["patch_centers_sha256"] == hashlib.sha256(drawn).hexdigest() - ) - def test_calculate_scale_dependent_leakage_runs_on_synthetic_catalog( self, tmp_path ): diff --git a/src/sp_validation/tests/test_custody.py b/src/sp_validation/tests/test_custody.py index 99568b13..65049bbb 100644 --- a/src/sp_validation/tests/test_custody.py +++ b/src/sp_validation/tests/test_custody.py @@ -124,6 +124,22 @@ def test_every_variant_shares_its_base(tmp_path_factory, entry, suffixes): assert cu.custody_of(cats, version, registry=registry) == base +def test_catalogues_reading_one_file_share_one_blind(tmp_path): + """A blind conceals a shear file, whatever entry reads it: an entry + reading a concealed catalogue's file under another custody is refused + until the blind covers it too.""" + registry = tmp_path / "blinds" + _write_blind(registry, "y3", ["TOY"]) + cats = _catalogues(TOY=None, TWIN="unblinded") + cats["TWIN"]["shear"]["path"] = cats["TOY"]["shear"]["path"] + for version in ("TOY", "TWIN"): + with pytest.raises(cu.CustodyError, match="not concealed alike"): + cu.custody_of(cats, version, registry=registry) + del cats["TWIN"]["blinding"] + (registry / "y3" / "bases").write_text("TOY\nTWIN\n") + assert cu.custody_of(cats, "TWIN", registry=registry).blind == "y3" + + def test_every_catalogue_in_the_repository_resolves(): """Each entry of the committed cat_config has a custody, from the repo registry.""" path = REPO / "cosmo_val" / "cat_config.yaml" diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py index a49a5f76..42454963 100644 --- a/src/sp_validation/tests/test_custody_e2e.py +++ b/src/sp_validation/tests/test_custody_e2e.py @@ -1,13 +1,11 @@ -"""End to end: one blinded catalogue through the rule scripts, then its reveal. - -A synthetic catalogue ``TOY`` is declared blinded, and its blind drawn with -``blinding init``; the rule scripts run as Snakemake runs them (``runpy`` with a -``snakemake`` object) for ``TOY`` and ``TOY_leak_corr``, on patch centres drawn -by their command: both ξ± grids, the ξ± figures, pseudo-Cℓ on -an nside-32 NaMaster workspace, ρ/τ, COSEBIs, pure-E/B and assembly. The blind -is then revealed, the declaration flipped, the chain re-run, and the audit must -prove blinded − true = shift(seed) on every part the reveal archived. Neither -the blinded run's files nor its figures may hold a true ξ± value. +"""End to end: one blinded catalogue through its rule scripts, then its reveal. + +A synthetic catalogue ``TOY`` is declared blinded and its blind drawn with +``blinding init``. The ξ± and assembly rule scripts run as Snakemake runs them +(``runpy`` with a ``snakemake`` object); the assembled file verifies against +the declaration, and no file of the blinded run holds a true ξ± value. The +blind is then revealed, the declaration flipped, the chain re-run, and the +audit proves blinded − true = shift(seed). """ import json @@ -17,25 +15,15 @@ from pathlib import Path import numpy as np -import pytest import yaml from _synthetic import write_synthetic_catalogs from sp_validation import blinding as bd from sp_validation import custody as cu from sp_validation import sacc_io as sio -from sp_validation.cosmo_val import CosmologyValidation -from sp_validation.cosmo_val.patch_centers import main as draw_patch_centers - -REPO = Path(__file__).resolve().parents[3] -SCRIPTS = REPO / "workflow" / "scripts" -GRIDS = { - "reporting": {"min_sep": 5.0, "max_sep": 60.0, "nbins": 6, "npatch": 4}, - "integration": {"min_sep": 1.0, "max_sep": 150.0, "nbins": 300, "npatch": 1}, -} -SCALE_CUT = [12.0, 60.0] -PARTS = ("xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau") +SCRIPTS = Path(__file__).resolve().parents[3] / "workflow" / "scripts" +GRID = {"min_sep": 5.0, "max_sep": 60.0, "nbins": 6, "npatch": 1} class Named(list): @@ -58,381 +46,104 @@ def get(self, key, default=None): return self._items.get(key, default) -def run_rule(script, *, input=None, output=None, params=None): - """Run a rule script as Snakemake runs a job: outputs removed, then ``script:``.""" - for path in (output or {}).values(): - Path(path).unlink(missing_ok=True) - smk = types.SimpleNamespace( - input=Named(**(input or {})), - output=Named(**(output or {})), - params=Named(**(params or {})), - ) +def run_rule(script, **fields): + smk = types.SimpleNamespace(**{k: Named(**v) for k, v in fields.items()}) runpy.run_path( str(SCRIPTS / script), init_globals={"snakemake": smk}, run_name="__main__" ) -def _namaster_part_inputs(): - """``(ell_eff, cl_all, workspace)`` on an nside-32 full-sky workspace.""" - import healpy as hp - import pymaster as nmt - - nside = 32 - npix = hp.nside2npix(nside) - field = nmt.NmtField(np.ones(npix), [np.zeros(npix), np.zeros(npix)], spin=2) - binning = nmt.NmtBin.from_nside_linear(nside, 8) - workspace = nmt.NmtWorkspace() - workspace.compute_coupling_matrix(field, field, binning) - ell = binning.get_effective_ells() - ee = 1e-8 * (ell / 30.0) ** -1.2 - return ell, np.array([ee, 0.01 * ee, 0.01 * ee, 0.02 * ee]), workspace - - -def _rho_tau_handlers(): - rng = np.random.default_rng(5) - theta = np.geomspace(5.0, 60.0, 6) - rho, tau = {"theta": theta}, {"theta": theta} - for k in range(6): - for sfx in ("p", "m"): - rho[f"rho_{k}_{sfx}"] = rng.normal(size=6) * 1e-6 - rho[f"varrho_{k}_{sfx}"] = rng.uniform(1e-14, 1e-13, 6) - for k in (0, 2, 5): - for sfx in ("p", "m"): - tau[f"tau_{k}_{sfx}"] = rng.normal(size=6) * 1e-6 - tau[f"vartau_{k}_{sfx}"] = rng.uniform(1e-14, 1e-13, 6) - return types.SimpleNamespace(rho_stats=rho), types.SimpleNamespace(tau_stats=tau) - - -def _covariances(root): - """ξ± shape-noise covariances on both grids, and a NaMaster Cℓ FITS. - - The ξ± covariances are the diagonal shape noise of the synthetic catalogue - (its density and per-component dispersion), in CosmoCov's text format. - """ - from astropy.io import fits - - from sp_validation.b_modes import log_bin_edges - - density, sigma_e, area = 20000 / (4.0 * 60) ** 2, 0.05, (4.0 * 60) ** 2 - paths = {} - for grid, b in GRIDS.items(): - left, right = log_bin_edges(b["min_sep"], b["max_sep"], b["nbins"]) - pairs = np.pi * area * density**2 * (right**2 - left**2) / 2.0 - var = 2.0 * sigma_e**4 / pairs - paths[grid] = root / f"cov_{grid}.txt" - np.savetxt(paths[grid], np.diag(np.concatenate([var, var]))) - nbp = len(_namaster_part_inputs()[0]) - paths["pseudo_cl"] = root / "pseudo_cl_cov.fits" - fits.HDUList( - [fits.PrimaryHDU()] - + [ - fits.ImageHDU(np.eye(nbp) * 1e-20, name=name) - for name in ("COVAR_EE_EE", "COVAR_BB_BB", "COVAR_EB_EB") - ] - ).writeto(paths["pseudo_cl"], overwrite=True) - return paths - - -def patch_centres(cat_config, version, out, npatch): - """``version``'s base catalogue's centres, drawn by their command if absent.""" - base = cu.base_catalogue(yaml.safe_load(Path(cat_config).read_text()), version) - centres = out / "patches" / f"{base}_npatch={npatch}.dat" - if not centres.exists(): - draw_patch_centers( - [ - base, - str(npatch), - "--cat-config", - str(cat_config), - "--output-dir", - str(out), - ] - ) - return centres - - -def run_chain(cat_config, version, out, cov): - """The cosmo_val chain for one version, into ``out``; returns the part paths.""" - out.mkdir(parents=True, exist_ok=True) - token = bd.declared_custody(cat_config, version).token - xi = {} - for grid, b in GRIDS.items(): - xi[grid] = out / f"{version}_xi_{grid}.sacc" - patches = ( - {"patches": str(patch_centres(cat_config, version, out, b["npatch"]))} - if b["npatch"] > 1 - else {} - ) - run_rule( - "run_2pcf.py", - input=patches, - output={"sacc": str(xi[grid])}, - params={ - "ver": version, - **b, - "cat_config": str(cat_config), - "output_dir": str(out), - "grid": grid, - "custody": token, - }, - ) - cv = CosmologyValidation( - versions=[version], catalog_config=str(cat_config), output_dir=str(out) - ) - paths = { - "xi_reporting": xi["reporting"], - "pseudo_cl": out / f"pseudo_cl_{version}.sacc", - "cosebis": out / f"{version}_cosebis.sacc", - "pure_eb": out / f"{version}_pure_eb.sacc", - "rho_tau": out / f"rho_tau_{version}.sacc", - } - cv.pseudo_cl_to_sacc_part( - version, str(paths["pseudo_cl"]), *_namaster_part_inputs() - ) - cv.rho_tau_to_sacc_part(version, str(out), version, *_rho_tau_handlers()) - - integ = GRIDS["integration"] - run_rule( - "cv_cosebis.py", - input={"xi": str(xi["integration"]), "cov": str(cov["integration"])}, - output={ - "npz": str(out / f"{version}_cosebis.npz"), - "sacc": str(paths["cosebis"]), - "figure_modes": str(out / f"{version}_cosebis_modes.png"), - "figure_covariance": str(out / f"{version}_cosebis_cov.png"), - "figure_scalecut_ptes": str(out / f"{version}_cosebis_ptes.png"), - }, - params={ - "version": version, - "min_sep": integ["min_sep"], - "max_sep": integ["max_sep"], - "nbins": integ["nbins"], - "nmodes": 5, - "scale_cuts": [SCALE_CUT], - "fiducial_scale_cut": SCALE_CUT, - }, - ) - pure_eb(version, pure_eb_outputs(version, out), xi, cov) - assemble(cat_config, version, out, paths, cov, token) - return paths - - -def pure_eb_outputs(version, out): - return { - "npz": str(out / f"{version}_pure_eb.npz"), - "sacc": str(out / f"{version}_pure_eb.sacc"), - "figure_integration_vs_reporting": str(out / f"{version}_eb_ivr.png"), - "figure_xis": str(out / f"{version}_eb_xis.png"), - "figure_ptes": str(out / f"{version}_eb_ptes.png"), - "figure_covariance": str(out / f"{version}_eb_cov.png"), - } - - -def pure_eb(version, output, xi, cov): - """Rule cv_pure_eb for ``version`` from the ξ± parts ``xi`` (by grid).""" - rep = GRIDS["reporting"] +def run_chain(cat_config, out, cov): + """Rules xi and assemble_sacc for TOY into ``out``.""" + out.mkdir(exist_ok=True) + token = bd.declared_custody(cat_config, "TOY").token + common = {"cat_config": str(cat_config), "custody": token} + xi = out / "TOY_xi_reporting.sacc" run_rule( - "cv_pure_eb.py", - input={ - "xi_reporting": str(xi["reporting"]), - "xi_integration": str(xi["integration"]), - "cov_integration": str(cov["integration"]), - }, - output=output, - params={ - "version": version, - "min_sep": rep["min_sep"], - "max_sep": rep["max_sep"], - "nbins": rep["nbins"], - # Enough draws for a full-rank 12 × 12 combined ξ±_B covariance. - "n_samples": 40, - "cosmo_params": {"transfer_function": "eisenstein_hu"}, - "fiducial_scale_cut": SCALE_CUT, - }, + "run_2pcf.py", + input={}, + output={"sacc": str(xi)}, + params={"ver": "TOY", **GRID, "output_dir": str(out), "grid": "reporting"} + | common, ) - - -def assemble(cat_config, version, out, paths, cov, token): run_rule( "assemble_sacc.py", - input={ - **{k: str(v) for k, v in paths.items()}, - "xi_cov": str(cov["reporting"]), - "pseudo_cl_cov": str(cov["pseudo_cl"]), - }, - output={"sacc": str(out / f"{version}.sacc")}, - params={ - "version": version, - "expected": list(PARTS), - "custody": token, - "cat_config": str(cat_config), - }, + input={"xi_reporting": str(xi), "xi_cov": str(cov)}, + output={"sacc": str(out / "TOY.sacc")}, + params={"version": "TOY", "expected": ["xi_reporting"]} | common, ) -def plot_2pcf(cat_config, out, xi): - """Rule cv_plot_2pcf over the reporting parts ``xi`` ({version: path}).""" - run_rule( - "cv_plot_2pcf.py", - input={"xi": [str(path) for path in xi.values()]}, - output={"sentinel": str(out / "plot_2pcf.done")}, - params={ - "cv_init": { - "versions": list(xi), - "catalog_config": str(cat_config), - "output_dir": str(out), - } - }, - ) - - -def recording_figures(monkeypatch): - """Record every array a figure draws; returns the list it fills.""" - import matplotlib.axes - - drawn = [] - for name in ("plot", "errorbar"): - - def draw(self, *args, _draw=getattr(matplotlib.axes.Axes, name), **kwargs): - drawn.extend(np.asarray(a, float) for a in args if np.ndim(a) == 1) - return _draw(self, *args, **kwargs) - - monkeypatch.setattr(matplotlib.axes.Axes, name, draw) - return drawn - - -def _near(x, values, rtol): - """Whether any finite, nonzero ``x`` lies within ``rtol`` of a sorted ``values``.""" - keep = np.isfinite(x) & (x != 0) - x, rtol = x[keep], np.broadcast_to(rtol, keep.shape)[keep] - i = np.clip(np.searchsorted(values, x), 1, len(values) - 1) - gap = np.minimum(np.abs(x - values[i - 1]), np.abs(x - values[i])) - return bool(np.any(gap <= rtol * np.abs(x))) - - def plaintext(blobs, values): - """Where the bytes of ``blobs`` ({name: bytes}) hold any of ``values``. - - A value is found as a float64 of either byte order at any offset, to the - float noise of a re-measurement, or as a number written in exponent - notation, to half a unit of its last digit. - """ + """Names of ``blobs`` ({name: bytes}) holding any of ``values``, as float64 + of either byte order at any offset, or as text in exponent notation.""" values = np.sort(np.asarray(values, float)) + + def near(x, rtol): + keep = np.isfinite(x) & (x != 0) + x, rtol = x[keep], np.broadcast_to(rtol, keep.shape)[keep] + i = np.clip(np.searchsorted(values, x), 1, len(values) - 1) + gap = np.minimum(np.abs(x - values[i - 1]), np.abs(x - values[i])) + return bool(np.any(gap <= rtol * np.abs(x))) + found = [] for name, blob in blobs.items(): for order in "<>": for offset in range(8): count = (len(blob) - offset) // 8 - if count > 0 and _near( - np.frombuffer(blob, f"{order}f8", count, offset), values, 1e-9 + if count > 0 and near( + np.frombuffer(blob, f"{order}f8", count, offset), 1e-9 ): - found.append(f"{name}: float64 {order} at offset {offset}") + found.append(name) tokens = re.findall(rb"(-?\d\.(\d+)e[-+]\d+)", blob) - if tokens and _near( - np.array([float(t) for t, _ in tokens]), - values, - np.array([0.51 * 10.0 ** -len(digits) for _, digits in tokens]), - ): - found.append(f"{name}: as text") + digits = np.array([0.51 * 10.0 ** -len(d) for _, d in tokens]) + if tokens and near(np.array([float(t) for t, _ in tokens]), digits): + found.append(name) return found -def stamps(root): - """``{relative path: stamp}`` of every SACC under ``root``.""" - return { - str(p.relative_to(root)): cu.read_stamp(sio.load(p).metadata).stamp - for p in sorted(root.rglob("*.sacc")) - if "revealed" not in p.relative_to(root).parts - } - - -@pytest.fixture -def toy(tmp_path, monkeypatch): - """The catalogue config, a blind for TOY, and the covariances the rules read.""" +def test_a_blinded_catalogue_from_birth_to_audit(tmp_path, monkeypatch): monkeypatch.syspath_prepend(str(SCRIPTS)) import cv_runner - import matplotlib # Rule jobs line-buffer their own streams; under pytest they are captured. monkeypatch.setattr(cv_runner, "_unbuffer_streams", lambda: None) - # The figures render with matplotlib's own text engine, whatever LaTeX - # setup the invoking user's matplotlibrc asks for. - monkeypatch.setitem(matplotlib.rcParams, "text.usetex", False) params, _ = write_synthetic_catalogs( - tmp_path, - n_gal=20000, - coherent_shear=True, - with_psf=True, - catalogues={"TOY": None}, - ) - (tmp_path / "fast.json").write_text( - json.dumps({"theory": {"transfer_function": "eisenstein_hu"}}) + tmp_path, n_gal=20000, coherent_shear=True, catalogues={"TOY": None} ) cat_config = Path(params["catalog_config"]) - # A fixed seed, so every run conceals under the same hidden point. + fast = tmp_path / "fast.json" + fast.write_text(json.dumps({"theory": {"transfer_function": "eisenstein_hu"}})) monkeypatch.setattr(bd.secrets, "token_hex", lambda n: "e2e-seed") bd.main( - [ - "init", - "toy", - "TOY", - "--cat-config", - str(cat_config), - "--config", - str(tmp_path / "fast.json"), - ] - ) - return types.SimpleNamespace( - root=tmp_path, cat_config=cat_config, cov=_covariances(tmp_path) + ["init", "toy", "TOY", "--cat-config", str(cat_config), "--config", str(fast)] ) - - -def test_a_blinded_catalogue_from_birth_to_audit(toy, monkeypatch): - out = toy.root / "cosmo_val" - blinded = bd.declared_custody(toy.cat_config, "TOY") - versions = ("TOY", "TOY_leak_corr") - - # --- a blinded run: every part concealed under the one blind ------------- - for version in versions: - run_chain(toy.cat_config, version, out, toy.cov) - with monkeypatch.context() as m: - drawn = recording_figures(m) - plot_2pcf( - toy.cat_config, out, {v: out / f"{v}_xi_reporting.sacc" for v in versions} - ) - born = stamps(out) - assert len(born) == 2 * (2 + len(PARTS)), sorted(born) - assert all(s == blinded.stamp for s in born.values()), born - blinded_run = { - str(p.relative_to(toy.root)): p.read_bytes() - for p in out.rglob("*") - if p.is_file() - } + cov = tmp_path / "cov.txt" + np.savetxt(cov, np.diag(np.full(2 * GRID["nbins"], 1e-10))) + out = tmp_path / "cosmo_val" + + # --- the blinded run ----------------------------------------------------- + run_chain(cat_config, out, cov) + blinded = bd.declared_custody(cat_config, "TOY") + for part in out.glob("*.sacc"): + assert cu.read_stamp(sio.load(part).metadata).stamp == blinded.stamp + verify = ["verify", str(out / "TOY.sacc"), "--cat-config", str(cat_config)] + assert bd.main(verify) == 0 + blinded_run = {str(p): p.read_bytes() for p in out.rglob("*") if p.is_file()} # --- the reveal: archive, flip the declaration, re-measure, audit -------- - cat_config = str(toy.cat_config) - assert ( - bd.main(["reveal", "toy", "--root", str(out), "--cat-config", cat_config]) == 0 - ) - archive = out / "revealed" / "toy" - config = yaml.safe_load(toy.cat_config.read_text()) + reveal = ["reveal", "toy", "--root", str(out), "--cat-config", str(cat_config)] + assert bd.main(reveal) == 0 + config = yaml.safe_load(cat_config.read_text()) config["TOY"]["blinding"] = "unblinded" - toy.cat_config.write_text(yaml.safe_dump(config, sort_keys=False)) - for version in versions: - run_chain(toy.cat_config, version, out, toy.cov) - report = bd.audit("toy", archive=archive, true_root=out, cat_config=toy.cat_config) + cat_config.write_text(yaml.safe_dump(config, sort_keys=False)) + run_chain(cat_config, out, cov) + archive = out / "revealed" / "toy" + report = bd.audit("toy", archive=archive, true_root=out, cat_config=cat_config) assert report["ok"], json.dumps(report, indent=1, default=str) - assert set(report["parts"]) == set(born) + assert len(report["parts"]) == 2 - # --- no true ξ± value was ever written or drawn by the blinded run ------- - true_xi = np.concatenate( - [ - np.concatenate([gg.xip, gg.xim]) - for v in versions - for grid in GRIDS - for gg in [sio.xi_correlation(sio.load(out / f"{v}_xi_{grid}.sacc"))] - ] - ) - figures = {"figures": np.concatenate(drawn).tobytes()} - assert not plaintext({**blinded_run, **figures}, true_xi) - assert not list(toy.root.rglob("*_xi_*.txt")) + # --- no true ξ± value was written by the blinded run --------------------- + gg = sio.xi_correlation(sio.load(out / "TOY_xi_reporting.sacc")) + assert not plaintext(blinded_run, np.concatenate([gg.xip, gg.xim])) + assert not list(tmp_path.rglob("*_xi_*.txt")) diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py index dbe1c21b..04f3fdf6 100644 --- a/workflow/tests/conftest.py +++ b/workflow/tests/conftest.py @@ -11,14 +11,10 @@ Snakemake is in. The ``candide`` tests need candide itself; CI deselects them. The ``toy`` fixture is a disposable checkout: copies of ``workflow/`` and -``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, a -``cosmo_val/cat_config.yaml`` declaring one catalogue per custody state, each -over its own touched catalogue file, a blind registry of hand-written records (the host -never decrypts, so no record needs a real seed), the processed CosmoCov -covariances already in place (their inputs live on candide), the toy -catalogue's patch centres, and both output roots in tmp. Its runs use a fake -image whose Python and Snakemake match the running ones, so the launch-time -parity check passes without apptainer. +``papers/cosmo_val/``, ``src/`` symlinked in, one catalogue per custody state, +hand-written blind records (the host never decrypts), stand-ins for the +CosmoCov covariances and patch centres, and a fake image matching the host's +Python and Snakemake. """ import dataclasses @@ -128,7 +124,6 @@ class Toy: env: dict config: dict common: object - covariances: dict # (version, "g" | "ng") -> the processed CosmoCov file def snakemake( self, *args, container=None, config=(), cwd=None, env=None, timeout=300 @@ -254,10 +249,9 @@ def toy(tmp_path_factory): # The processed CosmoCov covariances the cosmo_val rules read, in place. grids = common.xi_grids(config, config["fiducial"]) mask = "_masked" if config["covariance"].get("default_masked") else "" - covariances = {} for version in VERSIONS: for gaussian, grid in (("ng", grids["reporting"]), ("g", grids["integration"])): - path = covariances[version, gaussian] = Path( + path = Path( common.covariance_path( version, config["fiducial"]["blind"], @@ -288,7 +282,6 @@ def toy(tmp_path_factory): env=env, config=config, common=common, - covariances=covariances, ) diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 64e51342..52883796 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -34,47 +34,6 @@ def grids(toy): return toy.common.xi_grids(toy.config, toy.config["fiducial"]) -def test_assembly_gathers_every_part_and_its_covariances(toy, forced, grids): - """[P1] Each terminal file gathers every part and the analytic covariances: - ξ± and ρ/τ on the reporting grid with CosmoCov's covariance there, the - pseudo-Cℓ part and NaMaster covariance on one harmonic binning, and - COSEBIs and pure-E/B, which read the one integration-grid part and its - CosmoCov covariance.""" - jobs = forced[1] - reporting = toy.common.grid_binning(grids["reporting"]) - harmonic = toy.common.pseudo_cl_tag(toy.config) - assembled = [j for j in jobs if j.rule == "assemble_sacc"] - assert sorted(j.wildcards["version"] for j in assembled) == sorted(VERSIONS) - for job in assembled: - version = job.wildcards["version"] - assert [Path(o).name for o in job.output] == [f"{version}.sacc"] - assert {Path(f).name for f in job.input} == { - f"{version}_xi_{reporting}.sacc", - toy.covariances[version, "ng"].name, - f"pseudo_cl_{version}_{harmonic}.sacc", - f"pseudo_cl_cov_{version}_{harmonic}.fits", - f"{version}_cosebis.sacc", - f"{version}_pure_eb.sacc", - f"rho_tau_{version}_{reporting}.sacc", - }, job.input - - measured = [ - (j.wildcards["version"], toy.common.grid_of(grids, j.wildcards)) - for j in jobs - if j.rule == "xi" - ] - assert sorted(measured) == sorted((v, g) for v in VERSIONS for g in grids) - integration = toy.common.grid_binning(grids["integration"]) - for version in VERSIONS: - by_rule = { - j.rule: set(j.input) for j in jobs if j.wildcards.get("version") == version - } - part = str(toy.cosmo_val / f"{version}_xi_{integration}.sacc") - pair = {part, str(toy.covariances[version, "g"])} - assert by_rule["cv_cosebis"] == pair, by_rule["cv_cosebis"] - assert pair <= by_rule["cv_pure_eb"], by_rule["cv_pure_eb"] - - def test_xi_leaves_a_measurement_only_as_a_part(toy, forced, grids): """[P12] No job reads or writes a ξ± text dump; the ξ± figures draw the parts.""" jobs = forced[1] @@ -135,10 +94,6 @@ def _launch_refusals(toy, tmp_path): campaign = tmp_path / "campaign.yaml" config = dict(toy.config, cosmo_val=dict(toy.config["cosmo_val"], type="mock")) campaign.write_text(yaml.safe_dump(config)) - npatch = toy.common.xi_grids(toy.config, toy.config["fiducial"])["reporting"][ - "npatch" - ] - plain = toy.common._plain return { # [P6] a blinded catalogue no blind covers: pull, then draw or share "no_blind": ( @@ -157,22 +112,10 @@ def _launch_refusals(toy, tmp_path): {}, ["custody is declared per catalogue in cosmo_val/cat_config.yaml"], ), - # a tree without its patch centres: the command that draws them - "no_centres": ( - [], - dict(env={**toy.env, "COSMO_VAL": str(tmp_path)}), - [ - f"python -m sp_validation.cosmo_val.patch_centers {VERSIONS[0]} {npatch} " - f"--cat-config {plain(toy.root / 'cosmo_val' / 'cat_config.yaml')} " - f"--output-dir {plain(tmp_path)}" - ], - ), } -@pytest.mark.parametrize( - "case", ["no_blind", "unrevealed", "campaign_type", "no_centres"] -) +@pytest.mark.parametrize("case", ["no_blind", "unrevealed", "campaign_type"]) def test_a_launch_the_dag_cannot_honour_stops_with_the_fix(toy, tmp_path, case): args, launch, message = _launch_refusals(toy, tmp_path)[case] result = toy.snakemake("-n", "assemble_sacc_all", *args, **launch) diff --git a/workflow/tests/test_image_resolution.py b/workflow/tests/test_image_resolution.py deleted file mode 100644 index 66687468..00000000 --- a/workflow/tests/test_image_resolution.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Every launcher runs the image ``sp_validation/container.py`` resolves.""" - -import os -import subprocess - -import pytest -from conftest import REPO, container - -SWEEP_ENV = REPO / "papers" / "bmodes" / "scripts" / "container_env.sh" - - -@pytest.mark.parametrize("sandbox", [False, True], ids=["sif", "sandbox"]) -def test_sweep_drivers_run_the_resolved_image(tmp_path, monkeypatch, sandbox): - """The Paper II sweep drivers' CONTAINER is the image container.py resolves. - - Their shell copy of the resolution reads the same cache, whatever - XDG_CACHE_HOME says, and prefers the sandbox the same way. - """ - cache = tmp_path / "home" / ".cache" / "sp_validation" - cache.mkdir(parents=True) - (cache / "sp_validation.sif").touch() - if sandbox: - (cache / "sandbox").mkdir() - monkeypatch.setenv("HOME", str(tmp_path / "home")) - monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "node-local")) - for var in ("SPV_CONTAINER", "SPV_SANDBOX"): - monkeypatch.delenv(var, raising=False) - - shell = subprocess.run( - ["bash", "-c", f'. "{SWEEP_ENV}" && printf %s "$CONTAINER"'], - env=dict(os.environ), - capture_output=True, - text=True, - check=True, - ) - assert shell.stdout == container.resolve_image()[0] diff --git a/workflow/tests/test_toy_run.py b/workflow/tests/test_toy_run.py deleted file mode 100644 index 10e6b16c..00000000 --- a/workflow/tests/test_toy_run.py +++ /dev/null @@ -1,204 +0,0 @@ -"""Rule xi, run for real through apptainer, before and after a blind is drawn. - -The launch is the README's with the machine-independent default profile: the -host Snakemake, the image `spv-container` manages, jobs on this node. A toy -checkout under your home directory (which the profile binds) carries copies of -workflow/, papers/cosmo_val/ and src/, and one synthetic catalogue declared -unblinded. The first launch stops for want of its patch centres and names the -command that draws them; after it has run, its reporting parts and their figure -are made; then the catalogue is declared blinded, a blind is drawn for it, and -the same launch re-measures the parts concealed. The jobs run under a -matplotlibrc asking for a LaTeX package no image has, as a user's own may. -""" - -import json -import os -import shlex -import shutil -import subprocess -import sys -import tempfile -from pathlib import Path - -import pytest -import yaml -from conftest import REPO, container, on_candide - -VERSIONS = ("SP_v0.1", "SP_v0.1_leak_corr") -REPORTING = {"theta_min": 5.0, "theta_max": 60.0, "nbins": 6, "npatch": 4} -BINNING = "minsep=5.0_maxsep=60.0_nbins=6_npatch=4" - - -def _in_image(image, root, *command): - """``command``'s stdout, run in ``image`` on the toy checkout's src/.""" - return subprocess.run( - [ - "apptainer", - "exec", - "--cleanenv", - "--env", - f"PYTHONPATH={root / 'src'}", - str(image), - *command, - ], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout - - -def _stamps(image, root, parts): - """The custody stamp of each SACC part, read in the image.""" - script = ( - "import json, sys\n" - "from sp_validation import custody, sacc_io\n" - "print(json.dumps([custody.read_stamp(sacc_io.load(p).metadata).stamp" - " for p in sys.argv[1:]]))" - ) - return json.loads(_in_image(image, root, "python", "-c", script, *map(str, parts))) - - -def _toy_checkout(root, image): - """A checkout with one synthetic catalogue, SP_v0.1, declared unblinded.""" - skip = shutil.ignore_patterns(".snakemake", "__pycache__", "tests") - shutil.copytree(REPO / "workflow", root / "workflow", ignore=skip) - shutil.copytree( - REPO / "papers" / "cosmo_val", root / "papers" / "cosmo_val", ignore=skip - ) - shutil.copytree( - REPO / "src", root / "src", ignore=shutil.ignore_patterns("__pycache__") - ) - (root / "cosmo_val").mkdir() - _in_image( - image, - root, - "python", - "-c", - "import sys\n" - f"sys.path.insert(0, {str(root / 'src/sp_validation/tests')!r})\n" - "from pathlib import Path\n" - "from _synthetic import write_synthetic_catalogs\n" - f"write_synthetic_catalogs(Path({str(root / 'cosmo_val')!r})," - f" catalogues={{{VERSIONS[0]!r}: 'unblinded'}})", - ) - - config_path = root / "papers" / "cosmo_val" / "config" / "config.yaml" - config = yaml.safe_load(config_path.read_text()) - config["versions"] = list(VERSIONS) - config["fiducial"]["version"] = VERSIONS[1] - config["fiducial"]["mock_version"] = VERSIONS[0] - config["cosmo_val"].update(REPORTING) - config_path.write_text(yaml.safe_dump(config, sort_keys=False)) - - # A user's matplotlibrc typesetting with a package the image lacks. - rc = root / "xdg" / "matplotlib" / "matplotlibrc" - rc.parent.mkdir(parents=True) - rc.write_text( - "text.usetex: True\n" - "text.latex.preamble: \\usepackage{spvalidationabsentpackage}\n" - ) - (root / "fast.json").write_text( - json.dumps({"theory": {"transfer_function": "eisenstein_hu"}}) - ) - - -@pytest.mark.candide -@on_candide -def test_xi_before_and_after_its_catalogue_is_blinded(): - image, kind = container.resolve_image() - assert kind != "tag", "no local image; run `spv-container pull`" - root = Path(tempfile.mkdtemp(prefix="toy_run_", dir=Path.home())) - _toy_checkout(root, image) - out = root / "out" - cat_config = root / "cosmo_val" / "cat_config.yaml" - parts = [out / f"{v}_xi_{BINNING}.sacc" for v in VERSIONS] - target = str(out / "snakemake_sentinels" / "plot_2pcf.done") - - env = { - k: v - for k, v in os.environ.items() - if k not in ("SNAKEMAKE_PROFILE", "APPTAINERENV_PYTHONPATH") - } - env.update( - COSMO_VAL=str(out), - COSMO_INFERENCE=str(root / "inference"), - XDG_CACHE_HOME=str(root / "cache"), - APPTAINERENV_XDG_CONFIG_HOME=str(root / "xdg"), - PYTHONNOUSERSITE="1", - PYTHONUNBUFFERED="1", - ) - - def launch(*args): - return subprocess.run( - [ - sys.executable, - "-m", - "snakemake", - "--profile", - str(root / "workflow" / "profiles" / "default"), - "--cores", - "4", - *args, - "--config", - f"container={image}", - "--", - target, - ], - cwd=root / "papers" / "cosmo_val", - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=1800, - check=False, - ) - - # No centres: the launch names the command that draws them, which works. - result = launch() - assert result.returncode != 0, result.stdout - (command,) = [ - line.split("spv-container exec ", 1)[1] - for line in result.stdout.splitlines() - if "sp_validation.cosmo_val.patch_centers" in line - ] - _in_image(image, root, *shlex.split(command)) - assert (out / "patches" / "SP_v0.1_npatch=4.dat").is_file() - - # Declared unblinded: parts stamped unblinded, and their figure drawn. - result = launch() - assert result.returncode == 0, result.stdout - assert [s["blinding"] for s in _stamps(image, root, parts)] == ["unblinded"] * 2 - assert (out / "xi_p.png").is_file() - - # Declared blinded, and its blind drawn: the parts' params changed. - catalogues = yaml.safe_load(cat_config.read_text()) - catalogues[VERSIONS[0]]["blinding"] = "blinded" - cat_config.write_text(yaml.safe_dump(catalogues, sort_keys=False)) - _in_image( - image, - root, - "python", - "-m", - "sp_validation.blinding", - "init", - "toy", - VERSIONS[0], - "--cat-config", - str(cat_config), - "--config", - str(root / "fast.json"), - ) - dry = launch("-n") - assert dry.returncode == 0, dry.stdout - assert "Params have changed" in dry.stdout, dry.stdout - - result = launch() - assert result.returncode == 0, result.stdout - record = json.loads((root / "cosmo_val/blinds/toy/commitment.json").read_text()) - for stamp in _stamps(image, root, parts): - assert stamp["blinding"] == "blinded", stamp - assert stamp["blinding_commitment"] == record["seed_commitment"], stamp - assert not list(out.rglob("*_xi_*.txt")) - assert not list((root / "cosmo_val" / "output").iterdir()) - - shutil.rmtree(root) # kept on failure, for post-mortem From 82b14fe64eba25cb0db28c984ae9c683bf1dc1dc Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 04:33:58 +0200 Subject: [PATCH 117/160] Scrub the n(z) A/B/C blind: an n(z) is a catalogue entry's Choosing an n(z) is choosing a catalogue entry: each entry's shear.redshift_path is its n(z), read by CosmologyValidation.get_redshift and by the workflow's redshift_path(version). CosmologyValidation loses its blind override, and the pseudo-Cl, covariance, inference and papers/bmodes filenames lose their blind token and wildcard. - common: catalogue_entry()/redshift_path() replace build_redshift_path; covariance_{base,dir,path} and pseudo_cl_tag drop blind; the version constraint admits the _A/_B/_C entries. - covariance.smk: get_cat_params and the star-halo mask resolve through the catalogue entry. - papers/bmodes: bb_covariance_blind_independence and the talk n(z) plot compare fiducial.nz_realisations, the SP_v1.4.6.3_{A,B,C}_leak_corr entries; pure-E/B, COSEBIs PTE and pseudo-Cl paths are per version. - cat_config: SP_v1.4.6.3_{A,B,C} match SP_v1.4.6.3 but for their n(z); SP_v1.4.8 and SP_v1.4.11.3(_ecut07) name the n(z) their covariances used. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- cosmo_val/cat_config.yaml | 48 +++---- .../bb_covariance_blind_independence.md | 7 +- papers/bmodes/config/bmodes_paper.md | 2 +- papers/bmodes/config/cl.md | 6 +- papers/bmodes/config/config.yaml | 11 +- .../config/config_space_pte_matrices.md | 4 +- papers/bmodes/config/cosebis.md | 4 +- papers/bmodes/config/cosebis_data_vector.md | 4 +- papers/bmodes/config/covariance.md | 18 +-- papers/bmodes/config/ecut_spec.md | 3 +- .../config/harmonic_space_pte_matrices.md | 4 +- papers/bmodes/config/pure_eb.md | 4 +- papers/bmodes/config/pure_eb_covariance.md | 1 - papers/bmodes/config/pure_eb_data_vector.md | 2 +- papers/bmodes/rules/claims.smk | 123 ++++++++---------- papers/bmodes/rules/presentation.smk | 6 +- .../bb_covariance_blind_independence.py | 48 +++---- .../bmodes/scripts/calculate_pure_eb_ptes.py | 9 +- papers/bmodes/scripts/cl_data_vector.py | 15 +-- .../bmodes/scripts/cl_version_comparison.py | 15 +-- .../scripts/compute_cosebis_pte_single.py | 18 +-- .../scripts/config_space_pte_matrices.py | 48 +++---- .../scripts/cosebis_version_comparison.py | 7 +- .../bmodes/scripts/gather_pure_eb_chunks.py | 13 +- .../harmonic_config_cosebis_comparison.py | 14 +- .../scripts/harmonic_space_pte_matrices.py | 31 ++--- .../scripts/precompute_pure_eb_chunk.py | 7 +- papers/bmodes/scripts/pure_eb_covariance.py | 3 +- papers/bmodes/scripts/pure_eb_data_vector.py | 10 +- .../scripts/pure_eb_version_comparison.py | 10 +- papers/bmodes/scripts/run_cl_sweep.py | 21 ++- .../bmodes/scripts/run_cosebis_ptes_sweep.py | 15 +-- .../bmodes/scripts/run_pure_eb_ptes_sweep.sh | 13 +- .../scripts/run_pure_eb_semianalytic.sh | 13 +- papers/bmodes/scripts/run_pure_eb_sweep.sh | 17 ++- papers/cosmo_val/config/config.yaml | 2 - src/sp_validation/cosmo_val/core.py | 17 +-- .../tests/test_cv_init_params.py | 4 +- src/sp_validation/tests/test_pseudo_cl.py | 2 +- workflow/common.py | 70 +++------- workflow/rules/cosmo_val.smk | 1 - workflow/rules/covariance.smk | 51 ++++---- workflow/rules/glass_mock.smk | 6 +- workflow/rules/inference.smk | 34 ++--- workflow/rules/twopoint.smk | 18 +-- workflow/scripts/generate_pseudo_cl.py | 12 +- workflow/scripts/generate_pseudo_cl_cov.py | 12 +- workflow/tests/conftest.py | 1 - 48 files changed, 315 insertions(+), 489 deletions(-) diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index a010e072..0e493fec 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -557,19 +557,19 @@ SP_v1.4.6.3_B: getdist_colour: 0.0, 0.5, 1.0 ls: dashed marker: d + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits cov_th: - A: 2405.3892055695346 - n_e: 6.128201234871523 + A: 2894.0303815287743 + n_e: 4.957279270321334 n_psf: 0.752316232272063 - sigma_e: 0.379587601488189 - mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + sigma_e: 0.3783979556382064 psf: PSF_flag: HSM_FLAG_PSF PSF_size: HSM_T_PSF star_flag: HSM_FLAG_STAR star_size: HSM_T_STAR hdu: 1 - path: unions_shapepipe_psf_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_PSF_col: HSM_G1_PSF @@ -578,7 +578,7 @@ SP_v1.4.6.3_B: e2_star_col: HSM_G2_STAR shear: R: 1.0 - path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + path: /n17data/UNIONS/WL/v1.4.x/v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6.3/nz_SP_v1.4.6.3_B.txt w_col: w_des e1_col: e1 @@ -592,7 +592,7 @@ SP_v1.4.6.3_B: dec_col: Dec e1_col: e1 e2_col: e2 - path: unions_shapepipe_star_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.3_C: blinding: unblinded @@ -602,19 +602,19 @@ SP_v1.4.6.3_C: getdist_colour: 0.0, 0.5, 1.0 ls: dashed marker: d + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits cov_th: - A: 2405.3892055695346 - n_e: 6.128201234871523 + A: 2894.0303815287743 + n_e: 4.957279270321334 n_psf: 0.752316232272063 - sigma_e: 0.379587601488189 - mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + sigma_e: 0.3783979556382064 psf: PSF_flag: HSM_FLAG_PSF PSF_size: HSM_T_PSF star_flag: HSM_FLAG_STAR star_size: HSM_T_STAR hdu: 1 - path: unions_shapepipe_psf_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_PSF_col: HSM_G1_PSF @@ -623,7 +623,7 @@ SP_v1.4.6.3_C: e2_star_col: HSM_G2_STAR shear: R: 1.0 - path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + path: /n17data/UNIONS/WL/v1.4.x/v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6.3/nz_SP_v1.4.6.3_C.txt w_col: w_des e1_col: e1 @@ -637,7 +637,7 @@ SP_v1.4.6.3_C: dec_col: Dec e1_col: e1 e2_col: e2 - path: unions_shapepipe_star_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.3_A: blinding: unblinded @@ -647,19 +647,19 @@ SP_v1.4.6.3_A: getdist_colour: 0.0, 0.5, 1.0 ls: dashed marker: d + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits cov_th: - A: 2405.3892055695346 - n_e: 6.128201234871523 + A: 2894.0303815287743 + n_e: 4.957279270321334 n_psf: 0.752316232272063 - sigma_e: 0.379587601488189 - mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + sigma_e: 0.3783979556382064 psf: PSF_flag: HSM_FLAG_PSF PSF_size: HSM_T_PSF star_flag: HSM_FLAG_STAR star_size: HSM_T_STAR hdu: 1 - path: unions_shapepipe_psf_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_PSF_col: HSM_G1_PSF @@ -668,7 +668,7 @@ SP_v1.4.6.3_A: e2_star_col: HSM_G2_STAR shear: R: 1.0 - path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + path: /n17data/UNIONS/WL/v1.4.x/v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6.3/nz_SP_v1.4.6.3_A.txt w_col: w_des e1_col: e1 @@ -682,7 +682,7 @@ SP_v1.4.6.3_A: dec_col: Dec e1_col: e1 e2_col: e2 - path: unions_shapepipe_star_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6_ecut07: base: SP_v1.4.6 @@ -851,7 +851,7 @@ SP_v1.4.8: shear: R: 1.0 path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits - redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.8/nz_SP_v1.4.8_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected @@ -941,7 +941,7 @@ SP_v1.4.11.3: shear: R: 1.0 path: v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits - redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected @@ -1078,7 +1078,7 @@ SP_v1.4.11.3_ecut07: shear: R: 1.0 path: /n17data/cdaley/unions/pure_eb/results/ecut/SP_v1.4.11.3_ecut07.fits - redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected diff --git a/papers/bmodes/config/bb_covariance_blind_independence.md b/papers/bmodes/config/bb_covariance_blind_independence.md index 33ec2cb9..0af8f908 100644 --- a/papers/bmodes/config/bb_covariance_blind_independence.md +++ b/papers/bmodes/config/bb_covariance_blind_independence.md @@ -4,11 +4,13 @@ Method: [Covariance](covariance.md), [Pure E/B](pure_eb.md), [COSEBIS](cosebis.m ## Claim -BB covariances computed via **analytic propagation** are blind-independent, while BB covariances computed via **MC sampling** inherit noise that varies across blinds. +BB covariances computed via **analytic propagation** are independent of the A/B/C n(z) realisation, while BB covariances computed via **MC sampling** inherit noise that varies across realisations. + +Each realisation is its own catalogue entry, identical to the mock version's but for its n(z): `fiducial.nz_realisations` maps the labels A/B/C to those entries (`SP_v1.4.6.3_{A,B,C}_leak_corr`). ## Evidence -Diagonal ratios relative to blind A for both B-modes and E-modes: +Diagonal ratios relative to realisation A for both B-modes and E-modes: - Pure E/B: `cov(ξ+^B)`, `cov(ξ-^B)` vs `cov(ξ+^E)`, `cov(ξ-^E)` - COSEBIS: `cov(B_n)` vs `cov(E_n)` - Harmonic: `cov(C_ℓ^BB)` vs `cov(C_ℓ^EE)` @@ -22,6 +24,7 @@ Diagonal ratios relative to blind A for both B-modes and E-modes: | Parameter | Config Key | |-----------|------------| | Version | `fiducial.version` | +| n(z) realisations | `fiducial.nz_realisations` | | Scale range | `fiducial.min_sep` to `fiducial.max_sep` | | Bins | `fiducial.nbins` | | COSEBIS nmodes | `fiducial.nmodes` | diff --git a/papers/bmodes/config/bmodes_paper.md b/papers/bmodes/config/bmodes_paper.md index 5623fe8a..9c87d375 100644 --- a/papers/bmodes/config/bmodes_paper.md +++ b/papers/bmodes/config/bmodes_paper.md @@ -86,7 +86,7 @@ Note: Per-version PTEs come from `config_space_pte_matrices`, not `pure_eb_data_ ### Blind Handling -PTEs report the **minimum across blinds** (`pte_joint_min`) as the conservative estimate. This ensures reported values remain valid regardless of which blind is eventually unblinded. The fiducial blind `config["fiducial"]["blind"]` determines covariance matrix selection. +PTEs report the **minimum across blinds** (`pte_joint_min`) as the conservative estimate. This ensures reported values remain valid regardless of which blind is eventually unblinded. Covariances use each version's catalogue-entry n(z). ## Macros and Tables diff --git a/papers/bmodes/config/cl.md b/papers/bmodes/config/cl.md index ad0b7d70..5b30d26c 100644 --- a/papers/bmodes/config/cl.md +++ b/papers/bmodes/config/cl.md @@ -25,10 +25,10 @@ Pseudo-Cl estimation accounts for: ## Data Source Pseudo-Cl files generated by workflow rules using NaMaster: -- `pseudo_cl_{version}_blind={blind}_powspace_nbins={nbins}.sacc` — Power spectrum estimates -- `pseudo_cl_cov_{version}_blind={blind}_powspace_nbins={nbins}.fits` — Bandpower covariance matrix +- `pseudo_cl_{version}_powspace_nbins={nbins}.sacc` — Power spectrum estimates +- `pseudo_cl_cov_{version}_powspace_nbins={nbins}.fits` — Bandpower covariance matrix Location: `{COSMO_VAL_OUTPUT}/` (defined in Snakefile, typically `/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/output/`) -Default parameters: `blind=A` (from `fiducial.blind`), `nbins=32` (from `cl.n_ell_bins`). +Default parameters: `nbins=32` (from `cl.n_ell_bins`). diff --git a/papers/bmodes/config/config.yaml b/papers/bmodes/config/config.yaml index 47b385bd..3c8357d7 100644 --- a/papers/bmodes/config/config.yaml +++ b/papers/bmodes/config/config.yaml @@ -23,7 +23,6 @@ nbins_int: [100, 200, 300, 500] # fiducial parameters fiducial: version: SP_v1.4.6.3_leak_corr - blind: A npatch: 1 min_sep: 1.0 max_sep: 250.0 @@ -31,6 +30,13 @@ fiducial: mock_version: SP_v1.4.6.3 + # The mock version's catalogue under each n(z) realisation: entries identical + # but for shear.redshift_path, compared by bb_covariance_blind_independence. + nz_realisations: + A: SP_v1.4.6.3_A_leak_corr + B: SP_v1.4.6.3_B_leak_corr + C: SP_v1.4.6.3_C_leak_corr + min_sep_int: 0.5 max_sep_int: 300.0 nbins_int: 1000 @@ -79,10 +85,9 @@ cl: # Harmonic pseudo-Cl fiducial binning — the tag the producer (twopoint.smk # pseudo_cl / pseudo_cl_cov) stamps into the filename, which the inference -# consumer reconstructs to request the exact file. Canonical: A / powspace / 32. +# consumer reconstructs to request the exact file. Canonical: powspace / 32. harmonic: fiducial: - blind: A binning: powspace nbins: 32 diff --git a/papers/bmodes/config/config_space_pte_matrices.md b/papers/bmodes/config/config_space_pte_matrices.md index 40073add..a7ad89a0 100644 --- a/papers/bmodes/config/config_space_pte_matrices.md +++ b/papers/bmodes/config/config_space_pte_matrices.md @@ -10,9 +10,9 @@ Fiducial angular scale cuts are justified by PTE heatmaps across all (theta_min, Scale cuts from `fiducial.fiducial_xip_scale_cut` and `fiducial.fiducial_xim_scale_cut`. COSEBIS uses the same unified range. -## Blind Handling +## n(z) -Uses fiducial blind from `config["fiducial"]["blind"]`. Data vectors (ξ+^B, ξ-^B, COSEBIS B_n) are identical across blinds; covariances vary with blind via n(z)-dependent theoretical predictions. +Each version's covariance uses its catalogue entry's n(z) (`shear.redshift_path` in `cat_config.yaml`). ## Config References diff --git a/papers/bmodes/config/cosebis.md b/papers/bmodes/config/cosebis.md index 25756887..5242a08d 100644 --- a/papers/bmodes/config/cosebis.md +++ b/papers/bmodes/config/cosebis.md @@ -33,9 +33,9 @@ Compare across catalog versions from `config.versions`: - Fiducial: `fiducial.version` - All versions tested for consistency -## Blind Handling +## n(z) -Uses fiducial blind from `config["fiducial"]["blind"]`. COSEBIS B_n data vectors are identical across blinds; covariances vary with blind via n(z)-dependent theoretical predictions. +Each version's covariance uses its catalogue entry's n(z) (`shear.redshift_path` in `cat_config.yaml`). ## Analysis Decisions diff --git a/papers/bmodes/config/cosebis_data_vector.md b/papers/bmodes/config/cosebis_data_vector.md index 064ae63a..30e85731 100644 --- a/papers/bmodes/config/cosebis_data_vector.md +++ b/papers/bmodes/config/cosebis_data_vector.md @@ -8,9 +8,9 @@ Plotting: [1D Plots](1d_plots.md) COSEBIS B-modes at fiducial version (`fiducial.version`) are consistent with zero across the full angular range and at fiducial scale cuts. -## Blind Handling +## n(z) -Uses fiducial blind from `config["fiducial"]["blind"]`. COSEBIS B_n data vectors are identical across blinds; covariances vary with blind via n(z)-dependent theoretical predictions. Statistical evidence (PTEs) is in [Config-Space PTE Matrices](config_space_pte_matrices.md). +Each version's covariance uses its catalogue entry's n(z) (`shear.redshift_path` in `cat_config.yaml`). Statistical evidence (PTEs) is in [Config-Space PTE Matrices](config_space_pte_matrices.md). ## Config References diff --git a/papers/bmodes/config/covariance.md b/papers/bmodes/config/covariance.md index 54e7ad42..2959e433 100644 --- a/papers/bmodes/config/covariance.md +++ b/papers/bmodes/config/covariance.md @@ -40,17 +40,18 @@ Extracted from catalog config (`cat_config.yaml`) per version: ## File Naming ``` -covariance_{version}_{blind}_{gaussian}_minsep={min}_maxsep={max}_nbins={n}{mask_suffix}_processed.txt +covariance_{version}_{gaussian}_minsep={min}_maxsep={max}_nbins={n}{mask_suffix}_processed.txt ``` - `version`: SP_v1.4.6_leak_corr, etc. -- `blind`: A, B, or C - `gaussian`: g (Gaussian-only) or ng (non-Gaussian) - `mask_suffix`: empty or `_masked` -## Blind Handling +## n(z) -B-mode claims use the fiducial blind from `config["fiducial"]["blind"]`. Covariances are computed for the fiducial blind only. +A version's covariance uses its catalogue entry's n(z) (`shear.redshift_path` in +`cat_config.yaml`). Choosing another n(z) realisation means choosing another +catalogue entry (e.g. `SP_v1.4.6.3_B`). ## Covariance Usage Policy @@ -58,7 +59,7 @@ Official results use specific covariance sources for consistency and correctness | Quantity | Source | Gaussian | Binning | Notes | |----------|--------|----------|---------|-------| -| Total ξ± errors | CosmoCov | ng | 20-bin (reporting) | Per-blind, masked | +| Total ξ± errors | CosmoCov | ng | 20-bin (reporting) | Masked | | Pure E/B mode errors | MC propagation | g | 1000→20 bin | Conservative: underestimates uncertainty | | COSEBIS errors | MC propagation | g | 1000→scales | Per scale cut | @@ -76,10 +77,9 @@ only spatially-structured cuts (no galaxy selection cuts): | Standard footprint | 2894 deg² | v1.4.5, v1.4.6, v1.4.11.3 (and ecut variants) | | Star-halo footprint | 2517 deg² | v1.4.8 | -Each version gets its own covariance from its own survey properties (A, n_e, sigma_e). -`resolve_covariance_version()` is the identity function — no cross-version covariance sharing. -`MASK_CLS_FILES` (covariance.smk) maps to two mask power spectrum files based on whether -the version is in `STARHALO_VERSIONS`. +Each version gets its own covariance from its own catalogue entry's survey properties +(`cov_th`: A, n_e, sigma_e). `MASK_CLS_FILES` (covariance.smk) maps to two mask power +spectrum files based on whether the version's base catalogue is in `STARHALO_CATALOGUES`. ## Related Specs diff --git a/papers/bmodes/config/ecut_spec.md b/papers/bmodes/config/ecut_spec.md index 53041659..88634234 100644 --- a/papers/bmodes/config/ecut_spec.md +++ b/papers/bmodes/config/ecut_spec.md @@ -39,8 +39,7 @@ The 2PCF and covariance are independent and can run in parallel. Key resolution functions in `workflow/Snakefile`: - `get_shear_catalog()` — resolves catalog path from cat_config, strips `_leak_corr` -- `build_redshift_path()` — v1.4.11.x uses v1.4.6's n(z) -- `resolve_covariance_version()` — identity function (each version gets its own covariance) +- `redshift_path()` — the catalogue entry's `shear.redshift_path` (v1.4.11.3's entries point at v1.4.6's n(z)) - Wildcard constraint: `version=r"SP_v[\d.]+(_w_iv)?(_leak_corr)?"` — needs `_ecut\d+` Version comparison rules in `papers/bmodes/rules/claims.smk` (lines 131, 271, 386) use diff --git a/papers/bmodes/config/harmonic_space_pte_matrices.md b/papers/bmodes/config/harmonic_space_pte_matrices.md index 0f17b6fc..e98c4031 100644 --- a/papers/bmodes/config/harmonic_space_pte_matrices.md +++ b/papers/bmodes/config/harmonic_space_pte_matrices.md @@ -10,9 +10,9 @@ Harmonic-space B-mode PTEs are consistent with noise at fiducial multipole range Fiducial scale cuts from `cl.fiducial_ell_min` and `cl.fiducial_ell_max`. Full B-mode test range spans all multipole bins present in the input pseudo-Cℓ file. -## Blind Handling +## n(z) -Uses fiducial blind from `config["fiducial"]["blind"]`. The C_ℓ^BB data vector is identical across blinds; covariances vary with blind via n(z)-dependent theoretical predictions. +Each version's covariance uses its catalogue entry's n(z) (`shear.redshift_path` in `cat_config.yaml`). ## Config References diff --git a/papers/bmodes/config/pure_eb.md b/papers/bmodes/config/pure_eb.md index 6800b149..76f9c439 100644 --- a/papers/bmodes/config/pure_eb.md +++ b/papers/bmodes/config/pure_eb.md @@ -28,9 +28,9 @@ Uses semi-analytical covariance propagation through the decomposition. ## Data Products Precomputed decomposition stored in: -`results/paper_plots/intermediate/{version}_{blind}_pure_eb_semianalytic.npz` +`results/paper_plots/intermediate/{version}_pure_eb_semianalytic.npz` -Uses fiducial blind (A) from config. Each NPZ contains: +Each NPZ contains: - `theta`: Angular bins - `xip_E`, `xim_E`: Pure E-mode components - `xip_B`, `xim_B`: Pure B-mode components diff --git a/papers/bmodes/config/pure_eb_covariance.md b/papers/bmodes/config/pure_eb_covariance.md index 05453d23..bca1ed01 100644 --- a/papers/bmodes/config/pure_eb_covariance.md +++ b/papers/bmodes/config/pure_eb_covariance.md @@ -28,7 +28,6 @@ Block-wise condition numbers for the 120×120 pure E/B covariance (6 blocks of 2 | Parameter | Config Key | |-----------|------------| | Version | `fiducial.version` | -| Blind | `fiducial.blind` | | Integration bins | `fiducial.nbins_int` | | Reporting bins | `fiducial.nbins` | diff --git a/papers/bmodes/config/pure_eb_data_vector.md b/papers/bmodes/config/pure_eb_data_vector.md index 6e334d9f..a7018b96 100644 --- a/papers/bmodes/config/pure_eb_data_vector.md +++ b/papers/bmodes/config/pure_eb_data_vector.md @@ -20,7 +20,7 @@ B-mode signals in UNIONS cosmic shear are consistent with zero at fiducial scale ## Evidence -PTE values for B-mode null tests at two scale ranges, using fiducial blind (A): +PTE values for B-mode null tests at two scale ranges: 1. **Fiducial scale cuts**: Angular range used for cosmological inference 2. **Full theta range**: All measured angular bins (no cuts) diff --git a/papers/bmodes/rules/claims.smk b/papers/bmodes/rules/claims.smk index db60a9dc..4503c846 100644 --- a/papers/bmodes/rules/claims.smk +++ b/papers/bmodes/rules/claims.smk @@ -8,7 +8,7 @@ Claims depend on methods (for technique definitions) and compute outputs (for da # Configuration # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# CONFIG_DIR, TAPESTRY_DIR, PAPER_FIGURES_DIR, BLINDS, FIDUCIAL, PLANCK18 defined in Snakefile +# CONFIG_DIR, TAPESTRY_DIR, PAPER_FIGURES_DIR, FIDUCIAL, PLANCK18 defined in Snakefile # COSMO_VAL, COSMO_INFERENCE, covariance_path() defined in Snakefile COSMO_VAL_OUTPUT = str(COSMO_VAL) # String version for f-string interpolation @@ -31,6 +31,10 @@ VERSION_LABELS = config["plotting"].get("version_labels", {}) FIDUCIAL_VERSION = FIDUCIAL["version"] MOCK_VERSION = f"{FIDUCIAL['mock_version']}_leak_corr" +# Catalogues identical to the mock version but for their n(z) realisation, +# keyed by realisation label: bb_covariance_blind_independence compares them. +NZ_REALISATIONS = FIDUCIAL["nz_realisations"] + # Filter versions for different analysis types # Pure E/B and PTEs only apply to leak-corrected versions VERSIONS_LEAK_CORR = [v for v in config["versions"] if "_leak_corr" in v and "_ecut" not in v] @@ -72,9 +76,9 @@ def _per_version_figure_outputs(claim_dir): # Path Helper Functions # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -def _reporting_cov_path(version, blind): +def _reporting_cov_path(version): """Path to reporting-scale covariance (non-Gaussian, masked).""" - return covariance_path(version, blind, gaussian="ng") + return covariance_path(version, gaussian="ng") def _xi_reporting_path(version): @@ -93,10 +97,10 @@ def _xi_integration_path(version): ) -def _cov_integration_path(version, blind): +def _cov_integration_path(version): """Covariance path for integration bins (Gaussian, for COSEBIS PTE).""" return covariance_path( - version, blind, gaussian="g", + version, gaussian="g", min_sep=FIDUCIAL["min_sep_int"], max_sep=FIDUCIAL["max_sep_int"], nbins=FIDUCIAL["nbins_int"] ) @@ -116,17 +120,14 @@ def _pte_scale_cut_pairs(): PTE_SCALE_CUT_PAIRS = _pte_scale_cut_pairs() -def _pseudo_cl_path(version, blind="A", nbins=32): - """Return pseudo-Cl path for a catalog version. - - All leak-corrected versions use consistent local naming with blind and binning. - """ - return f"{COSMO_VAL_OUTPUT}/pseudo_cl_{version}_blind={blind}_powspace_nbins={nbins}.sacc" +def _pseudo_cl_path(version, nbins=32): + """Return pseudo-Cl path for a catalog version.""" + return f"{COSMO_VAL_OUTPUT}/pseudo_cl_{version}_powspace_nbins={nbins}.sacc" -def _pseudo_cl_cov_path(version, blind="A", nbins=32): +def _pseudo_cl_cov_path(version, nbins=32): """Return pseudo-Cl covariance path for a catalog version.""" - return f"{COSMO_VAL_OUTPUT}/pseudo_cl_cov_{version}_blind={blind}_powspace_nbins={nbins}.fits" + return f"{COSMO_VAL_OUTPUT}/pseudo_cl_cov_{version}_powspace_nbins={nbins}.fits" # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ @@ -147,7 +148,7 @@ rule cosebis_version_comparison: config=f"{CONFIG_DIR}/config.yaml", # COSEBIs only for leak-corrected versions xi_integration=[_xi_integration_path(ver) for ver in VERSIONS_LEAK_CORR], - cov_integration=[_cov_integration_path(ver, "A") for ver in VERSIONS_LEAK_CORR], + cov_integration=[_cov_integration_path(ver) for ver in VERSIONS_LEAK_CORR], params: version_labels=VERSION_LABELS, versions=VERSIONS_LEAK_CORR, @@ -179,7 +180,7 @@ rule cosebis_data_vector: config=f"{CONFIG_DIR}/config.yaml", # Per-version inputs: xi_{version} and cov_{version} for all versions **{f"xi_{ver}": _xi_integration_path(ver) for ver in VERSIONS_ALL_FOR_PLOTS}, - **{f"cov_{ver}": _cov_integration_path(ver, "A") for ver in VERSIONS_ALL_FOR_PLOTS}, + **{f"cov_{ver}": _cov_integration_path(ver) for ver in VERSIONS_ALL_FOR_PLOTS}, params: cov_base_dir=str(COSMO_INFERENCE / "data/covariance"), output: @@ -204,7 +205,7 @@ rule cosebis_binning_comparison: f"{COSMO_VAL_OUTPUT}/{FIDUCIAL_VERSION}_xi_minsep={FIDUCIAL['min_sep_int']}" f"_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch={FIDUCIAL['npatch']}.sacc" ), - cov_1k=_cov_integration_path(FIDUCIAL_VERSION, "A"), + cov_1k=_cov_integration_path(FIDUCIAL_VERSION), output: evidence=f"{TAPESTRY_DIR}/cosebis_binning_comparison/evidence.json", figure=f"{TAPESTRY_DIR}/cosebis_binning_comparison/figure.png", @@ -225,14 +226,13 @@ N_PURE_EB_CHUNKS = config["pure_eb"]["n_chunks"] rule precompute_pure_eb_chunk: """Compute a chunk of MC samples for pure E/B covariance (scatter).""" input: - cov_integration=lambda w: _cov_integration_path(w.version, w.blind), + cov_integration=lambda w: _cov_integration_path(w.version), xi_reporting=lambda w: _xi_reporting_path(w.version), xi_integration=lambda w: _xi_integration_path(w.version), output: - "results/paper_plots/intermediate/chunks/{version}_{blind}_pure_eb_chunk_{chunk_id}.npz", + "results/paper_plots/intermediate/chunks/{version}_pure_eb_chunk_{chunk_id}.npz", params: version="{version}", - blind="{blind}", chunk_id="{chunk_id}", n_chunks=N_PURE_EB_CHUNKS, n_samples=config["covariance"]["n_samples"], @@ -246,18 +246,15 @@ rule precompute_pure_eb_chunk: rule precompute_pure_eb: """Gather MC sample chunks and compute final pure E/B covariance.""" - wildcard_constraints: - version=r"[^_]+_v[\d.]+(_ecut\d+)?(_leak_corr)?", # e.g. SP_v1.4.6, SP_v1.4.6_ecut07_leak_corr - blind=r"[ABC]", input: chunks=expand( - "results/paper_plots/intermediate/chunks/{{version}}_{{blind}}_pure_eb_chunk_{chunk_id}.npz", + "results/paper_plots/intermediate/chunks/{{version}}_pure_eb_chunk_{chunk_id}.npz", chunk_id=range(N_PURE_EB_CHUNKS), ), xi_reporting=lambda w: _xi_reporting_path(w.version), xi_integration=lambda w: _xi_integration_path(w.version), output: - "results/paper_plots/intermediate/{version}_{blind}_pure_eb_semianalytic.npz", + "results/paper_plots/intermediate/{version}_pure_eb_semianalytic.npz", params: version="{version}", **FIDUCIAL_BINNING, @@ -271,8 +268,6 @@ rule precompute_pure_eb: rule pure_eb_data_vector: """B-mode null test: Pure E/B data vector at fiducial scale cuts. - Uses fiducial blind only (FIDUCIAL["blind"]) for PTE calculation. - Produces 9 figures: - figure.png: fiducial version, leak-corrected, no title (paper) - figure_v{X.Y.Z}.png: each version, leak-corrected, with title @@ -286,10 +281,9 @@ rule pure_eb_data_vector: ], config=f"{CONFIG_DIR}/config.yaml", # Per-version inputs: pure_eb_{version} and cov_{version} for all versions - **{f"pure_eb_{ver}": f"results/paper_plots/intermediate/{ver}_{FIDUCIAL['blind']}_pure_eb_semianalytic.npz" - for ver in VERSIONS_ALL_FOR_PLOTS}, - **{f"cov_{ver}": _reporting_cov_path(ver, FIDUCIAL["blind"]) + **{f"pure_eb_{ver}": f"results/paper_plots/intermediate/{ver}_pure_eb_semianalytic.npz" for ver in VERSIONS_ALL_FOR_PLOTS}, + **{f"cov_{ver}": _reporting_cov_path(ver) for ver in VERSIONS_ALL_FOR_PLOTS}, output: evidence=f"{TAPESTRY_DIR}/pure_eb_data_vector/evidence.json", paper_figure=f"{PAPER_FIGURES_DIR}/pure_eb_data_vector.pdf", @@ -313,7 +307,7 @@ rule pure_eb_version_comparison: config=f"{CONFIG_DIR}/config.yaml", # Pure E/B only for leak-corrected versions pure_eb_data=[ - f"results/paper_plots/intermediate/{ver}_A_pure_eb_semianalytic.npz" + f"results/paper_plots/intermediate/{ver}_pure_eb_semianalytic.npz" for ver in VERSIONS_LEAK_CORR ], params: @@ -334,8 +328,6 @@ rule pure_eb_covariance: - E and B blocks are well-conditioned (~10^5) - Ambiguous blocks are ill-conditioned (~10^15, expected) - Correlation structure across 6 blocks (E+/E-/B+/B-/amb+/amb-) - - Uses blind A covariance for visualization (structure is similar across blinds). """ input: specs=[ @@ -345,7 +337,7 @@ rule pure_eb_covariance: f"{CONFIG_DIR}/2d_plots.md", ], config=f"{CONFIG_DIR}/config.yaml", - pure_eb_data=f"results/paper_plots/intermediate/{FIDUCIAL_VERSION}_A_pure_eb_semianalytic.npz", + pure_eb_data=f"results/paper_plots/intermediate/{FIDUCIAL_VERSION}_pure_eb_semianalytic.npz", output: evidence=f"{TAPESTRY_DIR}/pure_eb_covariance/evidence.json", figure=f"{TAPESTRY_DIR}/pure_eb_covariance/figure.png", @@ -357,17 +349,12 @@ rule pure_eb_covariance: rule calculate_pure_eb_ptes: """PTE matrices for pure E/B-mode scale-cut robustness. - Nothing here varies with the blind: the data vectors come from the blind-A - gather and the PTEs are Hartlap-debiased by the MC draw count, not by a - per-blind covariance. The wildcard survives as the filename slot the - consumer (config_space_pte_matrices) reads, and only blind A is ever built. + The PTEs are Hartlap-debiased by the MC draw count. """ input: - pure_eb_data="results/paper_plots/intermediate/{version}_A_pure_eb_semianalytic.npz", + pure_eb_data="results/paper_plots/intermediate/{version}_pure_eb_semianalytic.npz", output: - "results/paper_plots/intermediate/{version}_{blind}_pure_eb_ptes.npz", - wildcard_constraints: - blind=r"[ABC]", + "results/paper_plots/intermediate/{version}_pure_eb_ptes.npz", params: version="{version}", n_samples=config["covariance"]["n_samples"], @@ -441,18 +428,17 @@ rule cl_version_comparison: # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ rule compute_cosebis_pte: - """Scatter: Compute COSEBIS B-mode PTE for a single (version, blind, i_min, i_max) tuple.""" + """Scatter: Compute COSEBIS B-mode PTE for a single (version, i_min, i_max) tuple.""" input: xi_integration=lambda w: _xi_integration_path(w.version), - cov_integration=lambda w: _cov_integration_path(w.version, w.blind), + cov_integration=lambda w: _cov_integration_path(w.version), output: - pte_json=f"{TAPESTRY_DIR}/cosebis_pte_matrix/pte_values/{{version}}/{{blind}}/pte_{{i_min}}_{{i_max}}.json", + pte_json=f"{TAPESTRY_DIR}/cosebis_pte_matrix/pte_values/{{version}}/pte_{{i_min}}_{{i_max}}.json", params: nmodes=FIDUCIAL["nmodes"], wildcard_constraints: i_min=r"\d{3}", i_max=r"\d{3}", - blind=r"[ABC]", threads: 1 resources: mem_mb=8000, @@ -482,14 +468,13 @@ rule config_space_pte_matrices: # Claim dependencies pure_eb_data_vector=f"{TAPESTRY_DIR}/pure_eb_data_vector/evidence.json", cosebis_data_vector=f"{TAPESTRY_DIR}/cosebis_data_vector/evidence.json", - # Data inputs (fiducial blind only) # Pure E/B and COSEBIs PTEs for both corrected and uncorrected versions pure_eb_pte=[ - f"results/paper_plots/intermediate/{ver}_{FIDUCIAL['blind']}_pure_eb_ptes.npz" + f"results/paper_plots/intermediate/{ver}_pure_eb_ptes.npz" for ver in VERSIONS_CONFIG_SPACE_PTES ], cosebis_pte_files=[ - f"{TAPESTRY_DIR}/cosebis_pte_matrix/pte_values/{ver}/{FIDUCIAL['blind']}/pte_{i:03d}_{j:03d}.json" + f"{TAPESTRY_DIR}/cosebis_pte_matrix/pte_values/{ver}/pte_{i:03d}_{j:03d}.json" for ver in VERSIONS_CONFIG_SPACE_PTES for i, j in PTE_SCALE_CUT_PAIRS ], @@ -509,7 +494,8 @@ rule harmonic_space_pte_matrices: Results: Single-panel Cl^BB PTE matrix for fiducial version Appendix: N-panel composite for all versions from config.versions - Uses fiducial blind covariance (blind independence validated in bb_covariance_blind_independence). + n(z)-realisation independence of the BB covariance is validated in + bb_covariance_blind_independence. """ input: specs=[ @@ -520,10 +506,7 @@ rule harmonic_space_pte_matrices: config=f"{CONFIG_DIR}/config.yaml", # Harmonic PTE matrices for both corrected and uncorrected versions pseudo_cl=[_pseudo_cl_path(ver) for ver in VERSIONS_CONFIG_SPACE_PTES], - pseudo_cl_cov=[ - _pseudo_cl_cov_path(ver, blind=FIDUCIAL["blind"]) - for ver in VERSIONS_CONFIG_SPACE_PTES - ], + pseudo_cl_cov=[_pseudo_cl_cov_path(ver) for ver in VERSIONS_CONFIG_SPACE_PTES], params: version_labels=VERSION_LABELS, output: @@ -537,16 +520,17 @@ rule harmonic_space_pte_matrices: rule bb_covariance_blind_independence: - """Test BB covariance blind-independence vs EE variation. + """Test BB covariance independence of the n(z) realisation vs EE variation. - BB covariances should be stable across blinds (null signal → no sample variance). - EE covariances should vary (~10%) due to sample variance from cosmological signal. + BB covariances should be stable across the A/B/C n(z) realisations (null + signal → no sample variance). EE covariances should vary (~10%) due to + sample variance from cosmological signal. Covers all three analysis spaces: Pure E/B, COSEBIS, and harmonic (pseudo-Cl). - Uses mock_version (v1.4.6) for per-blind covariances — blind independence is a - property of survey geometry, not catalog version. Avoids generating B/C covariances - for the fiducial version. + Compares NZ_REALISATIONS, catalogues identical to the mock version but for + their n(z): the independence is a property of survey geometry, not catalog + version. """ input: specs=[ @@ -557,16 +541,17 @@ rule bb_covariance_blind_independence: f"{CONFIG_DIR}/cl.md", ], config=f"{CONFIG_DIR}/config.yaml", - # Per-blind MC-propagated pure E/B covariances (using mock_version for all blinds) - **{f"pure_eb_{b}": f"results/paper_plots/intermediate/{MOCK_VERSION}_{b}_pure_eb_semianalytic.npz" - for b in BLINDS}, - # COSEBIS: xi integration file (shared) + per-blind config-space covariances + # Per-realisation MC-propagated pure E/B covariances + **{f"pure_eb_{label}": f"results/paper_plots/intermediate/{ver}_pure_eb_semianalytic.npz" + for label, ver in NZ_REALISATIONS.items()}, + # COSEBIS: xi integration file (shared) + per-realisation config-space covariances xi_integration=_xi_integration_path(MOCK_VERSION), - **{f"cov_integration_{b}": _cov_integration_path(MOCK_VERSION, b) for b in BLINDS}, - # Per-blind harmonic covariances - **{f"harmonic_{b}": _pseudo_cl_cov_path(MOCK_VERSION, b) for b in BLINDS}, - # Pseudo-Cl data vector (blind A only) for ell bin centers - pseudo_cl=_pseudo_cl_path(MOCK_VERSION, "A"), + **{f"cov_integration_{label}": _cov_integration_path(ver) + for label, ver in NZ_REALISATIONS.items()}, + # Per-realisation harmonic covariances + **{f"harmonic_{label}": _pseudo_cl_cov_path(ver) for label, ver in NZ_REALISATIONS.items()}, + # Pseudo-Cl data vector for ell bin centers + pseudo_cl=_pseudo_cl_path(MOCK_VERSION), params: nmodes=FIDUCIAL["nmodes"], theta_min=config["cosebis"]["theta_min"], @@ -612,7 +597,7 @@ rule harmonic_config_cosebis_comparison: **{f"pseudo_cl_{ver}": _pseudo_cl_path(ver, nbins=_COSEBIS_NBINS) for ver in VERSIONS_ALL_FOR_PLOTS}, **{f"pseudo_cl_cov_{ver}": _pseudo_cl_cov_path(ver, nbins=_COSEBIS_NBINS) for ver in VERSIONS_ALL_FOR_PLOTS}, **{f"xi_{ver}": _xi_integration_path(ver) for ver in VERSIONS_ALL_FOR_PLOTS}, - **{f"cov_{ver}": _cov_integration_path(ver, FIDUCIAL["blind"]) for ver in VERSIONS_ALL_FOR_PLOTS}, + **{f"cov_{ver}": _cov_integration_path(ver) for ver in VERSIONS_ALL_FOR_PLOTS}, params: scale_cut=lambda wildcards: _COSEBIS_ANGULAR_RANGES[wildcards.angular_range], output: diff --git a/papers/bmodes/rules/presentation.smk b/papers/bmodes/rules/presentation.smk index c59f0aa3..7af2037a 100644 --- a/papers/bmodes/rules/presentation.smk +++ b/papers/bmodes/rules/presentation.smk @@ -113,9 +113,7 @@ rule presentation_s8_with_unions: rule presentation_blind_nz_plot: """Plot all three blinded n(z) curves for Moriond presentation.""" input: - nz_A=lambda w: build_redshift_path(FIDUCIAL["version"], "A"), - nz_B=lambda w: build_redshift_path(FIDUCIAL["version"], "B"), - nz_C=lambda w: build_redshift_path(FIDUCIAL["version"], "C"), + **{f"nz_{label}": redshift_path(ver) for label, ver in NZ_REALISATIONS.items()}, output: f"{TALK_DIR}/images/blind_nz_ABC.png", script: @@ -146,7 +144,7 @@ rule presentation_pte_cosebis: """COSEBIS B_n PTE heatmap for Moriond talk (single panel, talk-sized).""" input: pte_files=[ - f"{TAPESTRY_DIR}/cosebis_pte_matrix/pte_values/{FIDUCIAL['version']}/{FIDUCIAL['blind']}/pte_{i:03d}_{j:03d}.json" + f"{TAPESTRY_DIR}/cosebis_pte_matrix/pte_values/{FIDUCIAL['version']}/pte_{i:03d}_{j:03d}.json" for i, j in PTE_SCALE_CUT_PAIRS ], output: diff --git a/papers/bmodes/scripts/bb_covariance_blind_independence.py b/papers/bmodes/scripts/bb_covariance_blind_independence.py index 8868baab..0cabd756 100644 --- a/papers/bmodes/scripts/bb_covariance_blind_independence.py +++ b/papers/bmodes/scripts/bb_covariance_blind_independence.py @@ -1,8 +1,10 @@ """ BB Covariance Blind Independence Claim -Tests whether BB covariances are blind-independent (as expected for null signals) -while EE covariances vary across blinds (due to sample variance from cosmological signal). +Tests whether BB covariances are independent of the A/B/C n(z) realisation (as +expected for null signals) while EE covariances vary across realisations (due to +sample variance from cosmological signal). Each realisation is a catalogue entry +of its own (config fiducial.nz_realisations), identical but for its n(z). Compares: - Pure E/B: cov(xi+^B), cov(xi-^B) vs cov(xi+^E), cov(xi-^E) @@ -29,7 +31,8 @@ plt.style.use(PAPER_MPLSTYLE) -BLINDS = ["A", "B", "C"] +# n(z) realisation labels: the keys of config fiducial.nz_realisations. +NZ_LABELS = ["A", "B", "C"] def load_pure_eb_diagonals(path): @@ -399,21 +402,21 @@ def main( """Run the BB-covariance blind-independence cross-check. ``pure_eb_paths`` / ``harmonic_paths`` / ``cov_integration_paths`` are dicts - keyed by blind (A/B/C). All paths are absolute; the per-blind mock-version + keyed by n(z) realisation (A/B/C). All paths are absolute; the per-realisation covariances are read directly, so the check is self-contained. """ version = config["fiducial"]["mock_version"] # Load pure E/B covariances for all blinds pure_eb_data = {} - for blind in BLINDS: + for blind in NZ_LABELS: pure_eb_data[blind] = load_pure_eb_diagonals(pure_eb_paths[blind]) theta = pure_eb_data["A"]["theta"] # Load harmonic covariances for all blinds harmonic_data = {} - for blind in BLINDS: + for blind in NZ_LABELS: harmonic_data[blind] = load_harmonic_diagonals(harmonic_paths[blind]) # Integration binning parameters from config @@ -422,7 +425,7 @@ def main( nbins_int = config["fiducial"]["nbins_int"] cosebis_data = {} - for blind in BLINDS: + for blind in NZ_LABELS: cosebis_data[blind] = load_cosebis_diagonals( xi_integration_path, cov_integration_paths[blind], @@ -617,9 +620,9 @@ def clean_ratios(d): def _from_snakemake(smk): config = smk.config - pure_eb_paths = {b: smk.input[f"pure_eb_{b}"] for b in BLINDS} - harmonic_paths = {b: smk.input[f"harmonic_{b}"] for b in BLINDS} - cov_integration_paths = {b: smk.input[f"cov_integration_{b}"] for b in BLINDS} + pure_eb_paths = {b: smk.input[f"pure_eb_{b}"] for b in NZ_LABELS} + harmonic_paths = {b: smk.input[f"harmonic_{b}"] for b in NZ_LABELS} + cov_integration_paths = {b: smk.input[f"cov_integration_{b}"] for b in NZ_LABELS} main( config=config, pure_eb_paths=pure_eb_paths, @@ -635,12 +638,11 @@ def _from_snakemake(smk): ) -def _cov_integration_path(cov_dir, version, blind, min_sep, max_sep, nbins): +def _cov_integration_path(cov_dir, version, min_sep, max_sep, nbins): """Reproduce common.covariance_path for the Gaussian integration-grid, masked covariance (suffix _processed.txt).""" base = ( - f"covariance_{version}_{blind}_g" - f"_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_masked" + f"covariance_{version}_g_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_masked" ) return os.path.join(cov_dir, base, f"{base}_processed.txt") @@ -658,7 +660,7 @@ def _from_cli(argv=None): ap.add_argument( "--pure-eb-dir", required=True, - help="Dir with {version}_{blind}_pure_eb_semianalytic.npz", + help="Dir with {version}_pure_eb_semianalytic.npz per n(z) realisation", ) ap.add_argument( "--cosmo-val-dir", @@ -688,22 +690,20 @@ def _from_cli(argv=None): ) npatch = fid["npatch"] + realisations = fid["nz_realisations"] pure_eb_paths = { - b: os.path.join(a.pure_eb_dir, f"{version}_{b}_pure_eb_semianalytic.npz") - for b in BLINDS + b: os.path.join(a.pure_eb_dir, f"{ver}_pure_eb_semianalytic.npz") + for b, ver in realisations.items() } harmonic_paths = { - b: os.path.join( - a.cosmo_val_dir, - f"pseudo_cl_cov_{version}_blind={b}_powspace_nbins=32.fits", - ) - for b in BLINDS + b: os.path.join(a.cosmo_val_dir, f"pseudo_cl_cov_{ver}_powspace_nbins=32.fits") + for b, ver in realisations.items() } cov_integration_paths = { b: _cov_integration_path( - a.covariance_dir, version, b, min_sep_int, max_sep_int, nbins_int + a.covariance_dir, ver, min_sep_int, max_sep_int, nbins_int ) - for b in BLINDS + for b, ver in realisations.items() } xi_integration_path = os.path.join( a.cosmo_val_dir, @@ -711,7 +711,7 @@ def _from_cli(argv=None): f"_nbins={nbins_int}_npatch={npatch}.sacc", ) pseudo_cl_path = os.path.join( - a.cosmo_val_dir, f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" + a.cosmo_val_dir, f"pseudo_cl_{version}_powspace_nbins=32.sacc" ) out_dir = Path(a.out) diff --git a/papers/bmodes/scripts/calculate_pure_eb_ptes.py b/papers/bmodes/scripts/calculate_pure_eb_ptes.py index 3a623669..feb0be78 100644 --- a/papers/bmodes/scripts/calculate_pure_eb_ptes.py +++ b/papers/bmodes/scripts/calculate_pure_eb_ptes.py @@ -5,10 +5,10 @@ ξ_+^B / ξ_-^B / joint ξ_tot^B χ² PTE matrices over the scale-cut grid via ``sp_validation.b_modes.calculate_eb_statistics`` (Hartlap-corrected inverse MC covariance, debiased by the draw count), and writes the PTE matrices to -``{out}/{version}_{blind}_pure_eb_ptes.npz``. +``{out}/{version}_pure_eb_ptes.npz``. python calculate_pure_eb_ptes.py \ - --version SP_v1.4.6.3_leak_corr --blind A \ + --version SP_v1.4.6.3_leak_corr \ --pure-eb-data <..._pure_eb_semianalytic.npz> \ --n-samples 2000 --out """ @@ -23,7 +23,6 @@ def calculate_ptes( version, - blind, pure_eb_data, n_samples, output_dir, @@ -57,7 +56,7 @@ def calculate_ptes( } os.makedirs(output_dir, exist_ok=True) - out_path = os.path.join(output_dir, f"{version}_{blind}_pure_eb_ptes.npz") + out_path = os.path.join(output_dir, f"{version}_pure_eb_ptes.npz") np.savez(out_path, **output_data) print(f"Saved PTE matrices to {out_path}") return out_path @@ -66,14 +65,12 @@ def calculate_ptes( def _from_cli(argv=None): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("--version", required=True) - ap.add_argument("--blind", default="A") ap.add_argument("--pure-eb-data", required=True, help="Gathered semianalytic .npz") ap.add_argument("--n-samples", type=int, default=2000) ap.add_argument("--out", required=True, help="Output directory (lc {output})") a = ap.parse_args(argv) calculate_ptes( version=a.version, - blind=a.blind, pure_eb_data=a.pure_eb_data, n_samples=a.n_samples, output_dir=a.out, diff --git a/papers/bmodes/scripts/cl_data_vector.py b/papers/bmodes/scripts/cl_data_vector.py index 4eeca960..724c38cb 100644 --- a/papers/bmodes/scripts/cl_data_vector.py +++ b/papers/bmodes/scripts/cl_data_vector.py @@ -161,14 +161,12 @@ def _create_cl_figure( # via _pseudo_cl_path() in claims.smk; the CLI reconstructs them from --results-dir # so the version sweep is self-contained (lc produces only the fiducial version). # --------------------------------------------------------------------------- -def _pseudo_cl(results_dir, ver, blind="A", nbins=32): - return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.sacc" +def _pseudo_cl(results_dir, ver, nbins=32): + return f"{results_dir}/pseudo_cl_{ver}_powspace_nbins={nbins}.sacc" -def _pseudo_cl_cov(results_dir, ver, blind="A", nbins=32): - return ( - f"{results_dir}/pseudo_cl_cov_{ver}_blind={blind}_powspace_nbins={nbins}.fits" - ) +def _pseudo_cl_cov(results_dir, ver, nbins=32): + return f"{results_dir}/pseudo_cl_cov_{ver}_powspace_nbins={nbins}.fits" def _resolve_pseudo_cl_paths( @@ -177,7 +175,6 @@ def _resolve_pseudo_cl_paths( fiducial_version=None, fiducial_pseudo_cl_path=None, fiducial_pseudo_cl_cov_path=None, - blind="A", nbins=32, ): """Resolve the pseudo-Cl and covariance paths for one version in the sweep. @@ -194,8 +191,8 @@ def _resolve_pseudo_cl_paths( ): return fiducial_pseudo_cl_path, fiducial_pseudo_cl_cov_path return ( - _pseudo_cl(results_dir, ver, blind, nbins), - _pseudo_cl_cov(results_dir, ver, blind, nbins), + _pseudo_cl(results_dir, ver, nbins), + _pseudo_cl_cov(results_dir, ver, nbins), ) diff --git a/papers/bmodes/scripts/cl_version_comparison.py b/papers/bmodes/scripts/cl_version_comparison.py index f70a6977..7f82d14f 100644 --- a/papers/bmodes/scripts/cl_version_comparison.py +++ b/papers/bmodes/scripts/cl_version_comparison.py @@ -34,14 +34,12 @@ plt.style.use(PAPER_MPLSTYLE) -def _pseudo_cl(results_dir, ver, blind="A", nbins=32): - return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.sacc" +def _pseudo_cl(results_dir, ver, nbins=32): + return f"{results_dir}/pseudo_cl_{ver}_powspace_nbins={nbins}.sacc" -def _pseudo_cl_cov(results_dir, ver, blind="A", nbins=32): - return ( - f"{results_dir}/pseudo_cl_cov_{ver}_blind={blind}_powspace_nbins={nbins}.fits" - ) +def _pseudo_cl_cov(results_dir, ver, nbins=32): + return f"{results_dir}/pseudo_cl_cov_{ver}_powspace_nbins={nbins}.fits" def _resolve_pseudo_cl_paths( @@ -50,7 +48,6 @@ def _resolve_pseudo_cl_paths( fiducial_version=None, fiducial_pseudo_cl_path=None, fiducial_pseudo_cl_cov_path=None, - blind="A", nbins=32, ): """Resolve the pseudo-Cl and covariance paths for one version in the sweep. @@ -67,8 +64,8 @@ def _resolve_pseudo_cl_paths( ): return fiducial_pseudo_cl_path, fiducial_pseudo_cl_cov_path return ( - _pseudo_cl(results_dir, ver, blind, nbins), - _pseudo_cl_cov(results_dir, ver, blind, nbins), + _pseudo_cl(results_dir, ver, nbins), + _pseudo_cl_cov(results_dir, ver, nbins), ) diff --git a/papers/bmodes/scripts/compute_cosebis_pte_single.py b/papers/bmodes/scripts/compute_cosebis_pte_single.py index fcfe9ec0..71f43da8 100644 --- a/papers/bmodes/scripts/compute_cosebis_pte_single.py +++ b/papers/bmodes/scripts/compute_cosebis_pte_single.py @@ -2,7 +2,7 @@ """Compute the COSEBIS B-mode PTE matrix over the full scale-cut pair grid. Reconciliation of the ASTRA ``cosebis_pte_per_cut`` output: the Snakemake DAG -*scattered* one JSON per (version, blind, i_min, i_max) cut via this script and +*scattered* one JSON per (version, i_min, i_max) cut via this script and gathered them downstream. The spec describes the gathered NPZ that scatter never materialised. Here a single CLI loops the same ``_pte_scale_cut_pairs()`` grid, runs the *identical* per-pair COSEBI computation (same theta grid, nmodes, NaN-on- @@ -79,15 +79,14 @@ def _compute_pair(gg, cov_path, nmodes, theta_min, theta_max): } -def main(config, xi_integration, cov_integration, out_dir, version=None, blind=None): +def main(config, xi_integration, cov_integration, out_dir, version=None): t_start = time.time() fid = config["fiducial"] - # Version/blind tag only the output filename + provenance record; the theta + # Version tags only the output filename + provenance record; the theta # grid, nmodes and per-pair COSEBI compute are version-independent (they read # from config["fiducial"]), so a sweep call over a non-fiducial catalog stays # bit-identical to the fiducial call save for the xi/cov inputs and the tag. version = version if version is not None else fid["version"] - blind = blind if blind is not None else fid["blind"] nmodes = int(fid["nmodes"]) # 20 for full computation gg = sacc_io.xi_correlation(sacc_io.load(xi_integration)) @@ -139,11 +138,10 @@ def _mat(): out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) - npz_path = out_dir / f"cosebis_ptes_{version}_{blind}.npz" + npz_path = out_dir / f"cosebis_ptes_{version}.npz" np.savez( npz_path, version=version, - blind=blind, nmodes=nmodes, mode_subsets=np.array([6, 20]), theta_grid=theta_grid, @@ -170,7 +168,7 @@ def _from_cli(argv=None): import yaml ap = argparse.ArgumentParser( - description="Gathered COSEBI B-mode PTE matrix over the scale-cut pair grid (fiducial version + blind)." + description="Gathered COSEBI B-mode PTE matrix over the scale-cut pair grid (fiducial version)." ) ap.add_argument( "--config", required=True, help="Absolute path to bmodes config.yaml" @@ -192,11 +190,6 @@ def _from_cli(argv=None): help="Catalog version tag (default: config.fiducial.version). Overridden " "by the version sweep to name the non-fiducial output NPZ.", ) - ap.add_argument( - "--blind", - default=None, - help="Blind tag (default: config.fiducial.blind)", - ) a = ap.parse_args(argv) with open(a.config) as f: config = yaml.safe_load(f) @@ -206,7 +199,6 @@ def _from_cli(argv=None): a.cov_integration, a.out, version=a.version, - blind=a.blind, ) diff --git a/papers/bmodes/scripts/config_space_pte_matrices.py b/papers/bmodes/scripts/config_space_pte_matrices.py index 474b71a7..e63b9ace 100644 --- a/papers/bmodes/scripts/config_space_pte_matrices.py +++ b/papers/bmodes/scripts/config_space_pte_matrices.py @@ -18,7 +18,7 @@ --config /path/to/config.yaml \ --pte-intermediate-dir /abs/.../paper_plots/intermediate \ --cosebis-pte-dir /abs/.../tapestry/cosebis_pte_matrix/pte_values \ - --blind A --out + --out """ import argparse @@ -76,7 +76,7 @@ def _resolve_overrides(version, fiducial_overrides): the sweep versions on the *same* gathered-NPZ provenance (adapted back to the per-pair PTE matrix by ``_cosebis_matrix_from_npz``) rather than the old per-pair-JSON scatter tree. Pure E/B has no such split — the sweep emits the - old-tree ``{ver}_{blind}_pure_eb_ptes.npz`` name, so non-fiducial versions + old-tree ``{ver}_pure_eb_ptes.npz`` name, so non-fiducial versions read it straight from the ``--pte-intermediate-dir`` file list (override None). Either element may be None. """ @@ -112,12 +112,10 @@ def load_cosebis_pte_matrix( ): """Load COSEBIS PTE values from JSON files into matrix. - Uses fiducial blind only (data vectors identical across blinds). - Parameters ---------- pte_files : list of str - Paths to PTE JSON files for fiducial blind. + Paths to PTE JSON files. version : str Version to filter for. config : dict @@ -174,12 +172,10 @@ def load_cosebis_pte_matrix( def load_pure_eb_pte_matrices(pte_files, version, override_path=None): """Load Pure E/B PTE matrices from npz files. - Uses fiducial blind only (data vectors identical across blinds). - Parameters ---------- pte_files : list of str - Paths to pure_eb_ptes.npz files for fiducial blind. + Paths to pure_eb_ptes.npz files. version : str Version to filter for. @@ -368,12 +364,12 @@ def extract_full_range_ptes( ): """Extract full-range PTEs from npz and JSON files. - Takes minimum PTE across blinds for each statistic. + Takes minimum PTE across matching files for each statistic. Parameters ---------- pure_eb_pte_files : list - All Pure E/B PTE npz files (includes all blinds). + All Pure E/B PTE npz files. cosebis_pte_files : list All COSEBIS PTE JSON files. version : str @@ -382,11 +378,11 @@ def extract_full_range_ptes( Returns ------- ptes : dict - Full-range PTEs for xip, xim, and cosebis (fiducial blind). + Full-range PTEs for xip, xim, and cosebis. """ pure_eb_override, cosebis_override = _resolve_overrides(version, fiducial_overrides) - # Get full-range PTEs from pure E/B (fiducial blind) + # Get full-range PTEs from pure E/B xip_ptes = [] xim_ptes = [] combined_ptes = [] @@ -428,7 +424,7 @@ def extract_full_range_ptes( ptes["cosebis_20"] = pte20 return ptes - # Old tree: pte_000_020.json for full theta range, min across blinds. + # Old tree: pte_000_020.json for full theta range. cosebis_ptes_6 = [] cosebis_ptes_20 = [] for pte_file in cosebis_pte_files: @@ -474,7 +470,7 @@ def create_3panel_composite( version : str Catalog version string (fiducial). pure_eb_pte_files : list - All Pure E/B PTE npz files (includes all blinds). + All Pure E/B PTE npz files. cosebis_pte_files : list All COSEBIS PTE JSON files. xip_fid, xim_fid : tuple @@ -625,7 +621,7 @@ def create_9panel_composite( versions : list of str Catalog version strings in display order. pure_eb_pte_files : list - All Pure E/B PTE npz files (includes all blinds). + All Pure E/B PTE npz files. cosebis_pte_files : list All COSEBIS PTE JSON files. xip_fid, xim_fid : tuple @@ -1050,16 +1046,15 @@ def _from_cli(argv=None): ap.add_argument( "--pte-intermediate-dir", required=True, - help="Directory holding {version}_{blind}_pure_eb_ptes.npz", + help="Directory holding {version}_pure_eb_ptes.npz", ) ap.add_argument( "--cosebis-pte-dir", required=True, help="COSEBI PTE source dir. lc cosebis_ptes sweep: gathered " - "cosebis_ptes_{version}_{blind}.npz per version (auto-detected, preferred). " - "Old tree fallback: {version}/{blind}/pte_{i:03d}_{j:03d}.json scatter.", + "cosebis_ptes_{version}.npz per version (auto-detected, preferred). " + "Old tree fallback: {version}/pte_{i:03d}_{j:03d}.json scatter.", ) - ap.add_argument("--blind", default="A", help="Fiducial blind (paper: A)") ap.add_argument("--out", required=True, help="Output directory (lc {output})") ap.add_argument( "--fiducial-version", @@ -1071,7 +1066,7 @@ def _from_cli(argv=None): "--fiducial-pure-eb-pte-path", default=None, help="lc pure_eb PTE NPZ for the fiducial version " - "(same format as old-tree {version}_{blind}_pure_eb_ptes.npz)", + "(same format as old-tree {version}_pure_eb_ptes.npz)", ) ap.add_argument( "--fiducial-cosebis-pte-path", @@ -1092,8 +1087,7 @@ def _from_cli(argv=None): versions = _versions_config_space(config) pure_eb_pte_files = [ - os.path.join(a.pte_intermediate_dir, f"{v}_{a.blind}_pure_eb_ptes.npz") - for v in versions + os.path.join(a.pte_intermediate_dir, f"{v}_pure_eb_ptes.npz") for v in versions ] pure_eb_pte_files = [p for p in pure_eb_pte_files if os.path.exists(p)] @@ -1101,18 +1095,14 @@ def _from_cli(argv=None): # version, same layout as the fiducial single-output) so fiducial and sweep # versions share provenance; fall back to the old per-pair-JSON scatter tree. cosebis_by_version = { - v: os.path.join(a.cosebis_pte_dir, f"cosebis_ptes_{v}_{a.blind}.npz") + v: os.path.join(a.cosebis_pte_dir, f"cosebis_ptes_{v}.npz") for v in versions - if os.path.exists( - os.path.join(a.cosebis_pte_dir, f"cosebis_ptes_{v}_{a.blind}.npz") - ) + if os.path.exists(os.path.join(a.cosebis_pte_dir, f"cosebis_ptes_{v}.npz")) } cosebis_pte_files = ( [] if cosebis_by_version - else sorted( - glob.glob(os.path.join(a.cosebis_pte_dir, "*", a.blind, "pte_*.json")) - ) + else sorted(glob.glob(os.path.join(a.cosebis_pte_dir, "*", "pte_*.json"))) ) fiducial_overrides = None diff --git a/papers/bmodes/scripts/cosebis_version_comparison.py b/papers/bmodes/scripts/cosebis_version_comparison.py index 9113cf5e..a912b31c 100644 --- a/papers/bmodes/scripts/cosebis_version_comparison.py +++ b/papers/bmodes/scripts/cosebis_version_comparison.py @@ -151,8 +151,8 @@ def _xi_integration(results_dir, ver): return f"{results_dir}/{ver}_xi_minsep=0.5_maxsep=300.0_nbins=1000_npatch=1.sacc" -def _cov_integration(cov_dir, ver, blind): - base = f"covariance_{ver}_{blind}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" +def _cov_integration(cov_dir, ver): + base = f"covariance_{ver}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" return f"{cov_dir}/{base}/{base}_processed.txt" @@ -170,7 +170,6 @@ def main( nmodes = config["fiducial"]["nmodes"] plotting_config = config["plotting"] version_labels = plotting_config["version_labels"] - blind = config["fiducial"]["blind"] # Fiducial version whose inputs may be overridden with explicit lc paths fiducial_version = fiducial_version or config["fiducial"]["version"] @@ -186,7 +185,7 @@ def main( cov_paths_list = [ fiducial_cov_path if v == fiducial_version and fiducial_cov_path - else _cov_integration(cov_dir, v, blind) + else _cov_integration(cov_dir, v) for v in versions ] diff --git a/papers/bmodes/scripts/gather_pure_eb_chunks.py b/papers/bmodes/scripts/gather_pure_eb_chunks.py index ddd6eace..f4a2837c 100644 --- a/papers/bmodes/scripts/gather_pure_eb_chunks.py +++ b/papers/bmodes/scripts/gather_pure_eb_chunks.py @@ -3,12 +3,12 @@ CLI refactor of the former Snakemake ``script:`` gather rule. Reads the actual ξ± data vectors (reporting + integration grids), computes the pure E/B/ambiguous decomposition (Schneider 2022), stacks the per-chunk MC sample blocks, and -forms the empirical 6-block covariance. Writes the per-(version, blind) -``__pure_eb_semianalytic.npz`` consumed by every downstream +forms the empirical 6-block covariance. Writes the per-version +``_pure_eb_semianalytic.npz`` consumed by every downstream pure-mode plot / PTE. python gather_pure_eb_chunks.py \ - --version SP_v1.4.6.3_leak_corr --blind A \ + --version SP_v1.4.6.3_leak_corr \ --xi-reporting \ --xi-integration \ --chunks-dir \ @@ -34,7 +34,6 @@ def _load_xi(path): def gather( version, - blind, xi_reporting, xi_integration, chunk_files, @@ -46,7 +45,7 @@ def gather( ): from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes - print(f"Gathering pure E/B for blind {blind}") + print(f"Gathering pure E/B for {version}") gg = _load_xi(xi_reporting) gg_int = _load_xi(xi_integration) @@ -90,7 +89,7 @@ def gather( } os.makedirs(output_dir, exist_ok=True) - out_path = os.path.join(output_dir, f"{version}_{blind}_pure_eb_semianalytic.npz") + out_path = os.path.join(output_dir, f"{version}_pure_eb_semianalytic.npz") np.savez(out_path, **package) print(f"Saved to {out_path}") return out_path @@ -109,7 +108,6 @@ def _resolve_chunks(args): def _from_cli(argv=None): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("--version", required=True) - ap.add_argument("--blind", default="A") ap.add_argument("--xi-reporting", required=True) ap.add_argument("--xi-integration", required=True) ap.add_argument("--chunks-dir", help="Directory holding pure_eb_chunk_*.npz") @@ -125,7 +123,6 @@ def _from_cli(argv=None): a = ap.parse_args(argv) gather( version=a.version, - blind=a.blind, xi_reporting=a.xi_reporting, xi_integration=a.xi_integration, chunk_files=_resolve_chunks(a), diff --git a/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py b/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py index e37bd34b..c7673bfd 100644 --- a/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py +++ b/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py @@ -797,12 +797,11 @@ def _versions_all_for_plots(config): return leak_corr + uncorrected -def _cov_integration_path(cov_dir, version, blind, min_sep, max_sep, nbins): +def _cov_integration_path(cov_dir, version, min_sep, max_sep, nbins): """Reproduce common.covariance_path for the Gaussian integration-grid, masked covariance (suffix _processed.txt).""" base = ( - f"covariance_{version}_{blind}_g" - f"_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_masked" + f"covariance_{version}_g_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_masked" ) return os.path.join(cov_dir, base, f"{base}_processed.txt") @@ -857,9 +856,6 @@ def _from_cli(argv=None): required=True, help="COSMO_INFERENCE data/covariance dir (Gaussian integration covariances)", ) - ap.add_argument( - "--blind", default="A", help="Blind for pseudo-Cl / covariance (paper: A)" - ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") ap.add_argument( "--fiducial-version", @@ -928,7 +924,7 @@ def _from_cli(argv=None): if is_fiducial and a.fiducial_pseudo_cl_path else os.path.join( a.cosmo_val_dir, - f"pseudo_cl_{ver}_blind={a.blind}_powspace_nbins={cosebis_nbins}.sacc", + f"pseudo_cl_{ver}_powspace_nbins={cosebis_nbins}.sacc", ) ) # 96-bin pseudo-Cl covariance is intentionally NOT lc-repointed: lc did not @@ -937,7 +933,7 @@ def _from_cli(argv=None): # version. inputs[f"pseudo_cl_cov_{ver}"] = os.path.join( a.cosmo_val_dir, - f"pseudo_cl_cov_{ver}_blind={a.blind}_powspace_nbins={cosebis_nbins}.fits", + f"pseudo_cl_cov_{ver}_powspace_nbins={cosebis_nbins}.fits", ) inputs[f"xi_{ver}"] = ( a.fiducial_xi_path @@ -952,7 +948,7 @@ def _from_cli(argv=None): a.fiducial_cov_path if is_fiducial and a.fiducial_cov_path else _cov_integration_path( - a.covariance_dir, ver, a.blind, min_sep_int, max_sep_int, nbins_int + a.covariance_dir, ver, min_sep_int, max_sep_int, nbins_int ) ) diff --git a/papers/bmodes/scripts/harmonic_space_pte_matrices.py b/papers/bmodes/scripts/harmonic_space_pte_matrices.py index aeb031c6..54c353f0 100644 --- a/papers/bmodes/scripts/harmonic_space_pte_matrices.py +++ b/papers/bmodes/scripts/harmonic_space_pte_matrices.py @@ -3,8 +3,6 @@ Produces: - Results: single-panel fiducial Cl^BB PTE matrix - Appendix: N-panel composite for all versions from config.versions - -Uses fiducial blind covariance only (blind independence validated elsewhere). """ import argparse @@ -31,14 +29,12 @@ plt.style.use(PAPER_MPLSTYLE) -def _pseudo_cl(results_dir, ver, blind="A", nbins=32): - return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.sacc" +def _pseudo_cl(results_dir, ver, nbins=32): + return f"{results_dir}/pseudo_cl_{ver}_powspace_nbins={nbins}.sacc" -def _pseudo_cl_cov(results_dir, ver, blind="A", nbins=32): - return ( - f"{results_dir}/pseudo_cl_cov_{ver}_blind={blind}_powspace_nbins={nbins}.fits" - ) +def _pseudo_cl_cov(results_dir, ver, nbins=32): + return f"{results_dir}/pseudo_cl_cov_{ver}_powspace_nbins={nbins}.fits" def compute_pte_matrix( @@ -355,7 +351,6 @@ def main( ): versions = [v for v in config["versions"] if "_ecut" not in v] fiducial_version = config["fiducial"]["version"] - fiducial_blind = config["fiducial"]["blind"] # Fiducial ell cuts from config fiducial_ell_min = config["cl"]["fiducial_ell_min"] @@ -365,16 +360,16 @@ def main( output_dir = Path(out_dir) output_dir.mkdir(parents=True, exist_ok=True) - # Per-version pseudo-Cl / covariance files (canonical COSMO_VAL tree, blind A). - # Mirrors _pseudo_cl_path(ver) / _pseudo_cl_cov_path(ver, blind) in claims.smk. + # Per-version pseudo-Cl / covariance files (canonical COSMO_VAL tree). + # Mirrors _pseudo_cl_path(ver) / _pseudo_cl_cov_path(ver) in claims.smk. # Fiducial-provenance repoint: when --fiducial-version matches and an explicit # lc override path is set, read that path instead of the reconstructed pattern - # (lc files lack the blind=/powspace_nbins= tokens, distinguished by directory). + # (lc files lack the powspace_nbins= token, distinguished by directory). version_to_cl = { ver: ( fiducial_pseudo_cl_path if ver == fiducial_version_override and fiducial_pseudo_cl_path is not None - else _pseudo_cl(results_dir, ver, blind=fiducial_blind) + else _pseudo_cl(results_dir, ver) ) for ver in versions } @@ -383,18 +378,18 @@ def main( fiducial_pseudo_cl_cov_path if ver == fiducial_version_override and fiducial_pseudo_cl_cov_path is not None - else _pseudo_cl_cov(results_dir, ver, blind=fiducial_blind) + else _pseudo_cl_cov(results_dir, ver) ) for ver in versions } - # Compute PTE matrices for all versions using fiducial blind + # Compute PTE matrices for all versions all_stats = {} all_matrices = {} all_ells = {} for version in versions: - print(f"\n--- Processing {version} (blind={fiducial_blind}) ---") + print(f"\n--- Processing {version} ---") if version not in version_to_cl: print(f" WARNING: No pseudo-Cl file found for {version}, skipping") @@ -425,14 +420,13 @@ def main( npz_payload = { "versions": np.array(list(all_matrices.keys())), "fiducial_version": fiducial_version, - "fiducial_blind": fiducial_blind, "fiducial_ell_min": fiducial_ell_min, "fiducial_ell_max": fiducial_ell_max, } for version in all_matrices: npz_payload[f"{version}__pte_matrix"] = all_matrices[version] npz_payload[f"{version}__ell"] = all_ells[version] - npz_path = output_dir / f"cl_pte_matrices_{fiducial_blind}.npz" + npz_path = output_dir / "cl_pte_matrices.npz" np.savez(npz_path, **npz_payload) print(f"Saved PTE matrices to {npz_path}") @@ -494,7 +488,6 @@ def main( "spec_id": "harmonic_space_pte_matrices", "generated": datetime.now().isoformat(), "evidence": { - "blind": fiducial_blind, "versions": {}, }, "output": { diff --git a/papers/bmodes/scripts/precompute_pure_eb_chunk.py b/papers/bmodes/scripts/precompute_pure_eb_chunk.py index 8ea6982a..da9bf2a5 100644 --- a/papers/bmodes/scripts/precompute_pure_eb_chunk.py +++ b/papers/bmodes/scripts/precompute_pure_eb_chunk.py @@ -12,7 +12,7 @@ python precompute_pure_eb_chunk.py \ --chunk-id 0 --n-chunks 20 --n-samples 2000 \ - --version SP_v1.4.6.3_leak_corr --blind A \ + --version SP_v1.4.6.3_leak_corr \ --cat-config /path/cosmo_val/cat_config.yaml \ --xi-reporting \ --xi-integration \ @@ -63,7 +63,6 @@ def compute_chunk( n_chunks, n_samples_total, version, - blind, cat_config, xi_reporting, xi_integration, @@ -104,10 +103,8 @@ def compute_chunk( catalog_config=cat_config, output_dir=output_dir, ) - cv.blind = blind z, nz = cv.get_redshift(version) z_dist = np.column_stack([z, nz]) - print(f"Using n(z) for blind {blind}") cosmo_cov = _build_cosmology(cosmo_params) @@ -175,7 +172,6 @@ def _from_cli(argv=None): ap.add_argument("--n-chunks", type=int, default=20) ap.add_argument("--n-samples", type=int, default=2000) ap.add_argument("--version", required=True) - ap.add_argument("--blind", default="A") ap.add_argument("--cat-config", required=True) ap.add_argument("--xi-reporting", required=True) ap.add_argument("--xi-integration", required=True) @@ -194,7 +190,6 @@ def _from_cli(argv=None): n_chunks=a.n_chunks, n_samples_total=a.n_samples, version=a.version, - blind=a.blind, cat_config=a.cat_config, xi_reporting=a.xi_reporting, xi_integration=a.xi_integration, diff --git a/papers/bmodes/scripts/pure_eb_covariance.py b/papers/bmodes/scripts/pure_eb_covariance.py index ee5ac77c..f978f6e4 100644 --- a/papers/bmodes/scripts/pure_eb_covariance.py +++ b/papers/bmodes/scripts/pure_eb_covariance.py @@ -191,7 +191,6 @@ def main(config, pure_eb_path, out_dir, specs=()): }, "parameters": { "version": version, - "blind": config["fiducial"]["blind"], }, "output": { "figure": "figure.png", @@ -229,7 +228,7 @@ def _from_cli(argv=None): ap.add_argument( "--pure-eb-data", required=True, - help="Fiducial __pure_eb_semianalytic.npz " + help="Fiducial _pure_eb_semianalytic.npz " "(provides the 6-block cov_pure_eb)", ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") diff --git a/papers/bmodes/scripts/pure_eb_data_vector.py b/papers/bmodes/scripts/pure_eb_data_vector.py index dbbaeda7..5c0c2dda 100644 --- a/papers/bmodes/scripts/pure_eb_data_vector.py +++ b/papers/bmodes/scripts/pure_eb_data_vector.py @@ -2,12 +2,12 @@ Fiducial catalog only: pure E/B/ambiguous decomposition of ξ± with B-modes consistent with zero at the fiducial scale cuts. Writes evidence.json with PTE -values including the joint B-mode test (fiducial blind, config.fiducial.blind). +values including the joint B-mode test. CLI: python pure_eb_data_vector.py \ --config config.yaml \ - --pure-eb-data __pure_eb_semianalytic.npz \ + --pure-eb-data _pure_eb_semianalytic.npz \ --reporting-cov /covariance_processed.txt \ --out [--specs spec.md ...] """ @@ -218,7 +218,6 @@ def setup_panel(ax, ylabel_text=None): def main(config, pure_eb_path, cov_path, out_dir, specs=()): - blind = config["fiducial"]["blind"] version = config["fiducial"]["version"] fiducial_xip_scale_cut = tuple(config["fiducial"]["fiducial_xip_scale_cut"]) fiducial_xim_scale_cut = tuple(config["fiducial"]["fiducial_xim_scale_cut"]) @@ -292,7 +291,7 @@ def main(config, pure_eb_path, cov_path, out_dir, specs=()): ) print( - f"Blind {blind} PTEs (fiducial): xi+^B={pte_xip_fid:.3f}, xi-^B={pte_xim_fid:.3f}, joint={pte_joint_fid:.3f}" + f"PTEs (fiducial): xi+^B={pte_xip_fid:.3f}, xi-^B={pte_xim_fid:.3f}, joint={pte_joint_fid:.3f}" ) # Write evidence.json (based on leak-corrected fiducial data only) @@ -319,7 +318,6 @@ def main(config, pure_eb_path, cov_path, out_dir, specs=()): "dof_joint_B": int(dof_joint_full), }, "version": version, - "blind": blind, }, "output": {"figure": "figure.png"}, } @@ -342,7 +340,7 @@ def _from_cli(argv=None): ap.add_argument( "--pure-eb-data", required=True, - help="Fiducial __pure_eb_semianalytic.npz " + help="Fiducial _pure_eb_semianalytic.npz " "(decomposed ξ± + 6-block MC covariance)", ) ap.add_argument( diff --git a/papers/bmodes/scripts/pure_eb_version_comparison.py b/papers/bmodes/scripts/pure_eb_version_comparison.py index 8027189f..3f97db43 100644 --- a/papers/bmodes/scripts/pure_eb_version_comparison.py +++ b/papers/bmodes/scripts/pure_eb_version_comparison.py @@ -242,8 +242,8 @@ def _create_version_comparison_figure( return fig -def _pure_eb_npz(results_dir, ver, blind): - return f"{results_dir}/{ver}_{blind}_pure_eb_semianalytic.npz" +def _pure_eb_npz(results_dir, ver): + return f"{results_dir}/{ver}_pure_eb_semianalytic.npz" def main( @@ -258,7 +258,6 @@ def main( version_labels = plotting_config["version_labels"] # Leak-corrected, non-ecut versions (matches VERSIONS_LEAK_CORR in claims.smk) versions = [v for v in config["versions"] if "_leak_corr" in v and "_ecut" not in v] - blind = config["fiducial"]["blind"] # Fiducial version whose NPZ may be overridden with an explicit lc path fiducial_version = fiducial_version or config["fiducial"]["version"] @@ -268,7 +267,7 @@ def main( data_paths = [ fiducial_pure_eb_data if v == fiducial_version and fiducial_pure_eb_data - else _pure_eb_npz(results_dir, v, blind) + else _pure_eb_npz(results_dir, v) for v in versions ] @@ -476,8 +475,7 @@ def _from_cli(argv=None): ap.add_argument( "--results-dir", required=True, - help="Directory holding per-version " - "__pure_eb_semianalytic.npz files", + help="Directory holding per-version _pure_eb_semianalytic.npz files", ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") ap.add_argument( diff --git a/papers/bmodes/scripts/run_cl_sweep.py b/papers/bmodes/scripts/run_cl_sweep.py index d5fd6eb0..1bba7192 100644 --- a/papers/bmodes/scripts/run_cl_sweep.py +++ b/papers/bmodes/scripts/run_cl_sweep.py @@ -5,8 +5,8 @@ fiducial cl_bb / covariance recipes call, once per version. Each producer writes its native ``pseudo_cl_{ver}.fits`` / ``pseudo_cl_cov_{ver}.fits`` into ``--out``; this driver then renames them to the tagged canonical names -``pseudo_cl_{ver}_blind={blind}_{binning}_nbins={nbins}.fits`` and -``pseudo_cl_cov_{ver}_blind={blind}_{binning}_nbins={nbins}.fits`` — the exact +``pseudo_cl_{ver}_{binning}_nbins={nbins}.fits`` and +``pseudo_cl_cov_{ver}_{binning}_nbins={nbins}.fits`` — the exact pattern ``cl_version_comparison._pseudo_cl`` / ``._pseudo_cl_cov`` reconstruct. Per-version footprint masks resolve internally from cat_config (the version's @@ -16,7 +16,7 @@ python run_cl_sweep.py \ --config .../config.yaml --cat-config .../cat_config.yaml \ --nside 1024 --npatch 1 --binning powspace --nbins 32 --power 0.5 \ - --blind A --out + --out """ import argparse @@ -40,17 +40,15 @@ from sweep_versions import nonfiducial_versions # noqa: E402 -def _canonical(prefix, ver, blind, binning, nbins): - return f"{prefix}_{ver}_blind={blind}_{binning}_nbins={nbins}.fits" +def _canonical(prefix, ver, binning, nbins): + return f"{prefix}_{ver}_{binning}_nbins={nbins}.fits" -def _run_and_tag(producer, prefix, ver, out, blind, binning, nbins, **kw): +def _run_and_tag(producer, prefix, ver, out, binning, nbins, **kw): """Run one per-version producer, then rename its native FITS to canonical.""" - producer( - version=ver, output_dir=out, blind=blind, binning=binning, nbins=nbins, **kw - ) + producer(version=ver, output_dir=out, binning=binning, nbins=nbins, **kw) native = os.path.join(out, f"{prefix}_{ver}.fits") - canonical = os.path.join(out, _canonical(prefix, ver, blind, binning, nbins)) + canonical = os.path.join(out, _canonical(prefix, ver, binning, nbins)) os.replace(native, canonical) print(f"[cl_sweep] {ver}: {os.path.basename(canonical)}") @@ -74,7 +72,6 @@ def _from_cli(argv=None): ) ap.add_argument("--nbins", type=int, default=32) ap.add_argument("--power", type=float, default=0.5) - ap.add_argument("--blind", choices=["A", "B", "C"], default="A") a = ap.parse_args(argv) with open(a.config) as f: config = yaml.safe_load(f) @@ -92,7 +89,6 @@ def _from_cli(argv=None): "pseudo_cl", ver, a.out, - a.blind, a.binning, a.nbins, **shared, @@ -102,7 +98,6 @@ def _from_cli(argv=None): "pseudo_cl_cov", ver, a.out, - a.blind, a.binning, a.nbins, **shared, diff --git a/papers/bmodes/scripts/run_cosebis_ptes_sweep.py b/papers/bmodes/scripts/run_cosebis_ptes_sweep.py index c00cd16e..5b11168d 100644 --- a/papers/bmodes/scripts/run_cosebis_ptes_sweep.py +++ b/papers/bmodes/scripts/run_cosebis_ptes_sweep.py @@ -9,7 +9,7 @@ driver reads that version's 1000-bin integration ξ_± from the xi_sweep output dir and its Gaussian integration covariance from the cov_sweep output dir (both by absolute path — lc does not wire cross-output deps, so run xi_sweep + cov_sweep -first), emitting the canonical ``cosebis_ptes_{ver}_{blind}.npz`` — the same +first), emitting the canonical ``cosebis_ptes_{ver}.npz`` — the same gathered layout the fiducial single-output writes, which config_space_pte_matrices.py adapts via ``_cosebis_matrix_from_npz`` — into ``--out``. Serial over versions (~25 min/version, 206 pairs). @@ -18,7 +18,7 @@ --config .../config.yaml \ --xi-sweep-dir \ --cov-sweep-dir \ - --out [--blind A] [--versions v1 v2 ...] + --out [--versions v1 v2 ...] """ import argparse @@ -47,8 +47,8 @@ def _xi_integration(xi_sweep_dir, ver): ) -def _cov_integration(cov_sweep_dir, ver, blind): - base = f"covariance_{ver}_{blind}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" +def _cov_integration(cov_sweep_dir, ver): + base = f"covariance_{ver}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" return os.path.join(cov_sweep_dir, base, f"{base}_processed.txt") @@ -70,7 +70,6 @@ def _from_cli(argv=None): help="cov_sweep output dir with per-version {base}/{base}_processed.txt cov", ) ap.add_argument("--out", required=True, help="Sweep output directory (lc {output})") - ap.add_argument("--blind", default="A", help="Blind tag (paper: A)") ap.add_argument("--versions", nargs="*", default=None, help="Explicit version keys") a = ap.parse_args(argv) @@ -81,15 +80,15 @@ def _from_cli(argv=None): for ver in versions: xi_int = _xi_integration(a.xi_sweep_dir, ver) - cov_int = _cov_integration(a.cov_sweep_dir, ver, a.blind) + cov_int = _cov_integration(a.cov_sweep_dir, ver) for f in (xi_int, cov_int): if not os.path.isfile(f): raise FileNotFoundError(f"MISSING upstream input for {ver}: {f}") print(f"[cosebis_ptes_sweep] {ver}", flush=True) - compute_cosebis_pte(config, xi_int, cov_int, a.out, version=ver, blind=a.blind) + compute_cosebis_pte(config, xi_int, cov_int, a.out, version=ver) print( f"[cosebis_ptes_sweep] {ver} -> " - f"{os.path.join(a.out, f'cosebis_ptes_{ver}_{a.blind}.npz')}", + f"{os.path.join(a.out, f'cosebis_ptes_{ver}.npz')}", flush=True, ) print(f"[cosebis_ptes_sweep] done -> {a.out}", flush=True) diff --git a/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh b/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh index 0e7ecdf5..2f4e4d91 100644 --- a/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh +++ b/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh @@ -7,26 +7,25 @@ # NPZ (data vectors + MC covariance) is read from the pure_eb_sweep output dir by # absolute path — lc does not wire cross-output deps, so the driver reads the # upstream sweep directly; run pure_eb_sweep before this. Each version emits the -# canonical ``{ver}_{blind}_pure_eb_ptes.npz`` — the exact name +# canonical ``{ver}_pure_eb_ptes.npz`` — the exact name # config_space_pte_matrices.py reconstructs from --pte-intermediate-dir — straight # into --out. Serial over versions; each version is fast (206-pair grid, ~seconds). # # Usage: # run_pure_eb_ptes_sweep.sh --config --cat-config \ # --pure-eb-sweep-dir \ -# --out [--blind A] [--versions "v1 v2 ..."] +# --out [--versions "v1 v2 ..."] set -euo pipefail . "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" -CONFIG=""; CATCONFIG=""; PUREEBSWEEP=""; OUT=""; BLIND="A"; VERSIONS="" +CONFIG=""; CATCONFIG=""; PUREEBSWEEP=""; OUT=""; VERSIONS="" while [ $# -gt 0 ]; do case "$1" in --config) CONFIG="$2"; shift 2;; --cat-config) CATCONFIG="$2"; shift 2;; --pure-eb-sweep-dir) PUREEBSWEEP="$2"; shift 2;; --out) OUT="$2"; shift 2;; - --blind) BLIND="$2"; shift 2;; --versions) VERSIONS="$2"; shift 2;; *) echo "unknown arg: $1" >&2; exit 2;; esac @@ -37,12 +36,12 @@ mkdir -p "$OUT" VERSIONS=$(sweep_versions "$CONFIG") for ver in $VERSIONS; do - pureeb="$PUREEBSWEEP/${ver}_${BLIND}_pure_eb_semianalytic.npz" + pureeb="$PUREEBSWEEP/${ver}_pure_eb_semianalytic.npz" [ -f "$pureeb" ] || { echo "MISSING upstream input for $ver: $pureeb" >&2; exit 1; } echo "[pure_eb_ptes_sweep] $ver" spv_python "$PSCRIPTS/calculate_pure_eb_ptes.py" \ - --version "$ver" --blind "$BLIND" \ + --version "$ver" \ --pure-eb-data "$pureeb" --n-samples 2000 --out "$OUT" - echo "[pure_eb_ptes_sweep] $ver -> $OUT/${ver}_${BLIND}_pure_eb_ptes.npz" + echo "[pure_eb_ptes_sweep] $ver -> $OUT/${ver}_pure_eb_ptes.npz" done echo "[pure_eb_ptes_sweep] done -> $OUT" diff --git a/papers/bmodes/scripts/run_pure_eb_semianalytic.sh b/papers/bmodes/scripts/run_pure_eb_semianalytic.sh index 88ddbbc1..3f78ed28 100644 --- a/papers/bmodes/scripts/run_pure_eb_semianalytic.sh +++ b/papers/bmodes/scripts/run_pure_eb_semianalytic.sh @@ -3,11 +3,11 @@ # # Runs the 20 independent MC chunks in parallel (each a fresh apptainer-exec # process, deterministic seed 42+chunk_id) throttled to the node core count, -# then gathers them into the per-(version,blind) semianalytic .npz. Bit-exact +# then gathers them into the per-version semianalytic .npz. Bit-exact # to the paper's scatter-gather (same per-chunk seeds/order); no nested dask. # # Usage: -# run_pure_eb_semianalytic.sh --version SP_v1.4.6.3_leak_corr --blind A \ +# run_pure_eb_semianalytic.sh --version SP_v1.4.6.3_leak_corr \ # --cat-config \ # --xi-reporting --xi-integration \ # --cov-integration \ @@ -16,7 +16,7 @@ set -euo pipefail . "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" -VERSION=""; BLIND="A"; CATCONFIG=""; XIREP=""; XIINT=""; COVINT=""; OUT="" +VERSION=""; CATCONFIG=""; XIREP=""; XIINT=""; COVINT=""; OUT="" NCHUNKS=20; NSAMPLES=2000; NPROC="${SLURM_CPUS_PER_TASK:-16}" MINSEP=1.0; MAXSEP=250.0; NBINS=20 MINSEPINT=0.5; MAXSEPINT=300.0; NBINSINT=1000; NPATCH=1 @@ -24,7 +24,6 @@ MINSEPINT=0.5; MAXSEPINT=300.0; NBINSINT=1000; NPATCH=1 while [ $# -gt 0 ]; do case "$1" in --version) VERSION="$2"; shift 2;; - --blind) BLIND="$2"; shift 2;; --cat-config) CATCONFIG="$2"; shift 2;; --xi-reporting) XIREP="$2"; shift 2;; --xi-integration) XIINT="$2"; shift 2;; @@ -46,13 +45,13 @@ done mkdir -p "$OUT/chunks" -echo "[pure_eb] $NCHUNKS chunks, $NSAMPLES samples, nproc=$NPROC, version=$VERSION blind=$BLIND" +echo "[pure_eb] $NCHUNKS chunks, $NSAMPLES samples, nproc=$NPROC, version=$VERSION" for i in $(seq 0 $((NCHUNKS-1))); do ( SPV_EXEC_EXTRA=$SINGLE_THREAD_ENV spv_python "$PSCRIPTS/precompute_pure_eb_chunk.py" \ --chunk-id "$i" --n-chunks "$NCHUNKS" --n-samples "$NSAMPLES" \ - --version "$VERSION" --blind "$BLIND" --cat-config "$CATCONFIG" \ + --version "$VERSION" --cat-config "$CATCONFIG" \ --xi-reporting "$XIREP" --xi-integration "$XIINT" --cov-integration "$COVINT" \ --min-sep "$MINSEP" --max-sep "$MAXSEP" --nbins "$NBINS" \ --min-sep-int "$MINSEPINT" --max-sep-int "$MAXSEPINT" --nbins-int "$NBINSINT" \ @@ -71,7 +70,7 @@ done echo "[pure_eb] all $NCHUNKS chunks done; gathering" spv_python "$PSCRIPTS/gather_pure_eb_chunks.py" \ - --version "$VERSION" --blind "$BLIND" \ + --version "$VERSION" \ --xi-reporting "$XIREP" --xi-integration "$XIINT" \ --chunks-dir "$OUT/chunks" \ --min-sep "$MINSEP" --max-sep "$MAXSEP" --nbins "$NBINS" \ diff --git a/papers/bmodes/scripts/run_pure_eb_sweep.sh b/papers/bmodes/scripts/run_pure_eb_sweep.sh index 8b8587d4..7add549e 100755 --- a/papers/bmodes/scripts/run_pure_eb_sweep.sh +++ b/papers/bmodes/scripts/run_pure_eb_sweep.sh @@ -8,7 +8,7 @@ # integration covariance from the cov_sweep output dir (both by absolute path — # lc does not wire cross-output deps, so the driver reads upstream sweeps # directly; run xi_sweep + cov_sweep before this). The gathered -# ``{ver}_{blind}_pure_eb_semianalytic.npz`` — already the canonical name +# ``{ver}_pure_eb_semianalytic.npz`` — already the canonical name # pure_eb_version_comparison (--results-dir) reads — is moved into --out; the # per-version MC chunks stage in a scratch subdir that is removed after gather. # @@ -16,12 +16,12 @@ # run_pure_eb_sweep.sh --config --cat-config \ # --xi-sweep-dir \ # --cov-sweep-dir \ -# --out [--blind A] [--versions "v1 v2 ..."] +# --out [--versions "v1 v2 ..."] set -euo pipefail . "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" -CONFIG=""; CATCONFIG=""; XISWEEP=""; COVSWEEP=""; OUT=""; BLIND="A"; VERSIONS="" +CONFIG=""; CATCONFIG=""; XISWEEP=""; COVSWEEP=""; OUT=""; VERSIONS="" while [ $# -gt 0 ]; do case "$1" in --config) CONFIG="$2"; shift 2;; @@ -29,7 +29,6 @@ while [ $# -gt 0 ]; do --xi-sweep-dir) XISWEEP="$2"; shift 2;; --cov-sweep-dir) COVSWEEP="$2"; shift 2;; --out) OUT="$2"; shift 2;; - --blind) BLIND="$2"; shift 2;; --versions) VERSIONS="$2"; shift 2;; *) echo "unknown arg: $1" >&2; exit 2;; esac @@ -42,7 +41,7 @@ VERSIONS=$(sweep_versions "$CONFIG") for ver in $VERSIONS; do xirep="$XISWEEP/${ver}_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.sacc" xiint="$XISWEEP/${ver}_xi_minsep=0.5_maxsep=300.0_nbins=1000_npatch=1.sacc" - covbase="covariance_${ver}_${BLIND}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" + covbase="covariance_${ver}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" covint="$COVSWEEP/$covbase/${covbase}_processed.txt" for f in "$xirep" "$xiint" "$covint"; do [ -f "$f" ] || { echo "MISSING upstream input for $ver: $f" >&2; exit 1; } @@ -50,12 +49,12 @@ for ver in $VERSIONS; do echo "[pure_eb_sweep] $ver" stage="$OUT/_stage_${ver}" bash "$PSCRIPTS/run_pure_eb_semianalytic.sh" \ - --version "$ver" --blind "$BLIND" --cat-config "$CATCONFIG" \ + --version "$ver" --cat-config "$CATCONFIG" \ --xi-reporting "$xirep" --xi-integration "$xiint" --cov-integration "$covint" \ --out "$stage" - mv "$stage/${ver}_${BLIND}_pure_eb_semianalytic.npz" \ - "$OUT/${ver}_${BLIND}_pure_eb_semianalytic.npz" + mv "$stage/${ver}_pure_eb_semianalytic.npz" \ + "$OUT/${ver}_pure_eb_semianalytic.npz" rm -rf "$stage" - echo "[pure_eb_sweep] $ver -> $OUT/${ver}_${BLIND}_pure_eb_semianalytic.npz" + echo "[pure_eb_sweep] $ver -> $OUT/${ver}_pure_eb_semianalytic.npz" done echo "[pure_eb_sweep] done -> $OUT" diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 5d8416c6..8501bfbc 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -94,7 +94,6 @@ cosmo_val: # --------------------------------------------------------------------------- fiducial: version: SP_v1.4.6.3_leak_corr - blind: A npatch: 1 min_sep: 1.0 max_sep: 250.0 @@ -124,7 +123,6 @@ cl: # stamps into the filename, reconstructed by the inference consumer. harmonic: fiducial: - blind: A binning: powspace nbins: 32 diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 3992d415..7465fc06 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -1,7 +1,6 @@ # %% import copy import os -import re from pathlib import Path import colorama @@ -257,7 +256,6 @@ def __init__( cell_seed=8192, path_onecovariance=None, cosmo_params=None, - blind=None, ): self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method @@ -286,7 +284,6 @@ def __init__( self.fiducial_input_inka = fiducial_input_inka self.nside_mask = nside_mask self.path_onecovariance = path_onecovariance - self.blind = blind assert self.cell_method in ["map", "catalog"], ( "cell_method must be 'map' or 'catalog'" @@ -451,20 +448,8 @@ def get_redshift(self, version): Redshift values nz : ndarray n(z) probability density - - Notes - ----- - If self.blind is set, the redshift path is modified to use the - specified blind (A, B, or C) by replacing the blind suffix in the - configured path. """ - redshift_path = self.cc[version]["shear"]["redshift_path"] - - # Override blind if specified - if self.blind is not None: - redshift_path = re.sub(r"_[ABC]\.txt$", f"_{self.blind}.txt", redshift_path) - - return np.loadtxt(redshift_path, unpack=True) + return np.loadtxt(self.cc[version]["shear"]["redshift_path"], unpack=True) def _write_catalog_config(self): with self.catalog_config_path.open("w") as file: diff --git a/src/sp_validation/tests/test_cv_init_params.py b/src/sp_validation/tests/test_cv_init_params.py index 240f9213..ffc87d8b 100644 --- a/src/sp_validation/tests/test_cv_init_params.py +++ b/src/sp_validation/tests/test_cv_init_params.py @@ -16,9 +16,7 @@ REPO = Path(__file__).resolve().parents[3] -EXEMPT = { - "blind": "None keeps the n(z) blind declared in the catalogue config", -} +EXEMPT = {} def _load_common(): diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index cdfec1f8..f0ddc8c8 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -598,7 +598,7 @@ def test_calculate_pseudo_cl_out_path_born_at_declared_name(cv): untagged native name — so the tagged and diagnostic rules stay disjoint.""" ver = cv._test_version cv._pseudo_cls = {} - tagged = cv._output_path(f"pseudo_cl_{ver}_blind=A_powspace_nbins=32.sacc") + tagged = cv._output_path(f"pseudo_cl_{ver}_powspace_nbins=32.sacc") native = cv._output_path(f"pseudo_cl_{ver}.sacc") cv.calculate_pseudo_cl(out_path=tagged) diff --git a/workflow/common.py b/workflow/common.py index ef9c41f3..6736ef53 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -97,10 +97,6 @@ def _load_checkout_module(name): # the blind registry beside it. CAT_CONFIG = str(REPO_ROOT / "cosmo_val" / "cat_config.yaml") REGISTRY = _custody.registry_of(CAT_CONFIG) -# "blind" is the n(z) A/B/C realisation convention, not Smokescreen blinding -# (custody, above). The name is baked into on-disk filenames we do not own -# (e.g. nz_{version}_{A|B|C}.txt). -BLINDS = ["A", "B", "C"] BLOCK_PAIRS = [("++", "1"), ("--", "2"), ("+-", "3")] # Fiducial cosmology: Planck 2018 (astropy Planck18, Table 2 + BAO) @@ -113,8 +109,7 @@ def _load_checkout_module(name): # Patterns must match all expected values; overly restrictive patterns cause # silent failures. Apply with: wildcard_constraints: **WILDCARD_CONSTRAINTS WILDCARD_CONSTRAINTS = { - "version": r"SP_v[\d.]+(_w_iv)?(_ecut\d+)?(_leak_corr)?", - "blind": r"[ABC]", + "version": r"SP_v[\d.]+(_[ABC])?(_w_iv)?(_ecut\d+)?(_leak_corr)?", "nbins": r"\d+", "min_sep": r"[0-9.]+", "max_sep": r"[0-9.]+", @@ -317,20 +312,13 @@ def fiducial_binning_suffix(fiducial=None): ) -def resolve_covariance_version(version): - """Map version to its covariance version.""" - return version - - def covariance_base( version, - blind, gaussian="ng", min_sep=None, max_sep=None, nbins=None, mask_suffix=None, - resolve_version=True, fiducial=None, default_mask_suffix=None, ): @@ -346,59 +334,31 @@ def covariance_base( DEFAULT_MASK_SUFFIX if default_mask_suffix is None else default_mask_suffix ) ) - cov_version = resolve_covariance_version(version) if resolve_version else version return ( - f"covariance_{cov_version}_{blind}_{gaussian}" + f"covariance_{version}_{gaussian}" f"_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}" ) def covariance_dir( - version, - blind, - gaussian="ng", - min_sep=None, - max_sep=None, - nbins=None, - mask_suffix=None, - resolve_version=True, + version, gaussian="ng", min_sep=None, max_sep=None, nbins=None, mask_suffix=None ): """Construct covariance directory path.""" - base = covariance_base( - version, - blind, - gaussian, - min_sep, - max_sep, - nbins, - mask_suffix, - resolve_version=resolve_version, - ) + base = covariance_base(version, gaussian, min_sep, max_sep, nbins, mask_suffix) return str(COSMO_INFERENCE / f"data/covariance/{base}") def covariance_path( version, - blind, gaussian="ng", min_sep=None, max_sep=None, nbins=None, mask_suffix=None, suffix="_processed.txt", - resolve_version=True, ): """Construct covariance file path.""" - base = covariance_base( - version, - blind, - gaussian, - min_sep, - max_sep, - nbins, - mask_suffix, - resolve_version=resolve_version, - ) + base = covariance_base(version, gaussian, min_sep, max_sep, nbins, mask_suffix) return str(COSMO_INFERENCE / f"data/covariance/{base}/{base}{suffix}") @@ -443,13 +403,17 @@ def patches_input(version, npatch): ) -def build_redshift_path(version, blind): - """Construct n(z) filepath for given catalog version and blind.""" - base = base_version(version) - if "v1.4.11" in base: - base = "SP_v1.4.6" - version_dir = base.replace("SP_", "") - return f"/n17data/sguerrini/UNIONS/WL/nz/{version_dir}/nz_{base}_{blind}.txt" +def catalogue_entry(version): + """The catalogue-config entry describing ``version`` (its own, or the one + its ``_leak_corr``/``_seed`` variant chain reaches).""" + return CATALOG_CONFIG[_custody.entry_name(CATALOG_CONFIG, version)] + + +def redshift_path(version): + """The n(z) file of ``version``: its catalogue entry's ``shear.redshift_path``, + relative to the entry's ``subdir`` unless absolute.""" + entry = catalogue_entry(version) + return os.path.join(entry.get("subdir", ""), entry["shear"]["redshift_path"]) # --------------------------------------------------------------------------- @@ -526,7 +490,7 @@ def grid_of(grids, binning): def pseudo_cl_tag(config): """Fiducial harmonic-binning tag stamped into pseudo-Cl filenames.""" fiducial = config["harmonic"]["fiducial"] - return f"blind={fiducial['blind']}_{fiducial['binning']}_nbins={fiducial['nbins']}" + return f"{fiducial['binning']}_nbins={fiducial['nbins']}" def shear_catalog(version): diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 8c614fad..1dd844c6 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -97,7 +97,6 @@ def _grid_cov(version, grid, gaussian): binning = XI_GRIDS[grid] return covariance_path( version, - FIDUCIAL["blind"], gaussian=gaussian, min_sep=binning["min_sep"], max_sep=binning["max_sep"], diff --git a/workflow/rules/covariance.smk b/workflow/rules/covariance.smk index 9c26b775..6ca907f9 100644 --- a/workflow/rules/covariance.smk +++ b/workflow/rules/covariance.smk @@ -2,11 +2,8 @@ def get_cat_params(version): - """Extract covariance parameters (area, n_e, sigma_e) from catalog config.""" - base_version = version.replace("_leak_corr", "") - if base_version not in CATALOG_CONFIG: - raise KeyError(f"Catalog configuration not found for {base_version}") - cov_th = CATALOG_CONFIG[base_version]["cov_th"] + """Covariance parameters (area, n_e, sigma_e) of ``version``'s catalogue entry.""" + cov_th = catalogue_entry(version)["cov_th"] return cov_th["A"], cov_th["n_e"], cov_th["sigma_e"] @@ -20,16 +17,14 @@ MASK_CLS_FILES = { "footprint_starhalo": f"{MASK_CLS_BASE}/mask_cls_footprint_starhalo_nside_4096_norm.txt", } -# v1.4.8 uses the star-halo footprint; all other versions use the standard footprint -STARHALO_VERSIONS = {"v1.4.8"} +# SP_v1.4.8 uses the star-halo footprint; every other catalogue the standard one. +STARHALO_CATALOGUES = {"SP_v1.4.8"} def get_mask_cls_path(version): """Return absolute mask Cl path for the requested catalog version.""" - version_dir = version.replace('_leak_corr', '').replace('SP_', '') - version_dir = re.sub(r'_ecut\d+', '', version_dir) - key = "footprint_starhalo" if version_dir in STARHALO_VERSIONS else "footprint" - return MASK_CLS_FILES[key] + starhalo = base_version(version) in STARHALO_CATALOGUES + return MASK_CLS_FILES["footprint_starhalo" if starhalo else "footprint"] rule cosmology_params: @@ -59,14 +54,13 @@ with open('{output}', 'w') as f: rule covariance_ini: input: - nz_file=lambda w: build_redshift_path(w.version, w.blind), + nz_file=lambda w: redshift_path(w.version), mask=lambda w: [] if w.mask_suffix != "_masked" else [get_mask_cls_path(w.version)], output: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.ini") + str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.ini") params: outdir=lambda w: covariance_dir( - w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, - resolve_version=False + w.version, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix ), ng_value=lambda wildcards: "1" if wildcards.gaussian == "ng" else "0", omega_m=PLANCK18["Omega_m"], @@ -153,16 +147,15 @@ rule covariance_cosmocov: input: rules.covariance_ini.output, output: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/cov_tmp_ssss_{block_pm}_cov_Ntheta{nbins}_Ntomo1_{block_i}") + str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/cov_tmp_ssss_{block_pm}_cov_Ntheta{nbins}_Ntomo1_{block_i}") params: block_i="{block_i}", outdir=lambda w: covariance_dir( - w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, - resolve_version=False + w.version, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix ), ini_path=lambda w: covariance_path( - w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, - suffix=".ini", resolve_version=False + w.version, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, + suffix=".ini" ), cosmocov=config["tools"]["cosmocov_executable"], container: @@ -184,13 +177,13 @@ rule covariance_cosmocov: rule covariance_cat: input: cov_block=lambda w: [ - f"{covariance_dir(w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, resolve_version=False)}" + f"{covariance_dir(w.version, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix)}" f"/cov_tmp_ssss_{pm}_cov_Ntheta{w.nbins}_Ntomo1_{idx}" for pm, idx in BLOCK_PAIRS ], threads: 1 output: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.txt") + str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.txt") shell: """ cat {input} > {output} @@ -243,24 +236,24 @@ rule generate_glass_mock_rhotau_samples: rule covariance_process: """Post-process a raw CosmoCov matrix into the analysis-ready form.""" input: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.txt") + str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.txt") output: - matrix=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed.txt"), - gaussian=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_g.txt"), - plot=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_plot.pdf") + matrix=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed.txt"), + gaussian=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_g.txt"), + plot=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_plot.pdf") threads: 1 script: "../scripts/cosmocov_process.py" def fiducial_covariance_outputs(mask_suffix=""): - """Return processed covariance files for fiducial version/blind.""" + """Return processed covariance files for the fiducial version.""" ng_path = covariance_path( - FIDUCIAL["version"], FIDUCIAL["blind"], "ng", + FIDUCIAL["version"], "ng", FIDUCIAL["min_sep"], FIDUCIAL["max_sep"], FIDUCIAL["nbins"], mask_suffix ) g_path = covariance_path( - FIDUCIAL["version"], FIDUCIAL["blind"], "g", + FIDUCIAL["version"], "g", FIDUCIAL["min_sep_int"], FIDUCIAL["max_sep_int"], FIDUCIAL["nbins_int"], mask_suffix ) return [ng_path, g_path] diff --git a/workflow/rules/glass_mock.smk b/workflow/rules/glass_mock.smk index 243076d6..090e3b55 100644 --- a/workflow/rules/glass_mock.smk +++ b/workflow/rules/glass_mock.smk @@ -118,11 +118,7 @@ rule mock_cosebis_bias_test: mock_id=GLASS_MOCK_IDS, ), xi_ref=f"{MOCK_RESULTS}/gg_glass_mock_00001_nbins=1000.fits", - cov=str( - COSMO_INFERENCE / "data/covariance" - / "covariance_SP_v1.4.6_leak_corr_A_g_minsep=0.5_maxsep=500.0_nbins=1000_masked" - / "covariance_SP_v1.4.6_leak_corr_A_g_minsep=0.5_maxsep=500.0_nbins=1000_masked_processed.txt" - ), + cov=covariance_path("SP_v1.4.6_leak_corr", "g", 0.5, 500.0, 1000, "_masked"), params: nmodes=config["fiducial"]["nmodes"], theta_min=config["cosebis"]["theta_min"], diff --git a/workflow/rules/inference.smk b/workflow/rules/inference.smk index d2847976..f707e895 100644 --- a/workflow/rules/inference.smk +++ b/workflow/rules/inference.smk @@ -1,4 +1,4 @@ -# Imports from Snakefile: FIDUCIAL, COSMO_INFERENCE, COSMO_VAL, covariance_path, build_redshift_path, fiducial_binning_suffix +# Imports from Snakefile: FIDUCIAL, COSMO_INFERENCE, COSMO_VAL, covariance_path, redshift_path, fiducial_binning_suffix, pseudo_cl_tag # NOTE: dormant subsystem. The file-name plumbing (config-driven paths + the # producer-tagged pseudo-Cl names) is fixed and the DAG is valid, but it has not # been run end-to-end. Reviving it still needs the FITS-CONTENT plumbing @@ -34,14 +34,8 @@ GLASS_MOCK_CONFIG_PATTERN = str( # Fiducial harmonic-binning tag the pseudo-Cl producer (twopoint.smk) stamps # into the filename. These are NOT inference_prep wildcards, so the consumer -# reads them from config to reconstruct the exact name the producer emits -# (canonical: blind=A, powspace, nbins=32 — see twopoint.smk pseudo_cl_all). -HARMONIC_FIDUCIAL = config["harmonic"]["fiducial"] -PSEUDO_CL_TAG = ( - f"blind={HARMONIC_FIDUCIAL['blind']}" - f"_{HARMONIC_FIDUCIAL['binning']}" - f"_nbins={HARMONIC_FIDUCIAL['nbins']}" -) +# reads them from config to reconstruct the exact name the producer emits. +PSEUDO_CL_TAG = pseudo_cl_tag(config) def pseudo_cl_assets(version): @@ -58,12 +52,12 @@ def pseudo_cl_assets(version): rule inference_prep: input: # Processed covariance matrix - use centralized covariance_path() - cov_matrix=lambda w: covariance_path(w.version, w.blind, min_sep=w.min_sep, max_sep=w.max_sep, nbins=w.nbins), + cov_matrix=lambda w: covariance_path(w.version, min_sep=w.min_sep, max_sep=w.max_sep, nbins=w.nbins), # Xi FITS files xi_plus=str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), xi_minus=str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - # n(z) file (using new location with base version mapping) - nz_file=lambda w: build_redshift_path(w.version, w.blind), + # n(z) file: the catalogue entry's + nz_file=lambda w: redshift_path(w.version), # rho/tau stats rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), @@ -74,15 +68,15 @@ rule inference_prep: output: fits_file=str( COSMO_INFERENCE_PROD - / "data/{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}/cosmosis_{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits" + / "data/{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}/cosmosis_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits" ), config_file=str( COSMO_INFERENCE_PROD - / "cosmosis_config/output/cosmosis_pipeline_{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.ini" + / "cosmosis_config/output/cosmosis_pipeline_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.ini" ) params: - cosmosis_root="{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}", - data_dir=f"{CHAINS_DIR}/{{version}}_{{blind}}_minsep={{min_sep}}_maxsep={{max_sep}}_nbins={{nbins}}_npatch={{npatch}}", + cosmosis_root="{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}", + data_dir=f"{CHAINS_DIR}/{{version}}_minsep={{min_sep}}_maxsep={{max_sep}}_nbins={{nbins}}_npatch={{npatch}}", output_root=str(COSMO_INFERENCE_PROD), threads: 1 resources: @@ -113,12 +107,12 @@ rule inference_fiducial: input: # Use the same output patterns as inference_prep with FIDUCIAL params rules.inference_prep.output.fits_file.format( - version=FIDUCIAL["version"], blind=FIDUCIAL["blind"], + version=FIDUCIAL["version"], min_sep=FIDUCIAL["min_sep"], max_sep=FIDUCIAL["max_sep"], nbins=FIDUCIAL["nbins"], npatch=FIDUCIAL["npatch"] ), rules.inference_prep.output.config_file.format( - version=FIDUCIAL["version"], blind=FIDUCIAL["blind"], + version=FIDUCIAL["version"], min_sep=FIDUCIAL["min_sep"], max_sep=FIDUCIAL["max_sep"], nbins=FIDUCIAL["nbins"], npatch=FIDUCIAL["npatch"] ) @@ -133,9 +127,9 @@ rule inference_prep_glass_mock: input: xi=f"{GLASS_MOCK_DATA_DIR}/xi_glass_mock_{{mock_id}}_4096_nbins=20.fits", # Use centralized covariance_path() with fiducial mock version - cov_matrix=covariance_path(FIDUCIAL["mock_version"], "A"), + cov_matrix=covariance_path(FIDUCIAL["mock_version"]), # n(z) file - nz_file=build_redshift_path(FIDUCIAL["mock_version"], "A"), + nz_file=redshift_path(FIDUCIAL["mock_version"]), # Rho/tau stats: rho from real data, tau sampled rho_stats=str(COSMO_VAL / f"rho_tau_stats/rho_stats_{FIDUCIAL['mock_version']}{fiducial_binning_suffix()}.fits"), tau_stats="results/glass_mock_rhotau_samples/{mock_id}/tau_stats_sampled.fits", diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 852c8722..42364eeb 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -72,7 +72,6 @@ rule rho_tau_stats: # Pseudo-Cl generation for harmonic-space data vectors and COSEBIS validation. -BASE_VERSIONS = [v.replace("_leak_corr", "") for v in config["versions"]] wildcard_constraints: binning="linear|logspace|powspace", @@ -81,12 +80,9 @@ wildcard_constraints: rule pseudo_cl: """Generate pseudo-Cl data vector (born as SACC) with configurable binning.""" output: - pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.sacc"), - wildcard_constraints: - blind="[ABC]", + pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_{binning}_nbins={nbins}.sacc"), params: version="{version}", - blind="{blind}", cat_config=CAT_CONFIG, nside=1024, npatch=1, @@ -106,12 +102,9 @@ rule pseudo_cl: rule pseudo_cl_cov: """Generate pseudo-Cl covariance with configurable binning.""" output: - pseudo_cl_cov=str(COSMO_VAL / "pseudo_cl_cov_{version}_blind={blind}_{binning}_nbins={nbins}.fits"), - wildcard_constraints: - blind="[ABC]", + pseudo_cl_cov=str(COSMO_VAL / "pseudo_cl_cov_{version}_{binning}_nbins={nbins}.fits"), params: version="{version}", - blind="{blind}", cat_config=CAT_CONFIG, nside=1024, npatch=1, @@ -134,7 +127,7 @@ rule pseudo_cl_all: """Generate pseudo-Cls for all versions.""" input: expand( - str(COSMO_VAL / "pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc"), + str(COSMO_VAL / "pseudo_cl_{version}_powspace_nbins=32.sacc"), version=PSEUDO_CL_VERSIONS, ), @@ -143,7 +136,7 @@ rule pseudo_cl_cov_all: """Generate pseudo-Cl covariances for all versions.""" input: expand( - str(COSMO_VAL / "pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits"), + str(COSMO_VAL / "pseudo_cl_cov_{version}_powspace_nbins=32.fits"), version=PSEUDO_CL_VERSIONS, ), @@ -152,7 +145,6 @@ rule pseudo_cl_fine_all: """Generate fine pseudo-Cls for COSEBIS.""" input: expand( - str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_linear_nbins=2040.sacc"), + str(COSMO_VAL / "pseudo_cl_{version}_linear_nbins=2040.sacc"), version=config["versions"], - blind=BLINDS, ), diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index a190695a..a8ef827c 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -34,7 +34,6 @@ def generate_pseudo_cl( cat_config: str, nside: int = 1024, npatch: int = 1, - blind: str = None, cosmo_params: dict = None, binning: str = "linear", nbins: int = None, @@ -56,8 +55,6 @@ def generate_pseudo_cl( HEALPix nside for map-based estimation npatch : int Number of jackknife patches - blind : str, optional - Blind identifier (A, B, or C) to override n(z) path cosmo_params : dict, optional Cosmological parameters. Keys: Omega_m, sigma_8, n_s, h, Omega_b. If None, uses Planck 2018 defaults. @@ -79,7 +76,6 @@ def generate_pseudo_cl( output_dir = os.path.dirname(out_path) os.makedirs(output_dir, exist_ok=True) - blind_str = f" blind={blind}" if blind else "" if binning == "linear": # For linear binning, nbins determines ell_step such that we cover 2-2048 ell_step = max(1, (2048 - 2) // nbins) @@ -90,7 +86,7 @@ def generate_pseudo_cl( bin_str = f"nbins={nbins}, power={power}" print(f"\n{'=' * 60}") - print(f"Generating pseudo-Cl for {version}{blind_str}") + print(f"Generating pseudo-Cl for {version}") print(f"Binning: {binning} ({bin_str})") if cosmo_params: print( @@ -119,7 +115,6 @@ def generate_pseudo_cl( nside=nside, cell_method="catalog", nrandom_cell=100, - blind=blind, cosmo_params=cosmo_params, npatch=npatch, theta_min=1.0, @@ -153,7 +148,6 @@ def _from_snakemake(smk): cat_config=p["cat_config"], nside=int(p["nside"]), npatch=int(p["npatch"]), - blind=p.get("blind", None), cosmo_params=p.get("cosmo_params", None), binning=p["binning"], nbins=int(p["nbins"]), @@ -197,9 +191,6 @@ def _from_cli(argv=None): default=0.5, help="Power for powspace binning (0.5 = sqrt spacing)", ) - ap.add_argument( - "--blind", choices=["A", "B", "C"], default=None, help="Blind identifier" - ) ap.add_argument( "--cosmo-json", default=None, @@ -221,7 +212,6 @@ def _from_cli(argv=None): cat_config=a.cat_config, nside=a.nside, npatch=a.npatch, - blind=a.blind, cosmo_params=cosmo_params, binning=a.binning, nbins=a.nbins, diff --git a/workflow/scripts/generate_pseudo_cl_cov.py b/workflow/scripts/generate_pseudo_cl_cov.py index 08f2cf07..247c5f33 100644 --- a/workflow/scripts/generate_pseudo_cl_cov.py +++ b/workflow/scripts/generate_pseudo_cl_cov.py @@ -39,7 +39,6 @@ def generate_pseudo_cl_cov( cat_config: str, nside: int = 1024, npatch: int = 1, - blind: str = None, cosmo_params: dict = None, binning: str = "powspace", nbins: int = 32, @@ -61,8 +60,6 @@ def generate_pseudo_cl_cov( HEALPix nside for map-based estimation npatch : int Number of jackknife patches - blind : str, optional - Blind identifier (A, B, or C) to override n(z) path cosmo_params : dict, optional Cosmological parameters. Keys: Omega_m, sigma_8, n_s, h, Omega_b. If None, uses Planck 2018 defaults. @@ -80,7 +77,6 @@ def generate_pseudo_cl_cov( """ os.makedirs(output_dir, exist_ok=True) - blind_str = f" blind={blind}" if blind else "" if binning == "linear": ell_step = max(1, (2048 - 2) // nbins) bin_str = f"nbins={nbins} (ell_step={ell_step})" @@ -90,7 +86,7 @@ def generate_pseudo_cl_cov( bin_str = f"nbins={nbins}, power={power}" print(f"\n{'=' * 60}") - print(f"Generating pseudo-Cl covariance for {version}{blind_str}") + print(f"Generating pseudo-Cl covariance for {version}") print(f"Binning: {binning} ({bin_str})") if cosmo_params: print( @@ -119,7 +115,6 @@ def generate_pseudo_cl_cov( nside=nside, cell_method="catalog", nrandom_cell=100, - blind=blind, cosmo_params=cosmo_params, npatch=npatch, theta_min=1.0, @@ -158,7 +153,6 @@ def _from_snakemake(smk): cat_config=p["cat_config"], nside=int(p["nside"]), npatch=int(p["npatch"]), - blind=p.get("blind", None), cosmo_params=p.get("cosmo_params", None), binning=p["binning"], nbins=int(p["nbins"]), @@ -205,9 +199,6 @@ def _from_cli(argv=None): default=0.5, help="Power for powspace binning (0.5 = sqrt spacing)", ) - ap.add_argument( - "--blind", choices=["A", "B", "C"], default=None, help="Blind identifier" - ) ap.add_argument( "--cosmo-json", default=None, @@ -226,7 +217,6 @@ def _from_cli(argv=None): cat_config=a.cat_config, nside=a.nside, npatch=a.npatch, - blind=a.blind, cosmo_params=cosmo_params, binning=a.binning, nbins=a.nbins, diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py index 04f3fdf6..c590851d 100644 --- a/workflow/tests/conftest.py +++ b/workflow/tests/conftest.py @@ -254,7 +254,6 @@ def toy(tmp_path_factory): path = Path( common.covariance_path( version, - config["fiducial"]["blind"], gaussian, grid["min_sep"], grid["max_sep"], From 5b75e7ad4c175cc4a9806a47816837da7751bb81 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 04:55:49 +0200 Subject: [PATCH 118/160] custody: one registry reader, one comparison, seed paths rendered once - custody.read_record reads a blind's public files; custody.records and blinding's opening share it. init writes the record to a staging directory, opens it from there as jobs do and conceals a row, then gives it its name. - Custody compares with ==, the registry excluded from equality; that is the one comparison in sacc_io._derive, blinding.verify and the audit. token stays as the Snakemake param. - custody.entry_name resolves the config entry describing a version, and custody.seed_path renders a _seed version's shear path; CosmologyValidation and custody.shear_file both use it, so the twins check sees the file a seed variant reads. - Drop the coverage check factors() made by construction and open_blind's repeated commitment check; inline is_signal/is_derived; name the audit's shift tolerances. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- src/sp_validation/blinding.py | 94 ++++++++++++------------- src/sp_validation/cosmo_val/core.py | 75 +++----------------- src/sp_validation/custody.py | 84 ++++++++++++++++------ src/sp_validation/sacc_io.py | 20 ++---- src/sp_validation/tests/test_custody.py | 2 +- 5 files changed, 126 insertions(+), 149 deletions(-) diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 68045fa4..166a65fa 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -157,23 +157,24 @@ class Blind: hidden: TheoryConfig -def _blind(name, seed, record): - """Blind ``name`` from its seed and config record, as it is opened.""" - config = _recorded_config(name, record) - return Blind(name, seed, config, hidden_theory(seed, config)) - - @functools.cache def _open(registry, name): + path = Path(registry) / name + if not path.is_dir(): + raise _custody.CustodyError(f"no blind {name} in {registry}") + return _read_blind(path) + + +def _read_blind(path): + """The blind stored at directory ``path``, its sealed seed checked against + its public record.""" from cryptography.fernet import Fernet, InvalidToken - record = _custody.records(registry).get(name) - if record is None: - raise _custody.CustodyError(f"no blind {name} in {registry}") - c = record.commitment + name = path.name + c = _custody.read_record(path).commitment try: - key = (registry / name / "key").read_bytes() - ciphertext = (registry / name / "seed.fernet").read_bytes() + key = (path / "key").read_bytes() + ciphertext = (path / "seed.fernet").read_bytes() payload = json.loads(Fernet(key).decrypt(ciphertext)) except (OSError, ValueError, InvalidToken) as err: raise _custody.CustodyError( @@ -203,19 +204,15 @@ def _open(registry, name): f"blind {name} was drawn under draw scheme {c['draw_scheme']}; this " f"install's smokescreen draws under {draw_scheme()}" ) - return _blind(name, payload["seed"], c["config"]) + config = _recorded_config(name, c["config"]) + return Blind(name, payload["seed"], config, hidden_theory(payload["seed"], config)) def open_blind(custody): """The verified blind a blinded custody names (cached per process).""" if custody.status != "blinded": raise _custody.CustodyError(f"{custody.catalogue} is {custody.status}") - blind = _open(Path(custody.registry), custody.blind) - if _custody.seed_commitment(blind.seed) != custody.commitment: - raise _custody.CustodyError( - f"blind {custody.blind} is not the one {custody.catalogue} resolved" - ) - return blind + return _open(Path(custody.registry), custody.blind) # --------------------------------------------------------------------------- # @@ -306,14 +303,9 @@ def _factor(block, fiducial, hidden): def factors(s, fiducial, hidden): - """Each shiftable block of ``s`` and its factor; together they cover every row.""" - blocks = shiftable_blocks(s) - covered = np.sort(np.concatenate([b.rows for b in blocks])) if blocks else [] - shiftable = [i for i, dp in enumerate(s.data) if dp.data_type in sacc_io.SHIFTABLE] - if not np.array_equal(covered, shiftable): - raise ValueError("shiftable blocks do not cover every ξ± and Cℓ_EE row once") + """Each shiftable block of ``s`` and its factor.""" out = [] - for block in blocks: + for block in shiftable_blocks(s): factor = _factor(block, fiducial, hidden) if factor.shape != block.rows.shape or not np.all(np.isfinite(factor)): raise ValueError( @@ -391,10 +383,9 @@ def _may_cover(catalogues, recs, name, bases): _custody.refuse_twins(catalogues, after, base) -def _conceal_one_row(name, seed, record): - """Open blind ``name`` from its seed and config record and conceal one ξ± - row under it, as every blinded birth does; raise if it cannot.""" - blind = _blind(name, seed, record) +def _conceal_one_row(blind): + """Conceal one ξ± row under ``blind``, as every blinded birth does; raise + if it cannot.""" z = np.linspace(0.0, 2.0, 101) s = sacc_io.new_sacc({0: (z, np.exp(-0.5 * ((z - 0.7) / 0.2) ** 2))}) sacc_io.add_xi(s, (0, 0), [10.0], [0.0], [0.0], grid="probe") @@ -402,7 +393,7 @@ def _conceal_one_row(name, seed, record): factors(s, blind.config.theory, blind.hidden) except Exception as err: raise _custody.CustodyError( - f"no row conceals under blind {name}'s config: {type(err).__name__}: {err}" + f"no row conceals under blind {blind.name}'s config: {type(err).__name__}: {err}" ) from err @@ -411,18 +402,19 @@ def init(name, bases, *, cat_config, config=None): The one place a seed is drawn. Existing state is refused, never replaced, and the seed is held in memory and written only as ciphertext. The record - is opened and conceals a row before anything is written, so every blind - on disk is one its jobs can conceal under. + is written to a staging directory, opened from there as jobs open it, and + made to conceal a row before it takes its name, so every blind in the + registry is one its jobs can conceal under. """ from cryptography.fernet import Fernet config = config or BlindingConfig() registry = _custody.registry_of(cat_config) catalogues = _catalogues(cat_config) - record, staging = registry / name, registry / f".{name}.tmp" + record, staging = registry / name, registry / f".{name}.tmp" / name if not name or name[0] in "-_." or set(name) - set(_BLIND_NAME): raise _custody.CustodyError(f"blind name {name!r}: use a-z, 0-9, - _ .") - for path in (record, staging): + for path in (record, staging.parent): if path.exists(): raise _custody.CustodyError( f"{path} exists; a blind is drawn once (delete a staging " @@ -448,7 +440,6 @@ def init(name, bases, *, cat_config, config=None): "theory_stack": _theory_stack(), "created": datetime.now(timezone.utc).isoformat(timespec="seconds"), } - _conceal_one_row(name, seed, commitment["config"]) staging.mkdir(parents=True) (staging / "commitment.json").write_text(json.dumps(commitment, indent=2) + "\n") @@ -457,7 +448,13 @@ def init(name, bases, *, cat_config, config=None): (staging / "bases").write_text("".join(f"{b}\n" for b in bases)) for part in ("commitment.json", "seed.fernet", "key"): os.chmod(staging / part, 0o444) + try: + _conceal_one_row(_read_blind(staging)) + except _custody.CustodyError: + shutil.rmtree(staging.parent) + raise os.replace(staging, record) + staging.parent.rmdir() print(f"[blinding] drew blind {name} for {', '.join(bases)}") print(f"[blinding] commit {record.relative_to(registry.parent.parent)}/") return record @@ -535,6 +532,11 @@ def reveal(name, *, root, cat_config): return archive +# How closely blinded − true must equal the seed's shift, relative to the +# block's largest shift: under the theory stack the blind was drawn with, and +# under another one. +SHIFT_TOLERANCE = 1e-6 +SHIFT_TOLERANCE_OTHER_STACK = 1e-3 # A re-measurement repeats a measurement to float noise (TreeCorr's threaded # sums reorder); the audit compares its numbers to this relative tolerance. _REMEASURED = 1e-10 @@ -583,7 +585,7 @@ def _same_covariance(a, b): def _signal(s): """``s``'s signal rows and their covariance, or None if it has none.""" - keep = np.array([sacc_io.is_signal(dp.data_type) for dp in s.data], bool) + keep = np.array([dp.data_type in sacc_io.SIGNAL for dp in s.data], bool) if not keep.any(): return None out = s.copy() @@ -599,7 +601,7 @@ def _audit_part(blinded, true, fiducial, hidden, tolerance): without signal is judged by its stamp alone. """ stamp, true_stamp = (_custody.read_stamp(x.metadata) for x in (blinded, true)) - if (true_stamp.status, true_stamp.catalogue) != ("unblinded", stamp.catalogue): + if true_stamp != _custody.Custody("unblinded", stamp.catalogue): return [f"the live file is stamped {true_stamp.token}"], None blinded, true = _signal(blinded), _signal(true) if blinded is None and true is None: @@ -628,7 +630,7 @@ def _audit_part(blinded, true, fiducial, hidden, tolerance): moved = {} for kind in dict.fromkeys(kinds[rest]): rows = rest & (kinds == kind) - if not sacc_io.is_derived(kind): + if kind not in sacc_io.DERIVED: if np.max(np.abs(delta[rows])) > _REMEASURED * np.max(np.abs(values[rows])): problems.append(f"{kind} moved, but the blind leaves it unshifted") elif cov is None: @@ -650,8 +652,8 @@ def audit(name, *, archive, true_root, cat_config, out=None): catalogue. Its signal rows must match the archived ones in tags, tracers, covariance and patch centres (numbers to a re-measurement's float noise), and blinded − true must equal the seed's shift on every ξ± and Cℓ_EE block - to 1e-6 of the block's largest shift (1e-3 when the theory stack differs - from the one the blind was drawn with); Cℓ_BB and Cℓ_EB may not move, and + to :data:`SHIFT_TOLERANCE` of the block's largest shift + (:data:`SHIFT_TOLERANCE_OTHER_STACK` under another theory stack); Cℓ_BB and Cℓ_EB may not move, and a derived statistic's B rows by at most :data:`B_SIGMA`. A derived statistic's E rows are reported, in σ, not proven: the archive does not record which ξ± parts, through which kernel, they came from. @@ -668,9 +670,9 @@ def audit(name, *, archive, true_root, cat_config, out=None): report["problems"].append("the published seed is not the committed one") return _write_report(report, out) stack = _theory_stack() - tolerance = 1e-6 + tolerance = SHIFT_TOLERANCE if stack != record.commitment.get("theory_stack"): - tolerance = 1e-3 + tolerance = SHIFT_TOLERANCE_OTHER_STACK report["theory_stack"] = { "record": record.commitment.get("theory_stack"), "now": stack, @@ -718,11 +720,9 @@ def verify(path, *, cat_config): except _custody.CustodyError as err: return [str(err)] problems = [] - keys = {**stamp.stamp, **declared.stamp} - differ = sorted(k for k in keys if stamp.stamp.get(k) != declared.stamp.get(k)) - if differ: + if stamp != declared: problems.append( - f"the stamp's {differ} differ from {stamp.catalogue}'s custody, " + f"the stamp, {stamp.token}, is not {stamp.catalogue}'s custody, " f"{declared.token}" ) if stamp.status == "blinded" and stamp.draw_scheme != draw_scheme(): diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 7465fc06..31db4722 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -13,7 +13,13 @@ _get_pte_from_scale_cut, find_conservative_scale_cut_key, ) -from ..custody import CustodyError, base_catalogue, custody_of, registry_of +from ..custody import ( + CustodyError, + base_catalogue, + custody_of, + registry_of, + seed_path, +) from ..statistics import chi2_and_pte from ..version import __version__ from .catalog_characterization import CatalogCharacterizationMixin @@ -165,64 +171,6 @@ def _split_seed_variant(version): return None, None return base, seed_label - @staticmethod - def _materialize_seed_path( - base_cfg, seed_label, version, base_version, catalog_config - ): - """Render the seed-specific shear path using Python string formatting.""" - shear_cfg = base_cfg["shear"] - template = shear_cfg.get("path_template") - - try: - seed_value = int(seed_label) - except ValueError as error: - raise ValueError( - f"Seed suffix for '{version}' is not numeric; cannot materialize path." - ) from error - - format_context = {"seed": seed_value, "seed_label": seed_label} - - if template: - try: - return template.format(**format_context) - except KeyError as error: - raise KeyError( - f"Missing placeholder '{error.args[0]}' in path_template for " - f"'{base_version}' while materializing '{version}'. Update " - f"{catalog_config}." - ) from error - except ValueError as error: - raise ValueError( - f"Invalid format specification in path_template for '{base_version}' " - f"while materializing '{version}'." - ) from error - - path = shear_cfg.get("path", "") - token_start = path.rfind("seed") - if token_start == -1: - raise ValueError( - f"Cannot materialize '{version}': '{base_version}' lacks a shear " - f"path_template and its shear path '{path}' does not contain a 'seed' " - f"token. Update {catalog_config}." - ) - cursor = token_start + 4 # len("seed") - if cursor < len(path) and not path[cursor].isdigit(): - cursor += 1 - digit_start = cursor - while cursor < len(path) and path[cursor].isdigit(): - cursor += 1 - digit_end = cursor - digits = path[digit_start:digit_end] - if not digits: - raise ValueError( - f"Cannot materialize '{version}': shear path '{path}' for base version " - f"'{base_version}' lacks digits after the seed token. Update " - f"{catalog_config}." - ) - - template = f"{path[:digit_start]}{{seed_label}}{path[digit_end:]}" - return template.format(**format_context) - def __init__( self, versions, @@ -376,14 +324,9 @@ def ensure_version_exists(ver): ensure_version_exists(seed_base) if ver not in cc: cc[ver] = copy.deepcopy(cc[seed_base]) - seed_path = self._materialize_seed_path( - cc[seed_base], - seed_label, - ver, - seed_base, - catalog_config, + cc[ver]["shear"]["path"] = seed_path( + cc[seed_base]["shear"], seed_label ) - cc[ver]["shear"]["path"] = seed_path resolve_paths_for_version(ver) processed.add(ver) return diff --git a/src/sp_validation/custody.py b/src/sp_validation/custody.py index 0be80614..474c1691 100644 --- a/src/sp_validation/custody.py +++ b/src/sp_validation/custody.py @@ -9,7 +9,7 @@ import json import os import re -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path STATUSES = ("blinded", "unblinded", "mock") @@ -50,7 +50,8 @@ class Custody: commitment: str | None = None config_digest: str | None = None draw_scheme: int | None = None - registry: Path | None = None + # Where the blind is opened from; two custodies are equal whatever it is. + registry: Path | None = field(default=None, compare=False) @property def token(self): @@ -178,23 +179,32 @@ class Record: revealed: str | None +def read_record(path): + """The public record of the blind stored at directory ``path``. + + The one reader of a blind's public files; its sealed seed is + ``blinding``'s to open. + """ + path = Path(path) + try: + commitment = json.loads((path / "commitment.json").read_text()) + bases = tuple((path / "bases").read_text().split()) + except (OSError, ValueError) as err: + raise CustodyError(f"blind record {path} is incomplete: {err}") from None + revealed = path / "revealed.json" + seed = json.loads(revealed.read_text())["seed"] if revealed.exists() else None + return Record(path.name, commitment, bases, seed) + + def records(registry): """Every blind in the registry, by name.""" - found = {} if registry is None or not Path(registry).is_dir(): - return found - for path in sorted(Path(registry).iterdir()): - if not path.is_dir() or path.name.startswith("."): - continue - try: - commitment = json.loads((path / "commitment.json").read_text()) - bases = tuple((path / "bases").read_text().split()) - except (OSError, ValueError) as err: - raise CustodyError(f"blind record {path} is incomplete: {err}") from None - revealed = path / "revealed.json" - seed = json.loads(revealed.read_text())["seed"] if revealed.exists() else None - found[path.name] = Record(path.name, commitment, bases, seed) - return found + return {} + return { + path.name: read_record(path) + for path in sorted(Path(registry).iterdir()) + if path.is_dir() and not path.name.startswith(".") + } def _reads(entry): @@ -205,11 +215,45 @@ def _reads(entry): return os.path.normpath(os.path.join(str(entry.get("subdir", "")), shear["path"])) +def entry_name(catalogues, version): + """The catalogue-config entry describing ``version``: its own, or that of + the first name its variant chain reaches.""" + for name in _variant_chain(version): + if name in catalogues and name not in NOT_CATALOGUES: + return name + raise CustodyError(f"no catalogue {version} in the catalogue config") + + +def seed_path(shear, seed): + """The shear path of seed label ``seed``, from an entry's ``shear`` block. + + Its ``path_template`` formatted with ``seed`` (an int) and ``seed_label``; + without one, its ``path`` with the digits after its last ``seed`` replaced. + """ + if shear.get("path_template"): + return shear["path_template"].format(seed=int(seed), seed_label=seed) + path = shear.get("path", "") + match = re.match(r"(.*seed\D?)\d+", path) + if match is None: + raise CustodyError( + f"shear path {path!r} has no path_template and no seed to " + "put a seed in" + ) + return match.group(1) + seed + path[match.end() :] + + def shear_file(catalogues, version): - """The shear catalogue file ``version`` reads: that of the first entry of - its variant chain the config holds, as CosmologyValidation reads it.""" + """The shear catalogue file ``version`` reads, as CosmologyValidation reads + it: its entry's, with the seed of a ``_seed`` variant rendered in.""" chain = _variant_chain(version) - return _reads(catalogues[next((n for n in chain if n in catalogues), chain[-1])]) + name = entry_name(catalogues, version) + entry = catalogues[name] + seeds = [_SEED_SUFFIX.search(n) for n in chain[: chain.index(name)]] + seeds = [m.group()[len("_seed") :] for m in seeds if m] + if seeds: + shear = {**entry["shear"], "path": seed_path(entry["shear"], seeds[0])} + entry = {**entry, "shear": shear} + return _reads(entry) def _concealer(catalogues, recs, base): @@ -381,7 +425,7 @@ def read_stamp(metadata): """The custody a SACC's stamp records; a missing or malformed stamp raises. The result carries no registry: it is what the file says, to compare with - what a catalogue is declared under (``.stamp`` or ``.token``). + what a catalogue is declared under (``==``). """ status = metadata.get("blinding") if status not in STATUSES: diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 4c9529c3..fe69d53f 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -991,22 +991,12 @@ def _check_grid_consistency(s, angle): ) -def is_signal(data_type): - """Whether ``data_type`` carries cosmological signal.""" - return data_type in SIGNAL - - -def is_derived(data_type): - """Whether ``data_type`` is a derived statistic (COSEBIs, pure-E/B).""" - return data_type in DERIVED - - def _refuse_unruled(s): """Refuse rows of a data type with no blinding rule (under a blind).""" unruled = sorted( t for t in {dp.data_type for dp in s.data} - if not is_signal(t) and not _SIGNAL_FREE.fullmatch(t) + if t not in SIGNAL and not _SIGNAL_FREE.fullmatch(t) ) if unruled: raise ValueError( @@ -1045,7 +1035,7 @@ def seal(s, custody): if custody.status == "blinded": _refuse_unruled(s) types = {dp.data_type for dp in s.data} - derived = {t for t in types if is_derived(t)} + derived = types & set(DERIVED) if custody.status == "blinded" and derived: raise ValueError( f"a blinded catalogue's {sorted(derived)} rows are derived " @@ -1071,13 +1061,13 @@ def _derive(s, parts, custody): if not parts: raise ValueError("a derivation needs its input parts") stamps = [_custody.read_stamp(p.metadata) for p in parts] - if len({tuple(sorted(st.stamp.items())) for st in stamps}) > 1: + stamp = stamps[0] + if any(st != stamp for st in stamps): raise ValueError( "input parts carry different custody stamps: " + "; ".join(f"part {i}: {st.token}" for i, st in enumerate(stamps)) ) - stamp = stamps[0] - if custody is not None and custody.stamp != stamp.stamp: + if custody is not None and custody != stamp: raise ValueError( f"parts are stamped {stamp.token}, but {custody.catalogue} is " f"declared {custody.token}" diff --git a/src/sp_validation/tests/test_custody.py b/src/sp_validation/tests/test_custody.py index 65049bbb..54b22a4f 100644 --- a/src/sp_validation/tests/test_custody.py +++ b/src/sp_validation/tests/test_custody.py @@ -47,7 +47,7 @@ def _catalogues(**declarations): for name, declaration in declarations.items(): entries[name] = { "shear": { - "path": f"{name}.fits", + "path": f"{name}_seed00001.fits", "e1_col_corrected": "e1_corrected", "e2_col_corrected": "e2_corrected", } From 4e0a128430029665b747ed931d12fe6c98b2460a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 04:55:50 +0200 Subject: [PATCH 119/160] =?UTF-8?q?cosmo=5Fval:=20jackknife=20pure-E/B=20a?= =?UTF-8?q?nd=20=E2=9F=A8M=5Fap=C2=B2=E2=9F=A9=20from=20sealed=20parts;=20?= =?UTF-8?q?one=20TreeCorr=20=CE=BE=C2=B1=20site?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No plaintext ξ± exists, even in memory: calculate_2pcf is the only ξ± measurement, and both former plaintext callers work from its sealed parts, blinded catalogues included. - Jackknife pure-E/B: the modes come from the reporting and integration parts; the covariance is the integration part's jackknife ξ± covariance pushed through the kernel (b_modes.pure_eb_covariance_from_xi), the reporting ξ± being the pair-weighted average of the integration ξ± in each reporting bin. The blind's shift is the same in every patch, so it leaves that covariance unchanged. - ⟨M_ap²⟩, ⟨M_ײ⟩: TreeCorr's calculateMapSq sum as a matrix on the part's ξ±, with covariance T·C·Tᵀ. The imaginary-part estimates and the map2 text file go: a part carries no imaginary ξ±, and text would leave the door unstamped. - The test compares the pure-E/B covariance with TreeCorr's jackknife of the modes on the synthetic catalogue. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- src/sp_validation/b_modes.py | 133 ++++++++++++++-------- src/sp_validation/cosmo_val/core.py | 18 +-- src/sp_validation/cosmo_val/pure_eb.py | 53 ++++----- src/sp_validation/cosmo_val/real_space.py | 123 +++++++++++--------- src/sp_validation/tests/test_cosmo_val.py | 72 +++++++----- 5 files changed, 225 insertions(+), 174 deletions(-) diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index 89dd5e0a..01743b66 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -11,7 +11,6 @@ import numpy as np import seaborn as sns import tqdm -import treecorr from cs_util.cosmo import get_theo_xi from mpl_toolkits.axes_grid1 import make_axes_locatable from scipy import sparse, stats @@ -128,24 +127,22 @@ def correlation_from_covariance(covariance): def calculate_pure_eb_correlation( gg, gg_int, - var_method="jackknife", cov_path_int=None, cosmo_cov=None, n_samples=1000, z_dist=None, ): """ - Calculate pure E/B modes from correlation function objects. + Calculate pure E/B modes from ξ± on the reporting and integration grids. Parameters ---------- - gg : treecorr.GGCorrelation - Correlation function for reporting binning (coarser binning for final results) - gg_int : treecorr.GGCorrelation - Correlation function for integration binning (fine binning for numerical - integration) - var_method : str, optional - Variance method ("jackknife" or "bootstrap") + gg : sacc_io.xi_correlation view + ξ± on the reporting binning (coarser binning for final results) + gg_int : sacc_io.xi_correlation view + ξ± on the integration binning (fine binning for numerical integration); + without ``cov_path_int``, its jackknife covariance is propagated + (:func:`pure_eb_covariance_from_xi`) cov_path_int : str, optional Path to integration covariance matrix for semi-analytical calculation cosmo_cov : pyccl.Cosmology, optional @@ -161,22 +158,6 @@ def calculate_pure_eb_correlation( dict Dictionary containing pure E/B mode results and covariance """ - # Calculate min_sep and max_sep from gg object - min_sep, max_sep = gg.left_edges[0], gg.right_edges[-1] - - def pure_EB(corrs): - gg, gg_int = corrs - return pure_eb_from_xi( - theta_report=gg.meanr, - xip_report=gg.xip, - xim_report=gg.xim, - theta_int=gg_int.meanr, - xip_int=gg_int.xip, - xim_int=gg_int.xim, - tmin=min_sep, - tmax=max_sep, - ) - # The results dict is self-describing: the grids it was measured on travel # with the modes, so every consumer downstream works from values alone. results = { @@ -190,9 +171,20 @@ def pure_EB(corrs): "theta_int": gg_int.meanr, "xip_int": gg_int.xip, "xim_int": gg_int.xim, - "n_eff": n_samples if cov_path_int is not None else gg.npatch1, + "n_eff": n_samples if cov_path_int is not None else gg_int.npatch1, } - results.update(pure_EB([gg, gg_int])) + results.update( + pure_eb_from_xi( + gg.meanr, + gg.xip, + gg.xim, + gg_int.meanr, + gg_int.xip, + gg_int.xim, + gg.left_edges[0], + gg.right_edges[-1], + ) + ) if cov_path_int is not None: if z_dist is None or cosmo_cov is None: @@ -212,12 +204,14 @@ def pure_EB(corrs): ) results.update({"cov": cov, "eb_samples": eb_samples}) else: - # Use existing treecorr covariance estimation - results["cov"] = treecorr.estimate_multi_cov( - [gg, gg_int], - var_method, - func=lambda x: _eb_vector(pure_EB(x)), - cross_patch_weight="match" if var_method == "jackknife" else None, + results["cov"] = pure_eb_covariance_from_xi( + theta=gg.meanr, + left_edges=gg.left_edges, + right_edges=gg.right_edges, + theta_int=gg_int.meanr, + xi_int=np.concatenate([gg_int.xip, gg_int.xim]), + weight_int=gg_int.weight, + cov_int=gg_int.cov, ) # Validate covariance matrix @@ -294,18 +288,7 @@ def pure_eb_covariance_mc( """ theta, theta_int = np.asarray(theta), np.asarray(theta_int) nbins_int = len(theta_int) - - # Each reporting bin averages the integration bins that fall inside it. - reporting_bin_edges = np.concatenate([left_edges, [right_edges[-1]]]) - bin_indices = np.digitize(theta_int, reporting_bin_edges) - 1 - valid_mask = (bin_indices >= 0) & (bin_indices < len(theta)) - row_indices, col_indices = (bin_indices[valid_mask], np.where(valid_mask)[0]) - binning_matrix = sparse.csr_matrix( - (np.ones(len(row_indices)), (row_indices, col_indices)), - shape=(len(theta), nbins_int), - ) - row_sums = np.array(binning_matrix.sum(axis=1)).flatten() - binning_matrix = sparse.diags(1 / row_sums) @ binning_matrix + binning_matrix = _reporting_binning(left_edges, right_edges, theta_int) # One n(z) gives one tracer pair: get_theo_xi's single (xi+, xi-) entry. (xi_pm,) = get_theo_xi( @@ -340,6 +323,64 @@ def eb_draw(i): return np.cov(eb_samples.T), eb_samples +def _reporting_binning(left_edges, right_edges, theta_int, weight_int=None): + """The matrix averaging integration-grid ξ± into each reporting bin. + + Each reporting bin averages the integration bins whose centres fall inside + it, weighted by ``weight_int`` (their pair weights: TreeCorr's estimate on + the reporting bin), or uniformly. + """ + edges = np.concatenate([left_edges, [right_edges[-1]]]) + rows = np.digitize(theta_int, edges) - 1 + inside = (rows >= 0) & (rows < len(left_edges)) + weight = np.ones(len(theta_int)) if weight_int is None else np.asarray(weight_int) + matrix = sparse.csr_matrix( + (weight[inside], (rows[inside], np.flatnonzero(inside))), + shape=(len(left_edges), len(theta_int)), + ) + return sparse.diags(1 / np.asarray(matrix.sum(axis=1)).ravel()) @ matrix + + +def pure_eb_covariance_from_xi( + *, theta, left_edges, right_edges, theta_int, xi_int, weight_int, cov_int +): + """Pure-E/B covariance T·C·Tᵀ from the integration-grid ξ± covariance C. + + The modes are linear in ξ±, and the reporting ξ± is the pair-weighted + average of the integration ξ± inside each reporting bin, so T is the + kernel composed with that average. Each column u of a factor C = Σ u uᵀ + is pushed through as K(ξ + u) − K(ξ), about the measured ``xi_int`` + ([ξ+, ξ−]), as a jackknife pushes each patch's ξ±: the kernel's + quadrature is accurate on a ξ± shaped like a measurement, not on a bare + fluctuation. A jackknife C has rank below its patch count, which bounds + the kernel calls. + """ + values, vectors = np.linalg.eigh(np.asarray(cov_int)) + keep = values > values.max() * len(values) * np.finfo(float).eps + factor = vectors[:, keep] * np.sqrt(values[keep]) + binning = _reporting_binning(left_edges, right_edges, theta_int, weight_int) + nbins_int = len(theta_int) + + def transformed(xi): + xip_int, xim_int = xi[:nbins_int], xi[nbins_int:] + modes = pure_eb_from_xi( + theta_report=theta, + xip_report=binning @ xip_int, + xim_report=binning @ xim_int, + theta_int=theta_int, + xip_int=xip_int, + xim_int=xim_int, + tmin=left_edges[0], + tmax=right_edges[-1], + ) + return _eb_vector(modes) + + xi_int = np.asarray(xi_int) + centre = transformed(xi_int) + columns = np.column_stack([transformed(xi_int + u) - centre for u in factor.T]) + return columns @ columns.T + + def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): """ Calculate COSEBIs modes from a correlation function for multiple scale cuts. diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 31db4722..aa4b7bd6 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -13,13 +13,7 @@ _get_pte_from_scale_cut, find_conservative_scale_cut_key, ) -from ..custody import ( - CustodyError, - base_catalogue, - custody_of, - registry_of, - seed_path, -) +from ..custody import base_catalogue, custody_of, registry_of, seed_path from ..statistics import chi2_and_pte from ..version import __version__ from .catalog_characterization import CatalogCharacterizationMixin @@ -475,16 +469,6 @@ def patch_centers_path(self, version, npatch): base = base_catalogue(self._declared, version) return self._output_path("patches", f"{base}_npatch={int(npatch)}.dat") - def _refuse_if_blinded(self, version, what): - """Refuse ``what``, which reads ``version``'s ξ± in plaintext, if blinded.""" - custody = self.custody(version) - if custody.status == "blinded": - raise CustodyError( - f"{what} works from {version}'s measured ξ± itself, and " - f"{custody.catalogue} is blinded; derive it from the concealed " - "part calculate_2pcf returns" - ) - def basename(self, version, treecorr_config=None, npatch=None): cfg = treecorr_config or self.treecorr_config patches = npatch or self.npatch diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 5f7632dd..aef21725 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -7,6 +7,7 @@ import numpy as np +from .. import sacc_io from ..b_modes import ( calculate_eb_statistics, calculate_pure_eb_correlation, @@ -29,7 +30,6 @@ def calculate_pure_eb( max_sep_int=300, nbins_int=1000, npatch=256, - var_method="jackknife", cov_path_int=None, cosmo_cov=None, n_samples=1000, @@ -59,14 +59,12 @@ def calculate_pure_eb( nbins_int : int, optional Number of bins for the integration binning. Defaults to 1000. npatch : int, optional - Number of patches for the jackknife or bootstrap resampling. Defaults to - the value in self.npatch if not provided. - var_method : str, optional - Variance estimation method. Defaults to "jackknife". + Number of jackknife patches. Defaults to the value in self.npatch if + not provided. cov_path_int : str, optional - Path to the covariance matrix for the reporting binning. Replaces the - treecorr covariance matrix if provided, meaning that var_method has no - effect on the results. + Path to the integration-grid ξ± covariance. When given, the + covariance is Monte Carlo through the kernel from it, instead of the + jackknife. cosmo_cov : pyccl.Cosmology, optional Cosmology object to use for theoretical xi+/xi- predictions in the semi-analytical covariance calculation. Defaults to self.cosmo if not @@ -99,25 +97,30 @@ def calculate_pure_eb( Notes ----- - - Both binnings are measured on the version's persisted patch centres - (:meth:`patch_centers_path`). The jackknife works from TreeCorr's own - measurement, so a blinded catalogue is refused. + - Both binnings are the version's sealed ξ± parts + (:meth:`calculate_2pcf`), split at its persisted patch centres, so a + blinded catalogue's modes are concealed. The jackknife covariance is + the integration part's, pushed through the kernel + (:func:`~sp_validation.b_modes.pure_eb_covariance_from_xi`): the + shift, the same in every patch, leaves it unchanged. """ self.print_start(f"Computing {version} pure E/B") - self._refuse_if_blinded(version, "The jackknife pure-E/B") - npatch = int(npatch or self.npatch) - centres = self._patch_centers(version, npatch) - gg = self._measure_xi( - version, npatch, centres, min_sep=min_sep, max_sep=max_sep, nbins=nbins - ) - gg_int = self._measure_xi( - version, - npatch, - centres, - min_sep=min_sep_int, - max_sep=max_sep_int, - nbins=nbins_int, + gg, gg_int = ( + sacc_io.xi_correlation( + self.calculate_2pcf( + version, + grid=grid, + npatch=npatch, + min_sep=lo, + max_sep=hi, + nbins=n, + ) + ) + for grid, lo, hi, n in ( + ("reporting", min_sep, max_sep, nbins), + ("integration", min_sep_int, max_sep_int, nbins_int), + ) ) # Get redshift distribution if using analytic covariance @@ -131,7 +134,6 @@ def calculate_pure_eb( results = calculate_pure_eb_correlation( gg=gg, gg_int=gg_int, - var_method=var_method, cov_path_int=cov_path_int, cosmo_cov=cosmo_cov, n_samples=n_samples, @@ -270,7 +272,6 @@ def plot_pure_eb( max_sep_int=max_sep_int, nbins_int=nbins_int, npatch=npatch, - var_method=var_method, cov_path_int=cov_path_int, cosmo_cov=cosmo_cov, n_samples=n_samples, diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 36abb25e..6f19ee24 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -77,39 +77,11 @@ def calculate_2pcf( if patch_centers is not None: with open(patch_centers, "rb") as f: metadata["patch_centers_sha256"] = hashlib.sha256(f.read()).hexdigest() - gg = self._measure_xi(ver, npatch, patch_centers, **treecorr_config) - jackknife = npatch > 1 - s = xi_to_sacc( - self.sacc_nz(ver), - metadata, - gg.meanr, - gg.xip, - gg.xim, - grid=grid, - theta_nom=gg.rnom, - npairs=gg.npairs, - weight=gg.weight, - covariance=gg.cov if jackknife else None, - variances=None if jackknife else np.concatenate([gg.varxip, gg.varxim]), - ) - part = ( - sacc_io.save(s, out, custody=custody) if out else sacc_io.seal(s, custody) - ) - self.xi_parts[ver, grid] = part - self.print_done("Done 2PCF") - return part - - def _measure_xi(self, ver, npatch, patch_centers=None, **treecorr_config): - """TreeCorr's ξ± of ``ver``, in plaintext: for this object's use only. - - Jackknife variances with patches, shot noise without. With patches and - no ``patch_centers``, TreeCorr draws its own centres. - """ gg = treecorr.GGCorrelation( { **self._binning(**treecorr_config), - "var_method": "jackknife" if npatch > 1 else "shot", + "var_method": "jackknife" if jackknife else "shot", } ) with self.results[ver].temporarily_read_data(): @@ -126,7 +98,26 @@ def _measure_xi(self, ver, npatch, patch_centers=None, **treecorr_config): patch_centers=patch_centers, ) gg.process(catalogue) - return gg + + s = xi_to_sacc( + self.sacc_nz(ver), + metadata, + gg.meanr, + gg.xip, + gg.xim, + grid=grid, + theta_nom=gg.rnom, + npairs=gg.npairs, + weight=gg.weight, + covariance=gg.cov if jackknife else None, + variances=None if jackknife else np.concatenate([gg.varxip, gg.varxim]), + ) + part = ( + sacc_io.save(s, out, custody=custody) if out else sacc_io.seal(s, custody) + ) + self.xi_parts[ver, grid] = part + self.print_done("Done 2PCF") + return part def _patch_centers(self, ver, npatch, path=None): """The existing centres file ``ver`` splits at, or None without patches.""" @@ -399,37 +390,39 @@ def plot_ratio_xi_sys_xi(self, threshold=0.1, offset=0.02): print(f"Ratio of xi_psf_sys to xi plot saved to {out_path}") def calculate_aperture_mass_dispersion( - self, theta_min=0.3, theta_max=200, nbins=500, nbins_map=15, npatch=25 + self, theta_min=0.3, theta_max=200, nbins=500, nbins_map=15, npatch=None ): + """⟨M_ap²⟩ and ⟨M_ײ⟩ of every version, from its sealed ξ± part. + + Both are linear in ξ± (TreeCorr's ``calculateMapSq`` sum, Schneider et + al. 2002 filter), so they and their jackknife covariance T·C·Tᵀ come + from the part on a fine grid: concealed on a blinded catalogue, whose + shift, the same in every patch, leaves C unchanged. + """ self.print_start("Computing aperture-mass dispersion") - self._map2 = {} theta_map = np.geomspace(theta_min * 5, theta_max / 2, nbins_map) - self._map2["theta_map"] = theta_map - + self._map2 = {"theta_map": theta_map} + bin_size = np.log(theta_max / theta_min) / nbins for ver in self.versions: self.print_magenta(ver) - self._refuse_if_blinded(ver, "The aperture-mass dispersion") - gg = self._measure_xi( - ver, npatch, min_sep=theta_min, max_sep=theta_max, nbins=nbins - ) - - mapsq, mapsq_im, mxsq, mxsq_im, varmapsq = gg.calculateMapSq( - R=theta_map, - m2_uform="Schneider", + part = self.calculate_2pcf( + ver, + grid="aperture_mass", + npatch=npatch, + min_sep=theta_min, + max_sep=theta_max, + nbins=nbins, ) - out_fname_map2 = self._output_path(f"map2_{ver}.txt") - if os.path.exists(out_fname_map2): - self.print_green(f"Skipping Map2, {out_fname_map2} exists") - else: - print(f"Writing Map2 to output file {out_fname_map2} ") - gg.writeMapSq(out_fname_map2, R=theta_map, m2_uform="Schneider") + gg = sacc_io.xi_correlation(part) + transform = _map2_transform(theta_map, gg.meanr, bin_size) + mapsq, mxsq = np.split(transform @ np.concatenate([gg.xip, gg.xim]), 2) + variances = np.split(np.diag(transform @ gg.cov @ transform.T), 2) self._map2[ver] = { "mapsq": mapsq, - "mapsq_im": mapsq_im, "mxsq": mxsq, - "mxsq_im": mxsq_im, - "varmapsq": varmapsq, + "varmapsq": variances[0], + "varmxsq": variances[1], } self.print_done("Done aperture-mass dispersion") @@ -441,10 +434,10 @@ def map2(self): return self._map2 def plot_aperture_mass_dispersion(self): - for mode in ["mapsq", "mapsq_im", "mxsq", "mxsq_im"]: + for mode in ["mapsq", "mxsq"]: x = [self.map2["theta_map"] for ver in self.versions] y = [self.map2[ver][mode] for ver in self.versions] - yerr = [np.sqrt(self.map2[ver]["varmapsq"]) for ver in self.versions] + yerr = [np.sqrt(self.map2[ver][f"var{mode}"]) for ver in self.versions] labels = list(self.versions) colors = [self.cc[ver]["colour"] for ver in self.versions] linestyles = [self.cc[ver]["ls"] for ver in self.versions] @@ -473,10 +466,10 @@ def plot_aperture_mass_dispersion(self): cs_plots.show() self.print_done(f"linear-scale {mode} plot saved to {out_path}") - for mode in ["mapsq", "mapsq_im", "mxsq", "mxsq_im"]: + for mode in ["mapsq", "mxsq"]: x = [self.map2["theta_map"] for ver in self.versions] y = [np.abs(self.map2[ver][mode]) for ver in self.versions] - yerr = [np.sqrt(self.map2[ver]["varmapsq"]) for ver in self.versions] + yerr = [np.sqrt(self.map2[ver][f"var{mode}"]) for ver in self.versions] xlabel = r"$\theta$ [arcmin]" ylabel = "dispersion" title = f"Aperture-mass dispersion mode {mode}" @@ -501,3 +494,23 @@ def plot_aperture_mass_dispersion(self): cs_plots.savefig(out_path, close_fig=False) cs_plots.show() self.print_done(f"log-scale {mode} plot saved to {out_path}") + + +def _map2_transform(radii, theta, bin_size): + """The matrix taking [ξ+, ξ−] on a log grid to [⟨M_ap²⟩, ⟨M_ײ⟩] at ``radii``. + + TreeCorr's ``calculateMapSq`` sum with the Schneider et al. (2002) filter: + ⟨M_ap²⟩, ⟨M_ײ⟩ = Σ s² (T+ ξ+ ± T− ξ−) dlnθ / 2, s = θ/R, T± zero for s ≥ 2. + """ + s = np.minimum(np.outer(1.0 / radii, theta), 2.0) + ssq = s * s + tp = 12.0 / (5.0 * np.pi) * (2.0 - 15.0 * ssq) * np.arccos(s / 2.0) + tp += ( + s + * np.sqrt(4.0 - ssq) + * (120.0 + ssq * (2320.0 + ssq * (-754.0 + ssq * (132.0 - 9.0 * ssq)))) + / (100.0 * np.pi) + ) + tm = 3.0 / (70.0 * np.pi) * s * ssq * (4.0 - ssq) ** 3.5 + tp, tm = (x * ssq * 0.5 * bin_size for x in (tp, tm)) + return np.block([[tp, tm], [tp, -tm]]) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 500999b3..8273032e 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -352,8 +352,10 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( The ξ± it measures equal the committed ``pure_eb_xi``, its modes are ``pure_eb_from_xi`` of those ξ± and edges, and every reporting bin is - finite. The jackknife covariance runs the transform once per - realisation. + finite. Its jackknife covariance, the integration part's pushed + through the kernel, matches TreeCorr's jackknife of the modes in each + statistic's total variance, loosely: on this sparse toy ξ± the kernel's + ξ− quadrature is additive to only ~30%. Finiteness: the Schneider (2022) integrals are near-singular where a reporting bin meets the integration boundary, so the integration grid @@ -386,11 +388,19 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( cv.treecorr_config.update(bin_slop=0, angle_slop=0) cv.write_patch_centers(version, npatch) + import treecorr + kernel, kernel_calls = b_modes.pure_eb_from_xi, [] monkeypatch.setattr( b_modes, "pure_eb_from_xi", - lambda **kw: kernel_calls.append(kw) or kernel(**kw), + lambda *a, **kw: kernel_calls.append(kw) or kernel(*a, **kw), + ) + process, correlations = treecorr.GGCorrelation.process, [] + monkeypatch.setattr( + treecorr.GGCorrelation, + "process", + lambda gg, *a, **kw: correlations.append(gg) or process(gg, *a, **kw), ) results = cv.calculate_pure_eb( version, @@ -399,8 +409,8 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( max_sep_int=300.0, nbins_int=600, ) - # The modes, then TreeCorr's jackknife: one sizing call and one per patch. - assert len(kernel_calls) <= npatch + 2, f"{len(kernel_calls)} transforms" + # The modes, the measured ξ±, and one per jackknife eigenvector (< npatch). + assert len(kernel_calls) <= npatch + 1, f"{len(kernel_calls)} transforms" measured = { "theta_report": results["theta"], @@ -425,9 +435,34 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( assert np.all(np.isfinite(vec)), f"{key} not finite" np.testing.assert_allclose(vec, modes[key], rtol=1e-10, err_msg=key) - # Jackknife covariance over the 6 stats (xip/xim x E/B/amb) x nbins. + # TreeCorr's jackknife of the modes, from the two measurements. + def modes_of(pair): + gg, gg_int = pair + return b_modes._eb_vector( + kernel( + gg.meanr, + gg.xip, + gg.xim, + gg_int.meanr, + gg_int.xip, + gg_int.xim, + gg.left_edges[0], + gg.right_edges[-1], + ) + ) + + reference = treecorr.estimate_multi_cov( + correlations, "jackknife", func=modes_of, cross_patch_weight="match" + ) cov = np.asarray(results["cov"]) - assert cov.shape == (6 * nbins, 6 * nbins) + assert cov.shape == reference.shape == (6 * nbins, 6 * nbins) + + def block_variances(c): + return np.diag(c).reshape(6, nbins).sum(axis=1) + + np.testing.assert_allclose( + block_variances(cov), block_variances(reference), rtol=0.4 + ) assert results["n_eff"] == npatch @@ -486,26 +521,3 @@ def test_a_blinded_catalogues_xi_leaves_concealed(blinded_and_twin): ) assert blinded.metadata["blinding"] == "blinded" assert cv.xi_parts["TOY", "reporting"] is blinded - - -@pytest.mark.parametrize( - "measure", - [ - lambda cv: cv.calculate_pure_eb("TOY", npatch=1), - lambda cv: cv.calculate_aperture_mass_dispersion(npatch=1), - ], - ids=["jackknife-pure-eb", "aperture-mass"], -) -def test_plaintext_measurements_refuse_a_blinded_catalogue( - blinded_and_twin, measure, monkeypatch -): - """The jackknife pure-E/B and the aperture mass work from TreeCorr's ξ± - itself, so on a blinded catalogue they refuse before measuring anything.""" - from sp_validation.custody import CustodyError - - def measured(*args, **kwargs): - raise AssertionError("measured a blinded catalogue's ξ± in plaintext") - - monkeypatch.setattr(CosmologyValidation, "_measure_xi", measured) - with pytest.raises(CustodyError, match="TOY is blinded"): - measure(blinded_and_twin) From a06051414792269604f7cf22799253ff71c3f93d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 05:05:20 +0200 Subject: [PATCH 120/160] custody: a seed variant's custody needs no file; only its readers do The twins check skips a seed file its entry cannot name, since no entry reads it; rule xi and CosmologyValidation refuse it where they read it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- src/sp_validation/custody.py | 7 ++++++- src/sp_validation/tests/test_custody.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/custody.py b/src/sp_validation/custody.py index 474c1691..5e7979e8 100644 --- a/src/sp_validation/custody.py +++ b/src/sp_validation/custody.py @@ -275,7 +275,12 @@ def refuse_twins(catalogues, recs, version): is concealed under one blind, or none is. """ base = base_catalogue(catalogues, version) - here = shear_file(catalogues, version) + try: + here = shear_file(catalogues, version) + except CustodyError: + # A seed file its entry cannot name, so no entry reads it: whatever + # would read it (rule xi, CosmologyValidation) refuses it there. + here = None readers = { name: base_catalogue(catalogues, name) for name, entry in catalogues.items() diff --git a/src/sp_validation/tests/test_custody.py b/src/sp_validation/tests/test_custody.py index 54b22a4f..65049bbb 100644 --- a/src/sp_validation/tests/test_custody.py +++ b/src/sp_validation/tests/test_custody.py @@ -47,7 +47,7 @@ def _catalogues(**declarations): for name, declaration in declarations.items(): entries[name] = { "shear": { - "path": f"{name}_seed00001.fits", + "path": f"{name}.fits", "e1_col_corrected": "e1_corrected", "e2_col_corrected": "e2_corrected", } From 91fa37a19012b55e434c347c272b7a95a0ad7045 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 05:05:21 +0200 Subject: [PATCH 121/160] cat_config: the leakage n(z) is a path template, not an A/B/C blind key nz.dndz.path names the file with {pipeline}; the same file as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- cosmo_val/cat_config.yaml | 4 ++-- src/sp_validation/cosmo_val/psf_systematics.py | 4 ++-- src/sp_validation/tests/_synthetic.py | 2 +- src/sp_validation/tests/test_cosmo_val.py | 2 +- src/sp_validation/tests/test_pseudo_cl.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index 0e493fec..59b0adc0 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -1191,9 +1191,9 @@ SP_v1.4.8_uncal: patch_number: 100 nz: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/nz + # The n(z) of the PSF-leakage theory, by pipeline. dndz: - blind: A - path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz + path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_{pipeline}_A.txt paths: output: ./output SP_v1.6.6: diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index 12652e81..5f6b8cbd 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -314,8 +314,8 @@ def set_params_leakage_scale(self, ver): # Set parameters params_in["input_path_shear"] = self.cc[ver]["shear"]["path"] params_in["input_path_PSF"] = self.cc[ver]["star"]["path"] - params_in["dndz_path"] = ( - f"{self.cc['nz']['dndz']['path']}_{self.cc[ver]['pipeline']}_{self.cc['nz']['dndz']['blind']}.txt" + params_in["dndz_path"] = self.cc["nz"]["dndz"]["path"].format( + pipeline=self.cc[ver]["pipeline"] ) params_in["output_dir"] = f"{self.cc['paths']['output']}/leakage_{ver}" diff --git a/src/sp_validation/tests/_synthetic.py b/src/sp_validation/tests/_synthetic.py index 9c6bef7e..18e36f25 100644 --- a/src/sp_validation/tests/_synthetic.py +++ b/src/sp_validation/tests/_synthetic.py @@ -131,7 +131,7 @@ def write_synthetic_catalogs( } config = { - "nz": {"subdir": str(nz_dir), "dndz": {"blind": "A", "path": "dndz"}}, + "nz": {"subdir": str(nz_dir), "dndz": {"path": "dndz_{pipeline}_A.txt"}}, "paths": {"output": str(output_dir)}, } for version, declaration in catalogues.items(): diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 8273032e..8ce2d89b 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -75,7 +75,7 @@ def _make_seed_config(tmp_path, shear_filename): config_data = { "nz": { "subdir": str(nz_dir), - "dndz": {"blind": "A", "path": "dndz.txt"}, + "dndz": {"path": "dndz.txt"}, }, "paths": {"output": str(output_dir)}, base_version: { diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index f0ddc8c8..2efe24e8 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -141,7 +141,7 @@ def _write_synthetic_config(tmp_path): "star_flag": "w", } config_data = { - "nz": {"subdir": str(nz_dir), "dndz": {"blind": "A", "path": "dndz"}}, + "nz": {"subdir": str(nz_dir), "dndz": {"path": "dndz_{pipeline}_A.txt"}}, "paths": {"output": str(output_dir)}, version: { "subdir": str(cat_dir), From 966954629ffc676ec033eced5effbe7edf6dd894 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 05:05:21 +0200 Subject: [PATCH 122/160] tests: restore the candide toy run of rule xi, before and after a blind Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- workflow/tests/test_toy_run.py | 204 +++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 workflow/tests/test_toy_run.py diff --git a/workflow/tests/test_toy_run.py b/workflow/tests/test_toy_run.py new file mode 100644 index 00000000..10e6b16c --- /dev/null +++ b/workflow/tests/test_toy_run.py @@ -0,0 +1,204 @@ +"""Rule xi, run for real through apptainer, before and after a blind is drawn. + +The launch is the README's with the machine-independent default profile: the +host Snakemake, the image `spv-container` manages, jobs on this node. A toy +checkout under your home directory (which the profile binds) carries copies of +workflow/, papers/cosmo_val/ and src/, and one synthetic catalogue declared +unblinded. The first launch stops for want of its patch centres and names the +command that draws them; after it has run, its reporting parts and their figure +are made; then the catalogue is declared blinded, a blind is drawn for it, and +the same launch re-measures the parts concealed. The jobs run under a +matplotlibrc asking for a LaTeX package no image has, as a user's own may. +""" + +import json +import os +import shlex +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import pytest +import yaml +from conftest import REPO, container, on_candide + +VERSIONS = ("SP_v0.1", "SP_v0.1_leak_corr") +REPORTING = {"theta_min": 5.0, "theta_max": 60.0, "nbins": 6, "npatch": 4} +BINNING = "minsep=5.0_maxsep=60.0_nbins=6_npatch=4" + + +def _in_image(image, root, *command): + """``command``'s stdout, run in ``image`` on the toy checkout's src/.""" + return subprocess.run( + [ + "apptainer", + "exec", + "--cleanenv", + "--env", + f"PYTHONPATH={root / 'src'}", + str(image), + *command, + ], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout + + +def _stamps(image, root, parts): + """The custody stamp of each SACC part, read in the image.""" + script = ( + "import json, sys\n" + "from sp_validation import custody, sacc_io\n" + "print(json.dumps([custody.read_stamp(sacc_io.load(p).metadata).stamp" + " for p in sys.argv[1:]]))" + ) + return json.loads(_in_image(image, root, "python", "-c", script, *map(str, parts))) + + +def _toy_checkout(root, image): + """A checkout with one synthetic catalogue, SP_v0.1, declared unblinded.""" + skip = shutil.ignore_patterns(".snakemake", "__pycache__", "tests") + shutil.copytree(REPO / "workflow", root / "workflow", ignore=skip) + shutil.copytree( + REPO / "papers" / "cosmo_val", root / "papers" / "cosmo_val", ignore=skip + ) + shutil.copytree( + REPO / "src", root / "src", ignore=shutil.ignore_patterns("__pycache__") + ) + (root / "cosmo_val").mkdir() + _in_image( + image, + root, + "python", + "-c", + "import sys\n" + f"sys.path.insert(0, {str(root / 'src/sp_validation/tests')!r})\n" + "from pathlib import Path\n" + "from _synthetic import write_synthetic_catalogs\n" + f"write_synthetic_catalogs(Path({str(root / 'cosmo_val')!r})," + f" catalogues={{{VERSIONS[0]!r}: 'unblinded'}})", + ) + + config_path = root / "papers" / "cosmo_val" / "config" / "config.yaml" + config = yaml.safe_load(config_path.read_text()) + config["versions"] = list(VERSIONS) + config["fiducial"]["version"] = VERSIONS[1] + config["fiducial"]["mock_version"] = VERSIONS[0] + config["cosmo_val"].update(REPORTING) + config_path.write_text(yaml.safe_dump(config, sort_keys=False)) + + # A user's matplotlibrc typesetting with a package the image lacks. + rc = root / "xdg" / "matplotlib" / "matplotlibrc" + rc.parent.mkdir(parents=True) + rc.write_text( + "text.usetex: True\n" + "text.latex.preamble: \\usepackage{spvalidationabsentpackage}\n" + ) + (root / "fast.json").write_text( + json.dumps({"theory": {"transfer_function": "eisenstein_hu"}}) + ) + + +@pytest.mark.candide +@on_candide +def test_xi_before_and_after_its_catalogue_is_blinded(): + image, kind = container.resolve_image() + assert kind != "tag", "no local image; run `spv-container pull`" + root = Path(tempfile.mkdtemp(prefix="toy_run_", dir=Path.home())) + _toy_checkout(root, image) + out = root / "out" + cat_config = root / "cosmo_val" / "cat_config.yaml" + parts = [out / f"{v}_xi_{BINNING}.sacc" for v in VERSIONS] + target = str(out / "snakemake_sentinels" / "plot_2pcf.done") + + env = { + k: v + for k, v in os.environ.items() + if k not in ("SNAKEMAKE_PROFILE", "APPTAINERENV_PYTHONPATH") + } + env.update( + COSMO_VAL=str(out), + COSMO_INFERENCE=str(root / "inference"), + XDG_CACHE_HOME=str(root / "cache"), + APPTAINERENV_XDG_CONFIG_HOME=str(root / "xdg"), + PYTHONNOUSERSITE="1", + PYTHONUNBUFFERED="1", + ) + + def launch(*args): + return subprocess.run( + [ + sys.executable, + "-m", + "snakemake", + "--profile", + str(root / "workflow" / "profiles" / "default"), + "--cores", + "4", + *args, + "--config", + f"container={image}", + "--", + target, + ], + cwd=root / "papers" / "cosmo_val", + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=1800, + check=False, + ) + + # No centres: the launch names the command that draws them, which works. + result = launch() + assert result.returncode != 0, result.stdout + (command,) = [ + line.split("spv-container exec ", 1)[1] + for line in result.stdout.splitlines() + if "sp_validation.cosmo_val.patch_centers" in line + ] + _in_image(image, root, *shlex.split(command)) + assert (out / "patches" / "SP_v0.1_npatch=4.dat").is_file() + + # Declared unblinded: parts stamped unblinded, and their figure drawn. + result = launch() + assert result.returncode == 0, result.stdout + assert [s["blinding"] for s in _stamps(image, root, parts)] == ["unblinded"] * 2 + assert (out / "xi_p.png").is_file() + + # Declared blinded, and its blind drawn: the parts' params changed. + catalogues = yaml.safe_load(cat_config.read_text()) + catalogues[VERSIONS[0]]["blinding"] = "blinded" + cat_config.write_text(yaml.safe_dump(catalogues, sort_keys=False)) + _in_image( + image, + root, + "python", + "-m", + "sp_validation.blinding", + "init", + "toy", + VERSIONS[0], + "--cat-config", + str(cat_config), + "--config", + str(root / "fast.json"), + ) + dry = launch("-n") + assert dry.returncode == 0, dry.stdout + assert "Params have changed" in dry.stdout, dry.stdout + + result = launch() + assert result.returncode == 0, result.stdout + record = json.loads((root / "cosmo_val/blinds/toy/commitment.json").read_text()) + for stamp in _stamps(image, root, parts): + assert stamp["blinding"] == "blinded", stamp + assert stamp["blinding_commitment"] == record["seed_commitment"], stamp + assert not list(out.rglob("*_xi_*.txt")) + assert not list((root / "cosmo_val" / "output").iterdir()) + + shutil.rmtree(root) # kept on failure, for post-mortem From ad03426f4ae3072f333132f1380f1b6b26fa4779 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 05:13:59 +0200 Subject: [PATCH 123/160] pure_eb: its reporting part has its own grid tag, so it never replaces the fiducial one Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- src/sp_validation/cosmo_val/pure_eb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index aef21725..cc6515a5 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -118,7 +118,7 @@ def calculate_pure_eb( ) ) for grid, lo, hi, n in ( - ("reporting", min_sep, max_sep, nbins), + ("pure_eb_reporting", min_sep, max_sep, nbins), ("integration", min_sep_int, max_sep_int, nbins_int), ) ) From bf1362be0c5185249d3cfa65e3d2f0a670ef58c7 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 02:39:42 +0200 Subject: [PATCH 124/160] profiles: wait 60 s for a job's outputs to appear P5 (one real SLURM job through the candide profile, from a login node) passed, but only on its retry: the job finished, its output took more than 5 s to show on the login node's /home, and Snakemake re-ran it. On a multi-hour job that retry costs hours, and a second miss fails the run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- workflow/profiles/candide/config.yaml | 6 ++++-- workflow/profiles/default/config.yaml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 01230785..466d7cd2 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -20,8 +20,10 @@ software-deployment-method: apptainer rerun-triggers: ["mtime", "params", "input", "code"] # Give an appearing output file a moment on networked filesystems before -# Snakemake calls a job failed for a missing output. -latency-wait: 5 +# Snakemake calls a job failed for a missing output. A job's write can take +# more than 5 s to show on the launching host (candide's /home does); a +# shorter wait reads a finished job as failed and re-runs it. +latency-wait: 60 # --- end GENERIC ------------------------------------------------------------ # No ``apptainer-prefix``: the entry Snakefiles resolve ``container:`` to this diff --git a/workflow/profiles/default/config.yaml b/workflow/profiles/default/config.yaml index bf272c03..cd42355a 100644 --- a/workflow/profiles/default/config.yaml +++ b/workflow/profiles/default/config.yaml @@ -25,8 +25,10 @@ software-deployment-method: apptainer rerun-triggers: ["mtime", "params", "input", "code"] # Give an appearing output file a moment on networked filesystems before -# Snakemake calls a job failed for a missing output. -latency-wait: 5 +# Snakemake calls a job failed for a missing output. A job's write can take +# more than 5 s to show on the launching host (candide's /home does); a +# shorter wait reads a finished job as failed and re-runs it. +latency-wait: 60 # --- end GENERIC ------------------------------------------------------------ # Binds are the one thing you almost certainly need to edit for your machine: From 6453924350a999b98b80d4d5bf040ac39f355cae Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 03:54:08 +0200 Subject: [PATCH 125/160] =?UTF-8?q?rho=5Ftau=5Fstats:=206=20h=20wall=20clo?= =?UTF-8?q?ck;=20the=20jackknife=20=CF=81/=CF=84=20outruns=20the=2060=20mi?= =?UTF-8?q?n=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- workflow/rules/twopoint.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index eeb11038..4e8b75f6 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -62,6 +62,7 @@ rule rho_tau_stats: resources: mem_mb=30000, disk_mb=20000, + runtime=360, script: "../scripts/run_rho_tau.py" From 411e4c2641a78d9c68d3a85da027cd26a9b13f1c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 04:33:58 +0200 Subject: [PATCH 126/160] Scrub the n(z) A/B/C blind: an n(z) is a catalogue entry's Choosing an n(z) is choosing a catalogue entry: each entry's shear.redshift_path is its n(z), read by CosmologyValidation.get_redshift and by the workflow's redshift_path(version). CosmologyValidation loses its blind override, and the pseudo-Cl, covariance, inference and papers/bmodes filenames lose their blind token and wildcard. - common: catalogue_entry()/redshift_path() replace build_redshift_path; covariance_{base,dir,path} and pseudo_cl_tag drop blind; the version constraint admits the _A/_B/_C entries. - covariance.smk: get_cat_params and the star-halo mask resolve through the catalogue entry. - papers/bmodes: bb_covariance_blind_independence and the talk n(z) plot compare fiducial.nz_realisations, the SP_v1.4.6.3_{A,B,C}_leak_corr entries; pure-E/B, COSEBIs PTE and pseudo-Cl paths are per version. - cat_config: SP_v1.4.6.3_{A,B,C} match SP_v1.4.6.3 but for their n(z); SP_v1.4.8 and SP_v1.4.11.3(_ecut07) name the n(z) their covariances used. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- cosmo_val/cat_config.yaml | 48 +++---- .../bb_covariance_blind_independence.md | 7 +- papers/bmodes/config/bmodes_paper.md | 2 +- papers/bmodes/config/cl.md | 6 +- papers/bmodes/config/config.yaml | 11 +- .../config/config_space_pte_matrices.md | 4 +- papers/bmodes/config/cosebis.md | 4 +- papers/bmodes/config/cosebis_data_vector.md | 4 +- papers/bmodes/config/covariance.md | 18 +-- papers/bmodes/config/ecut_spec.md | 3 +- .../config/harmonic_space_pte_matrices.md | 4 +- papers/bmodes/config/pure_eb.md | 4 +- papers/bmodes/config/pure_eb_covariance.md | 1 - papers/bmodes/config/pure_eb_data_vector.md | 2 +- papers/bmodes/rules/claims.smk | 123 ++++++++---------- papers/bmodes/rules/presentation.smk | 6 +- .../bb_covariance_blind_independence.py | 48 +++---- .../bmodes/scripts/calculate_pure_eb_ptes.py | 9 +- papers/bmodes/scripts/cl_data_vector.py | 15 +-- .../bmodes/scripts/cl_version_comparison.py | 15 +-- .../scripts/compute_cosebis_pte_single.py | 18 +-- .../scripts/config_space_pte_matrices.py | 48 +++---- .../scripts/cosebis_version_comparison.py | 7 +- .../bmodes/scripts/gather_pure_eb_chunks.py | 13 +- .../harmonic_config_cosebis_comparison.py | 14 +- .../scripts/harmonic_space_pte_matrices.py | 31 ++--- .../scripts/precompute_pure_eb_chunk.py | 7 +- papers/bmodes/scripts/pure_eb_covariance.py | 3 +- papers/bmodes/scripts/pure_eb_data_vector.py | 10 +- .../scripts/pure_eb_version_comparison.py | 10 +- papers/bmodes/scripts/run_cl_sweep.py | 21 ++- .../bmodes/scripts/run_cosebis_ptes_sweep.py | 15 +-- .../bmodes/scripts/run_pure_eb_ptes_sweep.sh | 13 +- .../scripts/run_pure_eb_semianalytic.sh | 13 +- papers/bmodes/scripts/run_pure_eb_sweep.sh | 17 ++- papers/cosmo_val/config/config.yaml | 2 - src/sp_validation/cosmo_val/core.py | 17 +-- .../tests/test_cv_init_params.py | 4 +- src/sp_validation/tests/test_pseudo_cl.py | 2 +- workflow/common.py | 72 +++------- workflow/rules/cosmo_val.smk | 1 - workflow/rules/covariance.smk | 51 ++++---- workflow/rules/glass_mock.smk | 6 +- workflow/rules/inference.smk | 34 ++--- workflow/rules/twopoint.smk | 18 +-- workflow/scripts/generate_pseudo_cl.py | 12 +- workflow/scripts/generate_pseudo_cl_cov.py | 12 +- workflow/tests/conftest.py | 1 - 48 files changed, 317 insertions(+), 489 deletions(-) diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index 218ba413..edbc4e85 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -539,19 +539,19 @@ SP_v1.4.6.3_B: getdist_colour: 0.0, 0.5, 1.0 ls: dashed marker: d + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits cov_th: - A: 2405.3892055695346 - n_e: 6.128201234871523 + A: 2894.0303815287743 + n_e: 4.957279270321334 n_psf: 0.752316232272063 - sigma_e: 0.379587601488189 - mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + sigma_e: 0.3783979556382064 psf: PSF_flag: HSM_FLAG_PSF PSF_size: HSM_T_PSF star_flag: HSM_FLAG_STAR star_size: HSM_T_STAR hdu: 1 - path: unions_shapepipe_psf_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_PSF_col: HSM_G1_PSF @@ -560,7 +560,7 @@ SP_v1.4.6.3_B: e2_star_col: HSM_G2_STAR shear: R: 1.0 - path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + path: /n17data/UNIONS/WL/v1.4.x/v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6.3/nz_SP_v1.4.6.3_B.txt w_col: w_des e1_col: e1 @@ -574,7 +574,7 @@ SP_v1.4.6.3_B: dec_col: Dec e1_col: e1 e2_col: e2 - path: unions_shapepipe_star_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.3_C: subdir: /n17data/UNIONS/WL/v1.4.x @@ -583,19 +583,19 @@ SP_v1.4.6.3_C: getdist_colour: 0.0, 0.5, 1.0 ls: dashed marker: d + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits cov_th: - A: 2405.3892055695346 - n_e: 6.128201234871523 + A: 2894.0303815287743 + n_e: 4.957279270321334 n_psf: 0.752316232272063 - sigma_e: 0.379587601488189 - mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + sigma_e: 0.3783979556382064 psf: PSF_flag: HSM_FLAG_PSF PSF_size: HSM_T_PSF star_flag: HSM_FLAG_STAR star_size: HSM_T_STAR hdu: 1 - path: unions_shapepipe_psf_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_PSF_col: HSM_G1_PSF @@ -604,7 +604,7 @@ SP_v1.4.6.3_C: e2_star_col: HSM_G2_STAR shear: R: 1.0 - path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + path: /n17data/UNIONS/WL/v1.4.x/v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6.3/nz_SP_v1.4.6.3_C.txt w_col: w_des e1_col: e1 @@ -618,7 +618,7 @@ SP_v1.4.6.3_C: dec_col: Dec e1_col: e1 e2_col: e2 - path: unions_shapepipe_star_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.3_A: subdir: /n17data/UNIONS/WL/v1.4.x @@ -627,19 +627,19 @@ SP_v1.4.6.3_A: getdist_colour: 0.0, 0.5, 1.0 ls: dashed marker: d + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits cov_th: - A: 2405.3892055695346 - n_e: 6.128201234871523 + A: 2894.0303815287743 + n_e: 4.957279270321334 n_psf: 0.752316232272063 - sigma_e: 0.379587601488189 - mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + sigma_e: 0.3783979556382064 psf: PSF_flag: HSM_FLAG_PSF PSF_size: HSM_T_PSF star_flag: HSM_FLAG_STAR star_size: HSM_T_STAR hdu: 1 - path: unions_shapepipe_psf_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_PSF_col: HSM_G1_PSF @@ -648,7 +648,7 @@ SP_v1.4.6.3_A: e2_star_col: HSM_G2_STAR shear: R: 1.0 - path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + path: /n17data/UNIONS/WL/v1.4.x/v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6.3/nz_SP_v1.4.6.3_A.txt w_col: w_des e1_col: e1 @@ -662,7 +662,7 @@ SP_v1.4.6.3_A: dec_col: Dec e1_col: e1 e2_col: e2 - path: unions_shapepipe_star_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6_ecut07: subdir: /n17data/UNIONS/WL/v1.4.x @@ -827,7 +827,7 @@ SP_v1.4.8: shear: R: 1.0 path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits - redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.8/nz_SP_v1.4.8_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected @@ -915,7 +915,7 @@ SP_v1.4.11.3: shear: R: 1.0 path: v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits - redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected @@ -1049,7 +1049,7 @@ SP_v1.4.11.3_ecut07: shear: R: 1.0 path: /n17data/cdaley/unions/pure_eb/results/ecut/SP_v1.4.11.3_ecut07.fits - redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected diff --git a/papers/bmodes/config/bb_covariance_blind_independence.md b/papers/bmodes/config/bb_covariance_blind_independence.md index 33ec2cb9..0af8f908 100644 --- a/papers/bmodes/config/bb_covariance_blind_independence.md +++ b/papers/bmodes/config/bb_covariance_blind_independence.md @@ -4,11 +4,13 @@ Method: [Covariance](covariance.md), [Pure E/B](pure_eb.md), [COSEBIS](cosebis.m ## Claim -BB covariances computed via **analytic propagation** are blind-independent, while BB covariances computed via **MC sampling** inherit noise that varies across blinds. +BB covariances computed via **analytic propagation** are independent of the A/B/C n(z) realisation, while BB covariances computed via **MC sampling** inherit noise that varies across realisations. + +Each realisation is its own catalogue entry, identical to the mock version's but for its n(z): `fiducial.nz_realisations` maps the labels A/B/C to those entries (`SP_v1.4.6.3_{A,B,C}_leak_corr`). ## Evidence -Diagonal ratios relative to blind A for both B-modes and E-modes: +Diagonal ratios relative to realisation A for both B-modes and E-modes: - Pure E/B: `cov(ξ+^B)`, `cov(ξ-^B)` vs `cov(ξ+^E)`, `cov(ξ-^E)` - COSEBIS: `cov(B_n)` vs `cov(E_n)` - Harmonic: `cov(C_ℓ^BB)` vs `cov(C_ℓ^EE)` @@ -22,6 +24,7 @@ Diagonal ratios relative to blind A for both B-modes and E-modes: | Parameter | Config Key | |-----------|------------| | Version | `fiducial.version` | +| n(z) realisations | `fiducial.nz_realisations` | | Scale range | `fiducial.min_sep` to `fiducial.max_sep` | | Bins | `fiducial.nbins` | | COSEBIS nmodes | `fiducial.nmodes` | diff --git a/papers/bmodes/config/bmodes_paper.md b/papers/bmodes/config/bmodes_paper.md index 5623fe8a..9c87d375 100644 --- a/papers/bmodes/config/bmodes_paper.md +++ b/papers/bmodes/config/bmodes_paper.md @@ -86,7 +86,7 @@ Note: Per-version PTEs come from `config_space_pte_matrices`, not `pure_eb_data_ ### Blind Handling -PTEs report the **minimum across blinds** (`pte_joint_min`) as the conservative estimate. This ensures reported values remain valid regardless of which blind is eventually unblinded. The fiducial blind `config["fiducial"]["blind"]` determines covariance matrix selection. +PTEs report the **minimum across blinds** (`pte_joint_min`) as the conservative estimate. This ensures reported values remain valid regardless of which blind is eventually unblinded. Covariances use each version's catalogue-entry n(z). ## Macros and Tables diff --git a/papers/bmodes/config/cl.md b/papers/bmodes/config/cl.md index ad0b7d70..5b30d26c 100644 --- a/papers/bmodes/config/cl.md +++ b/papers/bmodes/config/cl.md @@ -25,10 +25,10 @@ Pseudo-Cl estimation accounts for: ## Data Source Pseudo-Cl files generated by workflow rules using NaMaster: -- `pseudo_cl_{version}_blind={blind}_powspace_nbins={nbins}.sacc` — Power spectrum estimates -- `pseudo_cl_cov_{version}_blind={blind}_powspace_nbins={nbins}.fits` — Bandpower covariance matrix +- `pseudo_cl_{version}_powspace_nbins={nbins}.sacc` — Power spectrum estimates +- `pseudo_cl_cov_{version}_powspace_nbins={nbins}.fits` — Bandpower covariance matrix Location: `{COSMO_VAL_OUTPUT}/` (defined in Snakefile, typically `/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/output/`) -Default parameters: `blind=A` (from `fiducial.blind`), `nbins=32` (from `cl.n_ell_bins`). +Default parameters: `nbins=32` (from `cl.n_ell_bins`). diff --git a/papers/bmodes/config/config.yaml b/papers/bmodes/config/config.yaml index 47b385bd..3c8357d7 100644 --- a/papers/bmodes/config/config.yaml +++ b/papers/bmodes/config/config.yaml @@ -23,7 +23,6 @@ nbins_int: [100, 200, 300, 500] # fiducial parameters fiducial: version: SP_v1.4.6.3_leak_corr - blind: A npatch: 1 min_sep: 1.0 max_sep: 250.0 @@ -31,6 +30,13 @@ fiducial: mock_version: SP_v1.4.6.3 + # The mock version's catalogue under each n(z) realisation: entries identical + # but for shear.redshift_path, compared by bb_covariance_blind_independence. + nz_realisations: + A: SP_v1.4.6.3_A_leak_corr + B: SP_v1.4.6.3_B_leak_corr + C: SP_v1.4.6.3_C_leak_corr + min_sep_int: 0.5 max_sep_int: 300.0 nbins_int: 1000 @@ -79,10 +85,9 @@ cl: # Harmonic pseudo-Cl fiducial binning — the tag the producer (twopoint.smk # pseudo_cl / pseudo_cl_cov) stamps into the filename, which the inference -# consumer reconstructs to request the exact file. Canonical: A / powspace / 32. +# consumer reconstructs to request the exact file. Canonical: powspace / 32. harmonic: fiducial: - blind: A binning: powspace nbins: 32 diff --git a/papers/bmodes/config/config_space_pte_matrices.md b/papers/bmodes/config/config_space_pte_matrices.md index 40073add..a7ad89a0 100644 --- a/papers/bmodes/config/config_space_pte_matrices.md +++ b/papers/bmodes/config/config_space_pte_matrices.md @@ -10,9 +10,9 @@ Fiducial angular scale cuts are justified by PTE heatmaps across all (theta_min, Scale cuts from `fiducial.fiducial_xip_scale_cut` and `fiducial.fiducial_xim_scale_cut`. COSEBIS uses the same unified range. -## Blind Handling +## n(z) -Uses fiducial blind from `config["fiducial"]["blind"]`. Data vectors (ξ+^B, ξ-^B, COSEBIS B_n) are identical across blinds; covariances vary with blind via n(z)-dependent theoretical predictions. +Each version's covariance uses its catalogue entry's n(z) (`shear.redshift_path` in `cat_config.yaml`). ## Config References diff --git a/papers/bmodes/config/cosebis.md b/papers/bmodes/config/cosebis.md index 25756887..5242a08d 100644 --- a/papers/bmodes/config/cosebis.md +++ b/papers/bmodes/config/cosebis.md @@ -33,9 +33,9 @@ Compare across catalog versions from `config.versions`: - Fiducial: `fiducial.version` - All versions tested for consistency -## Blind Handling +## n(z) -Uses fiducial blind from `config["fiducial"]["blind"]`. COSEBIS B_n data vectors are identical across blinds; covariances vary with blind via n(z)-dependent theoretical predictions. +Each version's covariance uses its catalogue entry's n(z) (`shear.redshift_path` in `cat_config.yaml`). ## Analysis Decisions diff --git a/papers/bmodes/config/cosebis_data_vector.md b/papers/bmodes/config/cosebis_data_vector.md index 064ae63a..30e85731 100644 --- a/papers/bmodes/config/cosebis_data_vector.md +++ b/papers/bmodes/config/cosebis_data_vector.md @@ -8,9 +8,9 @@ Plotting: [1D Plots](1d_plots.md) COSEBIS B-modes at fiducial version (`fiducial.version`) are consistent with zero across the full angular range and at fiducial scale cuts. -## Blind Handling +## n(z) -Uses fiducial blind from `config["fiducial"]["blind"]`. COSEBIS B_n data vectors are identical across blinds; covariances vary with blind via n(z)-dependent theoretical predictions. Statistical evidence (PTEs) is in [Config-Space PTE Matrices](config_space_pte_matrices.md). +Each version's covariance uses its catalogue entry's n(z) (`shear.redshift_path` in `cat_config.yaml`). Statistical evidence (PTEs) is in [Config-Space PTE Matrices](config_space_pte_matrices.md). ## Config References diff --git a/papers/bmodes/config/covariance.md b/papers/bmodes/config/covariance.md index 54e7ad42..2959e433 100644 --- a/papers/bmodes/config/covariance.md +++ b/papers/bmodes/config/covariance.md @@ -40,17 +40,18 @@ Extracted from catalog config (`cat_config.yaml`) per version: ## File Naming ``` -covariance_{version}_{blind}_{gaussian}_minsep={min}_maxsep={max}_nbins={n}{mask_suffix}_processed.txt +covariance_{version}_{gaussian}_minsep={min}_maxsep={max}_nbins={n}{mask_suffix}_processed.txt ``` - `version`: SP_v1.4.6_leak_corr, etc. -- `blind`: A, B, or C - `gaussian`: g (Gaussian-only) or ng (non-Gaussian) - `mask_suffix`: empty or `_masked` -## Blind Handling +## n(z) -B-mode claims use the fiducial blind from `config["fiducial"]["blind"]`. Covariances are computed for the fiducial blind only. +A version's covariance uses its catalogue entry's n(z) (`shear.redshift_path` in +`cat_config.yaml`). Choosing another n(z) realisation means choosing another +catalogue entry (e.g. `SP_v1.4.6.3_B`). ## Covariance Usage Policy @@ -58,7 +59,7 @@ Official results use specific covariance sources for consistency and correctness | Quantity | Source | Gaussian | Binning | Notes | |----------|--------|----------|---------|-------| -| Total ξ± errors | CosmoCov | ng | 20-bin (reporting) | Per-blind, masked | +| Total ξ± errors | CosmoCov | ng | 20-bin (reporting) | Masked | | Pure E/B mode errors | MC propagation | g | 1000→20 bin | Conservative: underestimates uncertainty | | COSEBIS errors | MC propagation | g | 1000→scales | Per scale cut | @@ -76,10 +77,9 @@ only spatially-structured cuts (no galaxy selection cuts): | Standard footprint | 2894 deg² | v1.4.5, v1.4.6, v1.4.11.3 (and ecut variants) | | Star-halo footprint | 2517 deg² | v1.4.8 | -Each version gets its own covariance from its own survey properties (A, n_e, sigma_e). -`resolve_covariance_version()` is the identity function — no cross-version covariance sharing. -`MASK_CLS_FILES` (covariance.smk) maps to two mask power spectrum files based on whether -the version is in `STARHALO_VERSIONS`. +Each version gets its own covariance from its own catalogue entry's survey properties +(`cov_th`: A, n_e, sigma_e). `MASK_CLS_FILES` (covariance.smk) maps to two mask power +spectrum files based on whether the version's base catalogue is in `STARHALO_CATALOGUES`. ## Related Specs diff --git a/papers/bmodes/config/ecut_spec.md b/papers/bmodes/config/ecut_spec.md index 53041659..88634234 100644 --- a/papers/bmodes/config/ecut_spec.md +++ b/papers/bmodes/config/ecut_spec.md @@ -39,8 +39,7 @@ The 2PCF and covariance are independent and can run in parallel. Key resolution functions in `workflow/Snakefile`: - `get_shear_catalog()` — resolves catalog path from cat_config, strips `_leak_corr` -- `build_redshift_path()` — v1.4.11.x uses v1.4.6's n(z) -- `resolve_covariance_version()` — identity function (each version gets its own covariance) +- `redshift_path()` — the catalogue entry's `shear.redshift_path` (v1.4.11.3's entries point at v1.4.6's n(z)) - Wildcard constraint: `version=r"SP_v[\d.]+(_w_iv)?(_leak_corr)?"` — needs `_ecut\d+` Version comparison rules in `papers/bmodes/rules/claims.smk` (lines 131, 271, 386) use diff --git a/papers/bmodes/config/harmonic_space_pte_matrices.md b/papers/bmodes/config/harmonic_space_pte_matrices.md index 0f17b6fc..e98c4031 100644 --- a/papers/bmodes/config/harmonic_space_pte_matrices.md +++ b/papers/bmodes/config/harmonic_space_pte_matrices.md @@ -10,9 +10,9 @@ Harmonic-space B-mode PTEs are consistent with noise at fiducial multipole range Fiducial scale cuts from `cl.fiducial_ell_min` and `cl.fiducial_ell_max`. Full B-mode test range spans all multipole bins present in the input pseudo-Cℓ file. -## Blind Handling +## n(z) -Uses fiducial blind from `config["fiducial"]["blind"]`. The C_ℓ^BB data vector is identical across blinds; covariances vary with blind via n(z)-dependent theoretical predictions. +Each version's covariance uses its catalogue entry's n(z) (`shear.redshift_path` in `cat_config.yaml`). ## Config References diff --git a/papers/bmodes/config/pure_eb.md b/papers/bmodes/config/pure_eb.md index 6800b149..76f9c439 100644 --- a/papers/bmodes/config/pure_eb.md +++ b/papers/bmodes/config/pure_eb.md @@ -28,9 +28,9 @@ Uses semi-analytical covariance propagation through the decomposition. ## Data Products Precomputed decomposition stored in: -`results/paper_plots/intermediate/{version}_{blind}_pure_eb_semianalytic.npz` +`results/paper_plots/intermediate/{version}_pure_eb_semianalytic.npz` -Uses fiducial blind (A) from config. Each NPZ contains: +Each NPZ contains: - `theta`: Angular bins - `xip_E`, `xim_E`: Pure E-mode components - `xip_B`, `xim_B`: Pure B-mode components diff --git a/papers/bmodes/config/pure_eb_covariance.md b/papers/bmodes/config/pure_eb_covariance.md index 05453d23..bca1ed01 100644 --- a/papers/bmodes/config/pure_eb_covariance.md +++ b/papers/bmodes/config/pure_eb_covariance.md @@ -28,7 +28,6 @@ Block-wise condition numbers for the 120×120 pure E/B covariance (6 blocks of 2 | Parameter | Config Key | |-----------|------------| | Version | `fiducial.version` | -| Blind | `fiducial.blind` | | Integration bins | `fiducial.nbins_int` | | Reporting bins | `fiducial.nbins` | diff --git a/papers/bmodes/config/pure_eb_data_vector.md b/papers/bmodes/config/pure_eb_data_vector.md index 6e334d9f..a7018b96 100644 --- a/papers/bmodes/config/pure_eb_data_vector.md +++ b/papers/bmodes/config/pure_eb_data_vector.md @@ -20,7 +20,7 @@ B-mode signals in UNIONS cosmic shear are consistent with zero at fiducial scale ## Evidence -PTE values for B-mode null tests at two scale ranges, using fiducial blind (A): +PTE values for B-mode null tests at two scale ranges: 1. **Fiducial scale cuts**: Angular range used for cosmological inference 2. **Full theta range**: All measured angular bins (no cuts) diff --git a/papers/bmodes/rules/claims.smk b/papers/bmodes/rules/claims.smk index 85891b25..951a6c80 100644 --- a/papers/bmodes/rules/claims.smk +++ b/papers/bmodes/rules/claims.smk @@ -8,7 +8,7 @@ Claims depend on methods (for technique definitions) and compute outputs (for da # Configuration # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# CONFIG_DIR, TAPESTRY_DIR, PAPER_FIGURES_DIR, BLINDS, FIDUCIAL, PLANCK18 defined in Snakefile +# CONFIG_DIR, TAPESTRY_DIR, PAPER_FIGURES_DIR, FIDUCIAL, PLANCK18 defined in Snakefile # COSMO_VAL, COSMO_INFERENCE, covariance_path() defined in Snakefile COSMO_VAL_OUTPUT = str(COSMO_VAL) # String version for f-string interpolation @@ -31,6 +31,10 @@ VERSION_LABELS = config["plotting"].get("version_labels", {}) FIDUCIAL_VERSION = FIDUCIAL["version"] MOCK_VERSION = f"{FIDUCIAL['mock_version']}_leak_corr" +# Catalogues identical to the mock version but for their n(z) realisation, +# keyed by realisation label: bb_covariance_blind_independence compares them. +NZ_REALISATIONS = FIDUCIAL["nz_realisations"] + # Filter versions for different analysis types # Pure E/B and PTEs only apply to leak-corrected versions VERSIONS_LEAK_CORR = [v for v in config["versions"] if "_leak_corr" in v and "_ecut" not in v] @@ -72,9 +76,9 @@ def _per_version_figure_outputs(claim_dir): # Path Helper Functions # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -def _reporting_cov_path(version, blind): +def _reporting_cov_path(version): """Path to reporting-scale covariance (non-Gaussian, masked).""" - return covariance_path(version, blind, gaussian="ng") + return covariance_path(version, gaussian="ng") def _xi_reporting_path(version): @@ -93,10 +97,10 @@ def _xi_integration_path(version): ) -def _cov_integration_path(version, blind): +def _cov_integration_path(version): """Covariance path for integration bins (Gaussian, for COSEBIS PTE).""" return covariance_path( - version, blind, gaussian="g", + version, gaussian="g", min_sep=FIDUCIAL["min_sep_int"], max_sep=FIDUCIAL["max_sep_int"], nbins=FIDUCIAL["nbins_int"] ) @@ -116,17 +120,14 @@ def _pte_scale_cut_pairs(): PTE_SCALE_CUT_PAIRS = _pte_scale_cut_pairs() -def _pseudo_cl_path(version, blind="A", nbins=32): - """Return pseudo-Cl path for a catalog version. - - All leak-corrected versions use consistent local naming with blind and binning. - """ - return f"{COSMO_VAL_OUTPUT}/pseudo_cl_{version}_blind={blind}_powspace_nbins={nbins}.sacc" +def _pseudo_cl_path(version, nbins=32): + """Return pseudo-Cl path for a catalog version.""" + return f"{COSMO_VAL_OUTPUT}/pseudo_cl_{version}_powspace_nbins={nbins}.sacc" -def _pseudo_cl_cov_path(version, blind="A", nbins=32): +def _pseudo_cl_cov_path(version, nbins=32): """Return pseudo-Cl covariance path for a catalog version.""" - return f"{COSMO_VAL_OUTPUT}/pseudo_cl_cov_{version}_blind={blind}_powspace_nbins={nbins}.fits" + return f"{COSMO_VAL_OUTPUT}/pseudo_cl_cov_{version}_powspace_nbins={nbins}.fits" # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ @@ -147,7 +148,7 @@ rule cosebis_version_comparison: config=f"{CONFIG_DIR}/config.yaml", # COSEBIs only for leak-corrected versions xi_integration=[_xi_integration_path(ver) for ver in VERSIONS_LEAK_CORR], - cov_integration=[_cov_integration_path(ver, "A") for ver in VERSIONS_LEAK_CORR], + cov_integration=[_cov_integration_path(ver) for ver in VERSIONS_LEAK_CORR], params: version_labels=VERSION_LABELS, versions=VERSIONS_LEAK_CORR, @@ -179,7 +180,7 @@ rule cosebis_data_vector: config=f"{CONFIG_DIR}/config.yaml", # Per-version inputs: xi_{version} and cov_{version} for all versions **{f"xi_{ver}": _xi_integration_path(ver) for ver in VERSIONS_ALL_FOR_PLOTS}, - **{f"cov_{ver}": _cov_integration_path(ver, "A") for ver in VERSIONS_ALL_FOR_PLOTS}, + **{f"cov_{ver}": _cov_integration_path(ver) for ver in VERSIONS_ALL_FOR_PLOTS}, params: cov_base_dir=str(COSMO_INFERENCE / "data/covariance"), output: @@ -204,7 +205,7 @@ rule cosebis_binning_comparison: f"{COSMO_VAL_OUTPUT}/{FIDUCIAL_VERSION}_xi_minsep={FIDUCIAL['min_sep_int']}" f"_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch={FIDUCIAL['npatch']}.txt" ), - cov_1k=_cov_integration_path(FIDUCIAL_VERSION, "A"), + cov_1k=_cov_integration_path(FIDUCIAL_VERSION), output: evidence=f"{TAPESTRY_DIR}/cosebis_binning_comparison/evidence.json", figure=f"{TAPESTRY_DIR}/cosebis_binning_comparison/figure.png", @@ -225,14 +226,13 @@ N_PURE_EB_CHUNKS = config["pure_eb"]["n_chunks"] rule precompute_pure_eb_chunk: """Compute a chunk of MC samples for pure E/B covariance (scatter).""" input: - cov_integration=lambda w: _cov_integration_path(w.version, w.blind), + cov_integration=lambda w: _cov_integration_path(w.version), xi_reporting=lambda w: _xi_reporting_path(w.version), xi_integration=lambda w: _xi_integration_path(w.version), output: - "results/paper_plots/intermediate/chunks/{version}_{blind}_pure_eb_chunk_{chunk_id}.npz", + "results/paper_plots/intermediate/chunks/{version}_pure_eb_chunk_{chunk_id}.npz", params: version="{version}", - blind="{blind}", chunk_id="{chunk_id}", n_chunks=N_PURE_EB_CHUNKS, n_samples=config["covariance"]["n_samples"], @@ -246,18 +246,15 @@ rule precompute_pure_eb_chunk: rule precompute_pure_eb: """Gather MC sample chunks and compute final pure E/B covariance.""" - wildcard_constraints: - version=r"[^_]+_v[\d.]+(_ecut\d+)?(_leak_corr)?", # e.g. SP_v1.4.6, SP_v1.4.6_ecut07_leak_corr - blind=r"[ABC]", input: chunks=expand( - "results/paper_plots/intermediate/chunks/{{version}}_{{blind}}_pure_eb_chunk_{chunk_id}.npz", + "results/paper_plots/intermediate/chunks/{{version}}_pure_eb_chunk_{chunk_id}.npz", chunk_id=range(N_PURE_EB_CHUNKS), ), xi_reporting=lambda w: _xi_reporting_path(w.version), xi_integration=lambda w: _xi_integration_path(w.version), output: - "results/paper_plots/intermediate/{version}_{blind}_pure_eb_semianalytic.npz", + "results/paper_plots/intermediate/{version}_pure_eb_semianalytic.npz", params: version="{version}", **FIDUCIAL_BINNING, @@ -271,8 +268,6 @@ rule precompute_pure_eb: rule pure_eb_data_vector: """B-mode null test: Pure E/B data vector at fiducial scale cuts. - Uses fiducial blind only (FIDUCIAL["blind"]) for PTE calculation. - Produces 9 figures: - figure.png: fiducial version, leak-corrected, no title (paper) - figure_v{X.Y.Z}.png: each version, leak-corrected, with title @@ -286,10 +281,9 @@ rule pure_eb_data_vector: ], config=f"{CONFIG_DIR}/config.yaml", # Per-version inputs: pure_eb_{version} and cov_{version} for all versions - **{f"pure_eb_{ver}": f"results/paper_plots/intermediate/{ver}_{FIDUCIAL['blind']}_pure_eb_semianalytic.npz" - for ver in VERSIONS_ALL_FOR_PLOTS}, - **{f"cov_{ver}": _reporting_cov_path(ver, FIDUCIAL["blind"]) + **{f"pure_eb_{ver}": f"results/paper_plots/intermediate/{ver}_pure_eb_semianalytic.npz" for ver in VERSIONS_ALL_FOR_PLOTS}, + **{f"cov_{ver}": _reporting_cov_path(ver) for ver in VERSIONS_ALL_FOR_PLOTS}, output: evidence=f"{TAPESTRY_DIR}/pure_eb_data_vector/evidence.json", paper_figure=f"{PAPER_FIGURES_DIR}/pure_eb_data_vector.pdf", @@ -313,7 +307,7 @@ rule pure_eb_version_comparison: config=f"{CONFIG_DIR}/config.yaml", # Pure E/B only for leak-corrected versions pure_eb_data=[ - f"results/paper_plots/intermediate/{ver}_A_pure_eb_semianalytic.npz" + f"results/paper_plots/intermediate/{ver}_pure_eb_semianalytic.npz" for ver in VERSIONS_LEAK_CORR ], params: @@ -334,8 +328,6 @@ rule pure_eb_covariance: - E and B blocks are well-conditioned (~10^5) - Ambiguous blocks are ill-conditioned (~10^15, expected) - Correlation structure across 6 blocks (E+/E-/B+/B-/amb+/amb-) - - Uses blind A covariance for visualization (structure is similar across blinds). """ input: specs=[ @@ -345,7 +337,7 @@ rule pure_eb_covariance: f"{CONFIG_DIR}/2d_plots.md", ], config=f"{CONFIG_DIR}/config.yaml", - pure_eb_data=f"results/paper_plots/intermediate/{FIDUCIAL_VERSION}_A_pure_eb_semianalytic.npz", + pure_eb_data=f"results/paper_plots/intermediate/{FIDUCIAL_VERSION}_pure_eb_semianalytic.npz", output: evidence=f"{TAPESTRY_DIR}/pure_eb_covariance/evidence.json", figure=f"{TAPESTRY_DIR}/pure_eb_covariance/figure.png", @@ -357,17 +349,12 @@ rule pure_eb_covariance: rule calculate_pure_eb_ptes: """PTE matrices for pure E/B-mode scale-cut robustness. - Nothing here varies with the blind: the data vectors come from the blind-A - gather and the PTEs are Hartlap-debiased by the MC draw count, not by a - per-blind covariance. The wildcard survives as the filename slot the - consumer (config_space_pte_matrices) reads, and only blind A is ever built. + The PTEs are Hartlap-debiased by the MC draw count. """ input: - pure_eb_data="results/paper_plots/intermediate/{version}_A_pure_eb_semianalytic.npz", + pure_eb_data="results/paper_plots/intermediate/{version}_pure_eb_semianalytic.npz", output: - "results/paper_plots/intermediate/{version}_{blind}_pure_eb_ptes.npz", - wildcard_constraints: - blind=r"[ABC]", + "results/paper_plots/intermediate/{version}_pure_eb_ptes.npz", params: version="{version}", n_samples=config["covariance"]["n_samples"], @@ -441,18 +428,17 @@ rule cl_version_comparison: # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ rule compute_cosebis_pte: - """Scatter: Compute COSEBIS B-mode PTE for a single (version, blind, i_min, i_max) tuple.""" + """Scatter: Compute COSEBIS B-mode PTE for a single (version, i_min, i_max) tuple.""" input: xi_integration=lambda w: _xi_integration_path(w.version), - cov_integration=lambda w: _cov_integration_path(w.version, w.blind), + cov_integration=lambda w: _cov_integration_path(w.version), output: - pte_json=f"{TAPESTRY_DIR}/cosebis_pte_matrix/pte_values/{{version}}/{{blind}}/pte_{{i_min}}_{{i_max}}.json", + pte_json=f"{TAPESTRY_DIR}/cosebis_pte_matrix/pte_values/{{version}}/pte_{{i_min}}_{{i_max}}.json", params: nmodes=FIDUCIAL["nmodes"], wildcard_constraints: i_min=r"\d{3}", i_max=r"\d{3}", - blind=r"[ABC]", threads: 1 resources: mem_mb=8000, @@ -482,14 +468,13 @@ rule config_space_pte_matrices: # Claim dependencies pure_eb_data_vector=f"{TAPESTRY_DIR}/pure_eb_data_vector/evidence.json", cosebis_data_vector=f"{TAPESTRY_DIR}/cosebis_data_vector/evidence.json", - # Data inputs (fiducial blind only) # Pure E/B and COSEBIs PTEs for both corrected and uncorrected versions pure_eb_pte=[ - f"results/paper_plots/intermediate/{ver}_{FIDUCIAL['blind']}_pure_eb_ptes.npz" + f"results/paper_plots/intermediate/{ver}_pure_eb_ptes.npz" for ver in VERSIONS_CONFIG_SPACE_PTES ], cosebis_pte_files=[ - f"{TAPESTRY_DIR}/cosebis_pte_matrix/pte_values/{ver}/{FIDUCIAL['blind']}/pte_{i:03d}_{j:03d}.json" + f"{TAPESTRY_DIR}/cosebis_pte_matrix/pte_values/{ver}/pte_{i:03d}_{j:03d}.json" for ver in VERSIONS_CONFIG_SPACE_PTES for i, j in PTE_SCALE_CUT_PAIRS ], @@ -509,7 +494,8 @@ rule harmonic_space_pte_matrices: Results: Single-panel Cl^BB PTE matrix for fiducial version Appendix: N-panel composite for all versions from config.versions - Uses fiducial blind covariance (blind independence validated in bb_covariance_blind_independence). + n(z)-realisation independence of the BB covariance is validated in + bb_covariance_blind_independence. """ input: specs=[ @@ -520,10 +506,7 @@ rule harmonic_space_pte_matrices: config=f"{CONFIG_DIR}/config.yaml", # Harmonic PTE matrices for both corrected and uncorrected versions pseudo_cl=[_pseudo_cl_path(ver) for ver in VERSIONS_CONFIG_SPACE_PTES], - pseudo_cl_cov=[ - _pseudo_cl_cov_path(ver, blind=FIDUCIAL["blind"]) - for ver in VERSIONS_CONFIG_SPACE_PTES - ], + pseudo_cl_cov=[_pseudo_cl_cov_path(ver) for ver in VERSIONS_CONFIG_SPACE_PTES], params: version_labels=VERSION_LABELS, output: @@ -537,16 +520,17 @@ rule harmonic_space_pte_matrices: rule bb_covariance_blind_independence: - """Test BB covariance blind-independence vs EE variation. + """Test BB covariance independence of the n(z) realisation vs EE variation. - BB covariances should be stable across blinds (null signal → no sample variance). - EE covariances should vary (~10%) due to sample variance from cosmological signal. + BB covariances should be stable across the A/B/C n(z) realisations (null + signal → no sample variance). EE covariances should vary (~10%) due to + sample variance from cosmological signal. Covers all three analysis spaces: Pure E/B, COSEBIS, and harmonic (pseudo-Cl). - Uses mock_version (v1.4.6) for per-blind covariances — blind independence is a - property of survey geometry, not catalog version. Avoids generating B/C covariances - for the fiducial version. + Compares NZ_REALISATIONS, catalogues identical to the mock version but for + their n(z): the independence is a property of survey geometry, not catalog + version. """ input: specs=[ @@ -557,16 +541,17 @@ rule bb_covariance_blind_independence: f"{CONFIG_DIR}/cl.md", ], config=f"{CONFIG_DIR}/config.yaml", - # Per-blind MC-propagated pure E/B covariances (using mock_version for all blinds) - **{f"pure_eb_{b}": f"results/paper_plots/intermediate/{MOCK_VERSION}_{b}_pure_eb_semianalytic.npz" - for b in BLINDS}, - # COSEBIS: xi integration file (shared) + per-blind config-space covariances + # Per-realisation MC-propagated pure E/B covariances + **{f"pure_eb_{label}": f"results/paper_plots/intermediate/{ver}_pure_eb_semianalytic.npz" + for label, ver in NZ_REALISATIONS.items()}, + # COSEBIS: xi integration file (shared) + per-realisation config-space covariances xi_integration=_xi_integration_path(MOCK_VERSION), - **{f"cov_integration_{b}": _cov_integration_path(MOCK_VERSION, b) for b in BLINDS}, - # Per-blind harmonic covariances - **{f"harmonic_{b}": _pseudo_cl_cov_path(MOCK_VERSION, b) for b in BLINDS}, - # Pseudo-Cl data vector (blind A only) for ell bin centers - pseudo_cl=_pseudo_cl_path(MOCK_VERSION, "A"), + **{f"cov_integration_{label}": _cov_integration_path(ver) + for label, ver in NZ_REALISATIONS.items()}, + # Per-realisation harmonic covariances + **{f"harmonic_{label}": _pseudo_cl_cov_path(ver) for label, ver in NZ_REALISATIONS.items()}, + # Pseudo-Cl data vector for ell bin centers + pseudo_cl=_pseudo_cl_path(MOCK_VERSION), params: nmodes=FIDUCIAL["nmodes"], theta_min=config["cosebis"]["theta_min"], @@ -612,7 +597,7 @@ rule harmonic_config_cosebis_comparison: **{f"pseudo_cl_{ver}": _pseudo_cl_path(ver, nbins=_COSEBIS_NBINS) for ver in VERSIONS_ALL_FOR_PLOTS}, **{f"pseudo_cl_cov_{ver}": _pseudo_cl_cov_path(ver, nbins=_COSEBIS_NBINS) for ver in VERSIONS_ALL_FOR_PLOTS}, **{f"xi_{ver}": _xi_integration_path(ver) for ver in VERSIONS_ALL_FOR_PLOTS}, - **{f"cov_{ver}": _cov_integration_path(ver, FIDUCIAL["blind"]) for ver in VERSIONS_ALL_FOR_PLOTS}, + **{f"cov_{ver}": _cov_integration_path(ver) for ver in VERSIONS_ALL_FOR_PLOTS}, params: scale_cut=lambda wildcards: _COSEBIS_ANGULAR_RANGES[wildcards.angular_range], output: diff --git a/papers/bmodes/rules/presentation.smk b/papers/bmodes/rules/presentation.smk index c59f0aa3..7af2037a 100644 --- a/papers/bmodes/rules/presentation.smk +++ b/papers/bmodes/rules/presentation.smk @@ -113,9 +113,7 @@ rule presentation_s8_with_unions: rule presentation_blind_nz_plot: """Plot all three blinded n(z) curves for Moriond presentation.""" input: - nz_A=lambda w: build_redshift_path(FIDUCIAL["version"], "A"), - nz_B=lambda w: build_redshift_path(FIDUCIAL["version"], "B"), - nz_C=lambda w: build_redshift_path(FIDUCIAL["version"], "C"), + **{f"nz_{label}": redshift_path(ver) for label, ver in NZ_REALISATIONS.items()}, output: f"{TALK_DIR}/images/blind_nz_ABC.png", script: @@ -146,7 +144,7 @@ rule presentation_pte_cosebis: """COSEBIS B_n PTE heatmap for Moriond talk (single panel, talk-sized).""" input: pte_files=[ - f"{TAPESTRY_DIR}/cosebis_pte_matrix/pte_values/{FIDUCIAL['version']}/{FIDUCIAL['blind']}/pte_{i:03d}_{j:03d}.json" + f"{TAPESTRY_DIR}/cosebis_pte_matrix/pte_values/{FIDUCIAL['version']}/pte_{i:03d}_{j:03d}.json" for i, j in PTE_SCALE_CUT_PAIRS ], output: diff --git a/papers/bmodes/scripts/bb_covariance_blind_independence.py b/papers/bmodes/scripts/bb_covariance_blind_independence.py index 256a79b1..dfffe359 100644 --- a/papers/bmodes/scripts/bb_covariance_blind_independence.py +++ b/papers/bmodes/scripts/bb_covariance_blind_independence.py @@ -1,8 +1,10 @@ """ BB Covariance Blind Independence Claim -Tests whether BB covariances are blind-independent (as expected for null signals) -while EE covariances vary across blinds (due to sample variance from cosmological signal). +Tests whether BB covariances are independent of the A/B/C n(z) realisation (as +expected for null signals) while EE covariances vary across realisations (due to +sample variance from cosmological signal). Each realisation is a catalogue entry +of its own (config fiducial.nz_realisations), identical but for its n(z). Compares: - Pure E/B: cov(xi+^B), cov(xi-^B) vs cov(xi+^E), cov(xi-^E) @@ -29,7 +31,8 @@ plt.style.use(PAPER_MPLSTYLE) -BLINDS = ["A", "B", "C"] +# n(z) realisation labels: the keys of config fiducial.nz_realisations. +NZ_LABELS = ["A", "B", "C"] def load_pure_eb_diagonals(path): @@ -402,21 +405,21 @@ def main( """Run the BB-covariance blind-independence cross-check. ``pure_eb_paths`` / ``harmonic_paths`` / ``cov_integration_paths`` are dicts - keyed by blind (A/B/C). All paths are absolute; the per-blind mock-version + keyed by n(z) realisation (A/B/C). All paths are absolute; the per-realisation covariances are read directly, so the check is self-contained. """ version = config["fiducial"]["mock_version"] # Load pure E/B covariances for all blinds pure_eb_data = {} - for blind in BLINDS: + for blind in NZ_LABELS: pure_eb_data[blind] = load_pure_eb_diagonals(pure_eb_paths[blind]) theta = pure_eb_data["A"]["theta"] # Load harmonic covariances for all blinds harmonic_data = {} - for blind in BLINDS: + for blind in NZ_LABELS: harmonic_data[blind] = load_harmonic_diagonals(harmonic_paths[blind]) # Integration binning parameters from config @@ -425,7 +428,7 @@ def main( nbins_int = config["fiducial"]["nbins_int"] cosebis_data = {} - for blind in BLINDS: + for blind in NZ_LABELS: cosebis_data[blind] = load_cosebis_diagonals( xi_integration_path, cov_integration_paths[blind], @@ -620,9 +623,9 @@ def clean_ratios(d): def _from_snakemake(smk): config = smk.config - pure_eb_paths = {b: smk.input[f"pure_eb_{b}"] for b in BLINDS} - harmonic_paths = {b: smk.input[f"harmonic_{b}"] for b in BLINDS} - cov_integration_paths = {b: smk.input[f"cov_integration_{b}"] for b in BLINDS} + pure_eb_paths = {b: smk.input[f"pure_eb_{b}"] for b in NZ_LABELS} + harmonic_paths = {b: smk.input[f"harmonic_{b}"] for b in NZ_LABELS} + cov_integration_paths = {b: smk.input[f"cov_integration_{b}"] for b in NZ_LABELS} main( config=config, pure_eb_paths=pure_eb_paths, @@ -638,12 +641,11 @@ def _from_snakemake(smk): ) -def _cov_integration_path(cov_dir, version, blind, min_sep, max_sep, nbins): +def _cov_integration_path(cov_dir, version, min_sep, max_sep, nbins): """Reproduce common.covariance_path for the Gaussian integration-grid, masked covariance (suffix _processed.txt).""" base = ( - f"covariance_{version}_{blind}_g" - f"_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_masked" + f"covariance_{version}_g_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_masked" ) return os.path.join(cov_dir, base, f"{base}_processed.txt") @@ -661,7 +663,7 @@ def _from_cli(argv=None): ap.add_argument( "--pure-eb-dir", required=True, - help="Dir with {version}_{blind}_pure_eb_semianalytic.npz", + help="Dir with {version}_pure_eb_semianalytic.npz per n(z) realisation", ) ap.add_argument( "--cosmo-val-dir", @@ -691,22 +693,20 @@ def _from_cli(argv=None): ) npatch = fid["npatch"] + realisations = fid["nz_realisations"] pure_eb_paths = { - b: os.path.join(a.pure_eb_dir, f"{version}_{b}_pure_eb_semianalytic.npz") - for b in BLINDS + b: os.path.join(a.pure_eb_dir, f"{ver}_pure_eb_semianalytic.npz") + for b, ver in realisations.items() } harmonic_paths = { - b: os.path.join( - a.cosmo_val_dir, - f"pseudo_cl_cov_{version}_blind={b}_powspace_nbins=32.fits", - ) - for b in BLINDS + b: os.path.join(a.cosmo_val_dir, f"pseudo_cl_cov_{ver}_powspace_nbins=32.fits") + for b, ver in realisations.items() } cov_integration_paths = { b: _cov_integration_path( - a.covariance_dir, version, b, min_sep_int, max_sep_int, nbins_int + a.covariance_dir, ver, min_sep_int, max_sep_int, nbins_int ) - for b in BLINDS + for b, ver in realisations.items() } xi_integration_path = os.path.join( a.cosmo_val_dir, @@ -714,7 +714,7 @@ def _from_cli(argv=None): f"_nbins={nbins_int}_npatch={npatch}.txt", ) pseudo_cl_path = os.path.join( - a.cosmo_val_dir, f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" + a.cosmo_val_dir, f"pseudo_cl_{version}_powspace_nbins=32.sacc" ) out_dir = Path(a.out) diff --git a/papers/bmodes/scripts/calculate_pure_eb_ptes.py b/papers/bmodes/scripts/calculate_pure_eb_ptes.py index 3a623669..feb0be78 100644 --- a/papers/bmodes/scripts/calculate_pure_eb_ptes.py +++ b/papers/bmodes/scripts/calculate_pure_eb_ptes.py @@ -5,10 +5,10 @@ ξ_+^B / ξ_-^B / joint ξ_tot^B χ² PTE matrices over the scale-cut grid via ``sp_validation.b_modes.calculate_eb_statistics`` (Hartlap-corrected inverse MC covariance, debiased by the draw count), and writes the PTE matrices to -``{out}/{version}_{blind}_pure_eb_ptes.npz``. +``{out}/{version}_pure_eb_ptes.npz``. python calculate_pure_eb_ptes.py \ - --version SP_v1.4.6.3_leak_corr --blind A \ + --version SP_v1.4.6.3_leak_corr \ --pure-eb-data <..._pure_eb_semianalytic.npz> \ --n-samples 2000 --out """ @@ -23,7 +23,6 @@ def calculate_ptes( version, - blind, pure_eb_data, n_samples, output_dir, @@ -57,7 +56,7 @@ def calculate_ptes( } os.makedirs(output_dir, exist_ok=True) - out_path = os.path.join(output_dir, f"{version}_{blind}_pure_eb_ptes.npz") + out_path = os.path.join(output_dir, f"{version}_pure_eb_ptes.npz") np.savez(out_path, **output_data) print(f"Saved PTE matrices to {out_path}") return out_path @@ -66,14 +65,12 @@ def calculate_ptes( def _from_cli(argv=None): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("--version", required=True) - ap.add_argument("--blind", default="A") ap.add_argument("--pure-eb-data", required=True, help="Gathered semianalytic .npz") ap.add_argument("--n-samples", type=int, default=2000) ap.add_argument("--out", required=True, help="Output directory (lc {output})") a = ap.parse_args(argv) calculate_ptes( version=a.version, - blind=a.blind, pure_eb_data=a.pure_eb_data, n_samples=a.n_samples, output_dir=a.out, diff --git a/papers/bmodes/scripts/cl_data_vector.py b/papers/bmodes/scripts/cl_data_vector.py index 4eeca960..724c38cb 100644 --- a/papers/bmodes/scripts/cl_data_vector.py +++ b/papers/bmodes/scripts/cl_data_vector.py @@ -161,14 +161,12 @@ def _create_cl_figure( # via _pseudo_cl_path() in claims.smk; the CLI reconstructs them from --results-dir # so the version sweep is self-contained (lc produces only the fiducial version). # --------------------------------------------------------------------------- -def _pseudo_cl(results_dir, ver, blind="A", nbins=32): - return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.sacc" +def _pseudo_cl(results_dir, ver, nbins=32): + return f"{results_dir}/pseudo_cl_{ver}_powspace_nbins={nbins}.sacc" -def _pseudo_cl_cov(results_dir, ver, blind="A", nbins=32): - return ( - f"{results_dir}/pseudo_cl_cov_{ver}_blind={blind}_powspace_nbins={nbins}.fits" - ) +def _pseudo_cl_cov(results_dir, ver, nbins=32): + return f"{results_dir}/pseudo_cl_cov_{ver}_powspace_nbins={nbins}.fits" def _resolve_pseudo_cl_paths( @@ -177,7 +175,6 @@ def _resolve_pseudo_cl_paths( fiducial_version=None, fiducial_pseudo_cl_path=None, fiducial_pseudo_cl_cov_path=None, - blind="A", nbins=32, ): """Resolve the pseudo-Cl and covariance paths for one version in the sweep. @@ -194,8 +191,8 @@ def _resolve_pseudo_cl_paths( ): return fiducial_pseudo_cl_path, fiducial_pseudo_cl_cov_path return ( - _pseudo_cl(results_dir, ver, blind, nbins), - _pseudo_cl_cov(results_dir, ver, blind, nbins), + _pseudo_cl(results_dir, ver, nbins), + _pseudo_cl_cov(results_dir, ver, nbins), ) diff --git a/papers/bmodes/scripts/cl_version_comparison.py b/papers/bmodes/scripts/cl_version_comparison.py index f70a6977..7f82d14f 100644 --- a/papers/bmodes/scripts/cl_version_comparison.py +++ b/papers/bmodes/scripts/cl_version_comparison.py @@ -34,14 +34,12 @@ plt.style.use(PAPER_MPLSTYLE) -def _pseudo_cl(results_dir, ver, blind="A", nbins=32): - return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.sacc" +def _pseudo_cl(results_dir, ver, nbins=32): + return f"{results_dir}/pseudo_cl_{ver}_powspace_nbins={nbins}.sacc" -def _pseudo_cl_cov(results_dir, ver, blind="A", nbins=32): - return ( - f"{results_dir}/pseudo_cl_cov_{ver}_blind={blind}_powspace_nbins={nbins}.fits" - ) +def _pseudo_cl_cov(results_dir, ver, nbins=32): + return f"{results_dir}/pseudo_cl_cov_{ver}_powspace_nbins={nbins}.fits" def _resolve_pseudo_cl_paths( @@ -50,7 +48,6 @@ def _resolve_pseudo_cl_paths( fiducial_version=None, fiducial_pseudo_cl_path=None, fiducial_pseudo_cl_cov_path=None, - blind="A", nbins=32, ): """Resolve the pseudo-Cl and covariance paths for one version in the sweep. @@ -67,8 +64,8 @@ def _resolve_pseudo_cl_paths( ): return fiducial_pseudo_cl_path, fiducial_pseudo_cl_cov_path return ( - _pseudo_cl(results_dir, ver, blind, nbins), - _pseudo_cl_cov(results_dir, ver, blind, nbins), + _pseudo_cl(results_dir, ver, nbins), + _pseudo_cl_cov(results_dir, ver, nbins), ) diff --git a/papers/bmodes/scripts/compute_cosebis_pte_single.py b/papers/bmodes/scripts/compute_cosebis_pte_single.py index 452fc514..461ef4fc 100644 --- a/papers/bmodes/scripts/compute_cosebis_pte_single.py +++ b/papers/bmodes/scripts/compute_cosebis_pte_single.py @@ -2,7 +2,7 @@ """Compute the COSEBIS B-mode PTE matrix over the full scale-cut pair grid. Reconciliation of the ASTRA ``cosebis_pte_per_cut`` output: the Snakemake DAG -*scattered* one JSON per (version, blind, i_min, i_max) cut via this script and +*scattered* one JSON per (version, i_min, i_max) cut via this script and gathered them downstream. The spec describes the gathered NPZ that scatter never materialised. Here a single CLI loops the same ``_pte_scale_cut_pairs()`` grid, runs the *identical* per-pair COSEBI computation (same theta grid, nmodes, NaN-on- @@ -79,15 +79,14 @@ def _compute_pair(gg, cov_path, nmodes, theta_min, theta_max): } -def main(config, xi_integration, cov_integration, out_dir, version=None, blind=None): +def main(config, xi_integration, cov_integration, out_dir, version=None): t_start = time.time() fid = config["fiducial"] - # Version/blind tag only the output filename + provenance record; the theta + # Version tags only the output filename + provenance record; the theta # grid, nmodes and per-pair COSEBI compute are version-independent (they read # from config["fiducial"]), so a sweep call over a non-fiducial catalog stays # bit-identical to the fiducial call save for the xi/cov inputs and the tag. version = version if version is not None else fid["version"] - blind = blind if blind is not None else fid["blind"] nmodes = int(fid["nmodes"]) # 20 for full computation min_sep_int = fid["min_sep_int"] @@ -146,11 +145,10 @@ def _mat(): out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) - npz_path = out_dir / f"cosebis_ptes_{version}_{blind}.npz" + npz_path = out_dir / f"cosebis_ptes_{version}.npz" np.savez( npz_path, version=version, - blind=blind, nmodes=nmodes, mode_subsets=np.array([6, 20]), theta_grid=theta_grid, @@ -177,7 +175,7 @@ def _from_cli(argv=None): import yaml ap = argparse.ArgumentParser( - description="Gathered COSEBI B-mode PTE matrix over the scale-cut pair grid (fiducial version + blind)." + description="Gathered COSEBI B-mode PTE matrix over the scale-cut pair grid (fiducial version)." ) ap.add_argument( "--config", required=True, help="Absolute path to bmodes config.yaml" @@ -199,11 +197,6 @@ def _from_cli(argv=None): help="Catalog version tag (default: config.fiducial.version). Overridden " "by the version sweep to name the non-fiducial output NPZ.", ) - ap.add_argument( - "--blind", - default=None, - help="Blind tag (default: config.fiducial.blind)", - ) a = ap.parse_args(argv) with open(a.config) as f: config = yaml.safe_load(f) @@ -213,7 +206,6 @@ def _from_cli(argv=None): a.cov_integration, a.out, version=a.version, - blind=a.blind, ) diff --git a/papers/bmodes/scripts/config_space_pte_matrices.py b/papers/bmodes/scripts/config_space_pte_matrices.py index 474b71a7..e63b9ace 100644 --- a/papers/bmodes/scripts/config_space_pte_matrices.py +++ b/papers/bmodes/scripts/config_space_pte_matrices.py @@ -18,7 +18,7 @@ --config /path/to/config.yaml \ --pte-intermediate-dir /abs/.../paper_plots/intermediate \ --cosebis-pte-dir /abs/.../tapestry/cosebis_pte_matrix/pte_values \ - --blind A --out + --out """ import argparse @@ -76,7 +76,7 @@ def _resolve_overrides(version, fiducial_overrides): the sweep versions on the *same* gathered-NPZ provenance (adapted back to the per-pair PTE matrix by ``_cosebis_matrix_from_npz``) rather than the old per-pair-JSON scatter tree. Pure E/B has no such split — the sweep emits the - old-tree ``{ver}_{blind}_pure_eb_ptes.npz`` name, so non-fiducial versions + old-tree ``{ver}_pure_eb_ptes.npz`` name, so non-fiducial versions read it straight from the ``--pte-intermediate-dir`` file list (override None). Either element may be None. """ @@ -112,12 +112,10 @@ def load_cosebis_pte_matrix( ): """Load COSEBIS PTE values from JSON files into matrix. - Uses fiducial blind only (data vectors identical across blinds). - Parameters ---------- pte_files : list of str - Paths to PTE JSON files for fiducial blind. + Paths to PTE JSON files. version : str Version to filter for. config : dict @@ -174,12 +172,10 @@ def load_cosebis_pte_matrix( def load_pure_eb_pte_matrices(pte_files, version, override_path=None): """Load Pure E/B PTE matrices from npz files. - Uses fiducial blind only (data vectors identical across blinds). - Parameters ---------- pte_files : list of str - Paths to pure_eb_ptes.npz files for fiducial blind. + Paths to pure_eb_ptes.npz files. version : str Version to filter for. @@ -368,12 +364,12 @@ def extract_full_range_ptes( ): """Extract full-range PTEs from npz and JSON files. - Takes minimum PTE across blinds for each statistic. + Takes minimum PTE across matching files for each statistic. Parameters ---------- pure_eb_pte_files : list - All Pure E/B PTE npz files (includes all blinds). + All Pure E/B PTE npz files. cosebis_pte_files : list All COSEBIS PTE JSON files. version : str @@ -382,11 +378,11 @@ def extract_full_range_ptes( Returns ------- ptes : dict - Full-range PTEs for xip, xim, and cosebis (fiducial blind). + Full-range PTEs for xip, xim, and cosebis. """ pure_eb_override, cosebis_override = _resolve_overrides(version, fiducial_overrides) - # Get full-range PTEs from pure E/B (fiducial blind) + # Get full-range PTEs from pure E/B xip_ptes = [] xim_ptes = [] combined_ptes = [] @@ -428,7 +424,7 @@ def extract_full_range_ptes( ptes["cosebis_20"] = pte20 return ptes - # Old tree: pte_000_020.json for full theta range, min across blinds. + # Old tree: pte_000_020.json for full theta range. cosebis_ptes_6 = [] cosebis_ptes_20 = [] for pte_file in cosebis_pte_files: @@ -474,7 +470,7 @@ def create_3panel_composite( version : str Catalog version string (fiducial). pure_eb_pte_files : list - All Pure E/B PTE npz files (includes all blinds). + All Pure E/B PTE npz files. cosebis_pte_files : list All COSEBIS PTE JSON files. xip_fid, xim_fid : tuple @@ -625,7 +621,7 @@ def create_9panel_composite( versions : list of str Catalog version strings in display order. pure_eb_pte_files : list - All Pure E/B PTE npz files (includes all blinds). + All Pure E/B PTE npz files. cosebis_pte_files : list All COSEBIS PTE JSON files. xip_fid, xim_fid : tuple @@ -1050,16 +1046,15 @@ def _from_cli(argv=None): ap.add_argument( "--pte-intermediate-dir", required=True, - help="Directory holding {version}_{blind}_pure_eb_ptes.npz", + help="Directory holding {version}_pure_eb_ptes.npz", ) ap.add_argument( "--cosebis-pte-dir", required=True, help="COSEBI PTE source dir. lc cosebis_ptes sweep: gathered " - "cosebis_ptes_{version}_{blind}.npz per version (auto-detected, preferred). " - "Old tree fallback: {version}/{blind}/pte_{i:03d}_{j:03d}.json scatter.", + "cosebis_ptes_{version}.npz per version (auto-detected, preferred). " + "Old tree fallback: {version}/pte_{i:03d}_{j:03d}.json scatter.", ) - ap.add_argument("--blind", default="A", help="Fiducial blind (paper: A)") ap.add_argument("--out", required=True, help="Output directory (lc {output})") ap.add_argument( "--fiducial-version", @@ -1071,7 +1066,7 @@ def _from_cli(argv=None): "--fiducial-pure-eb-pte-path", default=None, help="lc pure_eb PTE NPZ for the fiducial version " - "(same format as old-tree {version}_{blind}_pure_eb_ptes.npz)", + "(same format as old-tree {version}_pure_eb_ptes.npz)", ) ap.add_argument( "--fiducial-cosebis-pte-path", @@ -1092,8 +1087,7 @@ def _from_cli(argv=None): versions = _versions_config_space(config) pure_eb_pte_files = [ - os.path.join(a.pte_intermediate_dir, f"{v}_{a.blind}_pure_eb_ptes.npz") - for v in versions + os.path.join(a.pte_intermediate_dir, f"{v}_pure_eb_ptes.npz") for v in versions ] pure_eb_pte_files = [p for p in pure_eb_pte_files if os.path.exists(p)] @@ -1101,18 +1095,14 @@ def _from_cli(argv=None): # version, same layout as the fiducial single-output) so fiducial and sweep # versions share provenance; fall back to the old per-pair-JSON scatter tree. cosebis_by_version = { - v: os.path.join(a.cosebis_pte_dir, f"cosebis_ptes_{v}_{a.blind}.npz") + v: os.path.join(a.cosebis_pte_dir, f"cosebis_ptes_{v}.npz") for v in versions - if os.path.exists( - os.path.join(a.cosebis_pte_dir, f"cosebis_ptes_{v}_{a.blind}.npz") - ) + if os.path.exists(os.path.join(a.cosebis_pte_dir, f"cosebis_ptes_{v}.npz")) } cosebis_pte_files = ( [] if cosebis_by_version - else sorted( - glob.glob(os.path.join(a.cosebis_pte_dir, "*", a.blind, "pte_*.json")) - ) + else sorted(glob.glob(os.path.join(a.cosebis_pte_dir, "*", "pte_*.json"))) ) fiducial_overrides = None diff --git a/papers/bmodes/scripts/cosebis_version_comparison.py b/papers/bmodes/scripts/cosebis_version_comparison.py index 9942bb28..588eb3c1 100644 --- a/papers/bmodes/scripts/cosebis_version_comparison.py +++ b/papers/bmodes/scripts/cosebis_version_comparison.py @@ -151,8 +151,8 @@ def _xi_integration(results_dir, ver): return f"{results_dir}/{ver}_xi_minsep=0.5_maxsep=300.0_nbins=1000_npatch=1.txt" -def _cov_integration(cov_dir, ver, blind): - base = f"covariance_{ver}_{blind}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" +def _cov_integration(cov_dir, ver): + base = f"covariance_{ver}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" return f"{cov_dir}/{base}/{base}_processed.txt" @@ -170,7 +170,6 @@ def main( nmodes = config["fiducial"]["nmodes"] plotting_config = config["plotting"] version_labels = plotting_config["version_labels"] - blind = config["fiducial"]["blind"] # Fiducial version whose inputs may be overridden with explicit lc paths fiducial_version = fiducial_version or config["fiducial"]["version"] @@ -186,7 +185,7 @@ def main( cov_paths_list = [ fiducial_cov_path if v == fiducial_version and fiducial_cov_path - else _cov_integration(cov_dir, v, blind) + else _cov_integration(cov_dir, v) for v in versions ] diff --git a/papers/bmodes/scripts/gather_pure_eb_chunks.py b/papers/bmodes/scripts/gather_pure_eb_chunks.py index 887e512f..b6d5d702 100644 --- a/papers/bmodes/scripts/gather_pure_eb_chunks.py +++ b/papers/bmodes/scripts/gather_pure_eb_chunks.py @@ -3,12 +3,12 @@ CLI refactor of the former Snakemake ``script:`` gather rule. Reads the actual ξ± data vectors (reporting + integration grids), computes the pure E/B/ambiguous decomposition (Schneider 2022), stacks the per-chunk MC sample blocks, and -forms the empirical 6-block covariance. Writes the per-(version, blind) -``__pure_eb_semianalytic.npz`` consumed by every downstream +forms the empirical 6-block covariance. Writes the per-version +``_pure_eb_semianalytic.npz`` consumed by every downstream pure-mode plot / PTE. python gather_pure_eb_chunks.py \ - --version SP_v1.4.6.3_leak_corr --blind A \ + --version SP_v1.4.6.3_leak_corr \ --xi-reporting \ --xi-integration \ --chunks-dir \ @@ -32,7 +32,6 @@ def _load_xi(path, nbins): def gather( version, - blind, xi_reporting, xi_integration, chunk_files, @@ -44,7 +43,7 @@ def gather( ): from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes - print(f"Gathering pure E/B for blind {blind}") + print(f"Gathering pure E/B for {version}") gg = _load_xi(xi_reporting, nbins) gg_int = _load_xi(xi_integration, nbins_int) @@ -88,7 +87,7 @@ def gather( } os.makedirs(output_dir, exist_ok=True) - out_path = os.path.join(output_dir, f"{version}_{blind}_pure_eb_semianalytic.npz") + out_path = os.path.join(output_dir, f"{version}_pure_eb_semianalytic.npz") np.savez(out_path, **package) print(f"Saved to {out_path}") return out_path @@ -107,7 +106,6 @@ def _resolve_chunks(args): def _from_cli(argv=None): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("--version", required=True) - ap.add_argument("--blind", default="A") ap.add_argument("--xi-reporting", required=True) ap.add_argument("--xi-integration", required=True) ap.add_argument("--chunks-dir", help="Directory holding pure_eb_chunk_*.npz") @@ -123,7 +121,6 @@ def _from_cli(argv=None): a = ap.parse_args(argv) gather( version=a.version, - blind=a.blind, xi_reporting=a.xi_reporting, xi_integration=a.xi_integration, chunk_files=_resolve_chunks(a), diff --git a/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py b/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py index dd34db22..55904886 100644 --- a/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py +++ b/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py @@ -807,12 +807,11 @@ def _versions_all_for_plots(config): return leak_corr + uncorrected -def _cov_integration_path(cov_dir, version, blind, min_sep, max_sep, nbins): +def _cov_integration_path(cov_dir, version, min_sep, max_sep, nbins): """Reproduce common.covariance_path for the Gaussian integration-grid, masked covariance (suffix _processed.txt).""" base = ( - f"covariance_{version}_{blind}_g" - f"_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_masked" + f"covariance_{version}_g_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_masked" ) return os.path.join(cov_dir, base, f"{base}_processed.txt") @@ -867,9 +866,6 @@ def _from_cli(argv=None): required=True, help="COSMO_INFERENCE data/covariance dir (Gaussian integration covariances)", ) - ap.add_argument( - "--blind", default="A", help="Blind for pseudo-Cl / covariance (paper: A)" - ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") ap.add_argument( "--fiducial-version", @@ -938,7 +934,7 @@ def _from_cli(argv=None): if is_fiducial and a.fiducial_pseudo_cl_path else os.path.join( a.cosmo_val_dir, - f"pseudo_cl_{ver}_blind={a.blind}_powspace_nbins={cosebis_nbins}.sacc", + f"pseudo_cl_{ver}_powspace_nbins={cosebis_nbins}.sacc", ) ) # 96-bin pseudo-Cl covariance is intentionally NOT lc-repointed: lc did not @@ -947,7 +943,7 @@ def _from_cli(argv=None): # version. inputs[f"pseudo_cl_cov_{ver}"] = os.path.join( a.cosmo_val_dir, - f"pseudo_cl_cov_{ver}_blind={a.blind}_powspace_nbins={cosebis_nbins}.fits", + f"pseudo_cl_cov_{ver}_powspace_nbins={cosebis_nbins}.fits", ) inputs[f"xi_{ver}"] = ( a.fiducial_xi_path @@ -962,7 +958,7 @@ def _from_cli(argv=None): a.fiducial_cov_path if is_fiducial and a.fiducial_cov_path else _cov_integration_path( - a.covariance_dir, ver, a.blind, min_sep_int, max_sep_int, nbins_int + a.covariance_dir, ver, min_sep_int, max_sep_int, nbins_int ) ) diff --git a/papers/bmodes/scripts/harmonic_space_pte_matrices.py b/papers/bmodes/scripts/harmonic_space_pte_matrices.py index aeb031c6..54c353f0 100644 --- a/papers/bmodes/scripts/harmonic_space_pte_matrices.py +++ b/papers/bmodes/scripts/harmonic_space_pte_matrices.py @@ -3,8 +3,6 @@ Produces: - Results: single-panel fiducial Cl^BB PTE matrix - Appendix: N-panel composite for all versions from config.versions - -Uses fiducial blind covariance only (blind independence validated elsewhere). """ import argparse @@ -31,14 +29,12 @@ plt.style.use(PAPER_MPLSTYLE) -def _pseudo_cl(results_dir, ver, blind="A", nbins=32): - return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.sacc" +def _pseudo_cl(results_dir, ver, nbins=32): + return f"{results_dir}/pseudo_cl_{ver}_powspace_nbins={nbins}.sacc" -def _pseudo_cl_cov(results_dir, ver, blind="A", nbins=32): - return ( - f"{results_dir}/pseudo_cl_cov_{ver}_blind={blind}_powspace_nbins={nbins}.fits" - ) +def _pseudo_cl_cov(results_dir, ver, nbins=32): + return f"{results_dir}/pseudo_cl_cov_{ver}_powspace_nbins={nbins}.fits" def compute_pte_matrix( @@ -355,7 +351,6 @@ def main( ): versions = [v for v in config["versions"] if "_ecut" not in v] fiducial_version = config["fiducial"]["version"] - fiducial_blind = config["fiducial"]["blind"] # Fiducial ell cuts from config fiducial_ell_min = config["cl"]["fiducial_ell_min"] @@ -365,16 +360,16 @@ def main( output_dir = Path(out_dir) output_dir.mkdir(parents=True, exist_ok=True) - # Per-version pseudo-Cl / covariance files (canonical COSMO_VAL tree, blind A). - # Mirrors _pseudo_cl_path(ver) / _pseudo_cl_cov_path(ver, blind) in claims.smk. + # Per-version pseudo-Cl / covariance files (canonical COSMO_VAL tree). + # Mirrors _pseudo_cl_path(ver) / _pseudo_cl_cov_path(ver) in claims.smk. # Fiducial-provenance repoint: when --fiducial-version matches and an explicit # lc override path is set, read that path instead of the reconstructed pattern - # (lc files lack the blind=/powspace_nbins= tokens, distinguished by directory). + # (lc files lack the powspace_nbins= token, distinguished by directory). version_to_cl = { ver: ( fiducial_pseudo_cl_path if ver == fiducial_version_override and fiducial_pseudo_cl_path is not None - else _pseudo_cl(results_dir, ver, blind=fiducial_blind) + else _pseudo_cl(results_dir, ver) ) for ver in versions } @@ -383,18 +378,18 @@ def main( fiducial_pseudo_cl_cov_path if ver == fiducial_version_override and fiducial_pseudo_cl_cov_path is not None - else _pseudo_cl_cov(results_dir, ver, blind=fiducial_blind) + else _pseudo_cl_cov(results_dir, ver) ) for ver in versions } - # Compute PTE matrices for all versions using fiducial blind + # Compute PTE matrices for all versions all_stats = {} all_matrices = {} all_ells = {} for version in versions: - print(f"\n--- Processing {version} (blind={fiducial_blind}) ---") + print(f"\n--- Processing {version} ---") if version not in version_to_cl: print(f" WARNING: No pseudo-Cl file found for {version}, skipping") @@ -425,14 +420,13 @@ def main( npz_payload = { "versions": np.array(list(all_matrices.keys())), "fiducial_version": fiducial_version, - "fiducial_blind": fiducial_blind, "fiducial_ell_min": fiducial_ell_min, "fiducial_ell_max": fiducial_ell_max, } for version in all_matrices: npz_payload[f"{version}__pte_matrix"] = all_matrices[version] npz_payload[f"{version}__ell"] = all_ells[version] - npz_path = output_dir / f"cl_pte_matrices_{fiducial_blind}.npz" + npz_path = output_dir / "cl_pte_matrices.npz" np.savez(npz_path, **npz_payload) print(f"Saved PTE matrices to {npz_path}") @@ -494,7 +488,6 @@ def main( "spec_id": "harmonic_space_pte_matrices", "generated": datetime.now().isoformat(), "evidence": { - "blind": fiducial_blind, "versions": {}, }, "output": { diff --git a/papers/bmodes/scripts/precompute_pure_eb_chunk.py b/papers/bmodes/scripts/precompute_pure_eb_chunk.py index 99153096..2db6bf92 100644 --- a/papers/bmodes/scripts/precompute_pure_eb_chunk.py +++ b/papers/bmodes/scripts/precompute_pure_eb_chunk.py @@ -12,7 +12,7 @@ python precompute_pure_eb_chunk.py \ --chunk-id 0 --n-chunks 20 --n-samples 2000 \ - --version SP_v1.4.6.3_leak_corr --blind A \ + --version SP_v1.4.6.3_leak_corr \ --cat-config /path/cosmo_val/cat_config.yaml \ --xi-reporting \ --xi-integration \ @@ -64,7 +64,6 @@ def compute_chunk( n_chunks, n_samples_total, version, - blind, cat_config, xi_reporting, xi_integration, @@ -105,10 +104,8 @@ def compute_chunk( catalog_config=cat_config, output_dir=output_dir, ) - cv.blind = blind z, nz = cv.get_redshift(version) z_dist = np.column_stack([z, nz]) - print(f"Using n(z) for blind {blind}") cosmo_cov = _build_cosmology(cosmo_params) @@ -176,7 +173,6 @@ def _from_cli(argv=None): ap.add_argument("--n-chunks", type=int, default=20) ap.add_argument("--n-samples", type=int, default=2000) ap.add_argument("--version", required=True) - ap.add_argument("--blind", default="A") ap.add_argument("--cat-config", required=True) ap.add_argument("--xi-reporting", required=True) ap.add_argument("--xi-integration", required=True) @@ -195,7 +191,6 @@ def _from_cli(argv=None): n_chunks=a.n_chunks, n_samples_total=a.n_samples, version=a.version, - blind=a.blind, cat_config=a.cat_config, xi_reporting=a.xi_reporting, xi_integration=a.xi_integration, diff --git a/papers/bmodes/scripts/pure_eb_covariance.py b/papers/bmodes/scripts/pure_eb_covariance.py index ee5ac77c..f978f6e4 100644 --- a/papers/bmodes/scripts/pure_eb_covariance.py +++ b/papers/bmodes/scripts/pure_eb_covariance.py @@ -191,7 +191,6 @@ def main(config, pure_eb_path, out_dir, specs=()): }, "parameters": { "version": version, - "blind": config["fiducial"]["blind"], }, "output": { "figure": "figure.png", @@ -229,7 +228,7 @@ def _from_cli(argv=None): ap.add_argument( "--pure-eb-data", required=True, - help="Fiducial __pure_eb_semianalytic.npz " + help="Fiducial _pure_eb_semianalytic.npz " "(provides the 6-block cov_pure_eb)", ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") diff --git a/papers/bmodes/scripts/pure_eb_data_vector.py b/papers/bmodes/scripts/pure_eb_data_vector.py index dbbaeda7..5c0c2dda 100644 --- a/papers/bmodes/scripts/pure_eb_data_vector.py +++ b/papers/bmodes/scripts/pure_eb_data_vector.py @@ -2,12 +2,12 @@ Fiducial catalog only: pure E/B/ambiguous decomposition of ξ± with B-modes consistent with zero at the fiducial scale cuts. Writes evidence.json with PTE -values including the joint B-mode test (fiducial blind, config.fiducial.blind). +values including the joint B-mode test. CLI: python pure_eb_data_vector.py \ --config config.yaml \ - --pure-eb-data __pure_eb_semianalytic.npz \ + --pure-eb-data _pure_eb_semianalytic.npz \ --reporting-cov /covariance_processed.txt \ --out [--specs spec.md ...] """ @@ -218,7 +218,6 @@ def setup_panel(ax, ylabel_text=None): def main(config, pure_eb_path, cov_path, out_dir, specs=()): - blind = config["fiducial"]["blind"] version = config["fiducial"]["version"] fiducial_xip_scale_cut = tuple(config["fiducial"]["fiducial_xip_scale_cut"]) fiducial_xim_scale_cut = tuple(config["fiducial"]["fiducial_xim_scale_cut"]) @@ -292,7 +291,7 @@ def main(config, pure_eb_path, cov_path, out_dir, specs=()): ) print( - f"Blind {blind} PTEs (fiducial): xi+^B={pte_xip_fid:.3f}, xi-^B={pte_xim_fid:.3f}, joint={pte_joint_fid:.3f}" + f"PTEs (fiducial): xi+^B={pte_xip_fid:.3f}, xi-^B={pte_xim_fid:.3f}, joint={pte_joint_fid:.3f}" ) # Write evidence.json (based on leak-corrected fiducial data only) @@ -319,7 +318,6 @@ def main(config, pure_eb_path, cov_path, out_dir, specs=()): "dof_joint_B": int(dof_joint_full), }, "version": version, - "blind": blind, }, "output": {"figure": "figure.png"}, } @@ -342,7 +340,7 @@ def _from_cli(argv=None): ap.add_argument( "--pure-eb-data", required=True, - help="Fiducial __pure_eb_semianalytic.npz " + help="Fiducial _pure_eb_semianalytic.npz " "(decomposed ξ± + 6-block MC covariance)", ) ap.add_argument( diff --git a/papers/bmodes/scripts/pure_eb_version_comparison.py b/papers/bmodes/scripts/pure_eb_version_comparison.py index 8027189f..3f97db43 100644 --- a/papers/bmodes/scripts/pure_eb_version_comparison.py +++ b/papers/bmodes/scripts/pure_eb_version_comparison.py @@ -242,8 +242,8 @@ def _create_version_comparison_figure( return fig -def _pure_eb_npz(results_dir, ver, blind): - return f"{results_dir}/{ver}_{blind}_pure_eb_semianalytic.npz" +def _pure_eb_npz(results_dir, ver): + return f"{results_dir}/{ver}_pure_eb_semianalytic.npz" def main( @@ -258,7 +258,6 @@ def main( version_labels = plotting_config["version_labels"] # Leak-corrected, non-ecut versions (matches VERSIONS_LEAK_CORR in claims.smk) versions = [v for v in config["versions"] if "_leak_corr" in v and "_ecut" not in v] - blind = config["fiducial"]["blind"] # Fiducial version whose NPZ may be overridden with an explicit lc path fiducial_version = fiducial_version or config["fiducial"]["version"] @@ -268,7 +267,7 @@ def main( data_paths = [ fiducial_pure_eb_data if v == fiducial_version and fiducial_pure_eb_data - else _pure_eb_npz(results_dir, v, blind) + else _pure_eb_npz(results_dir, v) for v in versions ] @@ -476,8 +475,7 @@ def _from_cli(argv=None): ap.add_argument( "--results-dir", required=True, - help="Directory holding per-version " - "__pure_eb_semianalytic.npz files", + help="Directory holding per-version _pure_eb_semianalytic.npz files", ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") ap.add_argument( diff --git a/papers/bmodes/scripts/run_cl_sweep.py b/papers/bmodes/scripts/run_cl_sweep.py index d5fd6eb0..1bba7192 100644 --- a/papers/bmodes/scripts/run_cl_sweep.py +++ b/papers/bmodes/scripts/run_cl_sweep.py @@ -5,8 +5,8 @@ fiducial cl_bb / covariance recipes call, once per version. Each producer writes its native ``pseudo_cl_{ver}.fits`` / ``pseudo_cl_cov_{ver}.fits`` into ``--out``; this driver then renames them to the tagged canonical names -``pseudo_cl_{ver}_blind={blind}_{binning}_nbins={nbins}.fits`` and -``pseudo_cl_cov_{ver}_blind={blind}_{binning}_nbins={nbins}.fits`` — the exact +``pseudo_cl_{ver}_{binning}_nbins={nbins}.fits`` and +``pseudo_cl_cov_{ver}_{binning}_nbins={nbins}.fits`` — the exact pattern ``cl_version_comparison._pseudo_cl`` / ``._pseudo_cl_cov`` reconstruct. Per-version footprint masks resolve internally from cat_config (the version's @@ -16,7 +16,7 @@ python run_cl_sweep.py \ --config .../config.yaml --cat-config .../cat_config.yaml \ --nside 1024 --npatch 1 --binning powspace --nbins 32 --power 0.5 \ - --blind A --out + --out """ import argparse @@ -40,17 +40,15 @@ from sweep_versions import nonfiducial_versions # noqa: E402 -def _canonical(prefix, ver, blind, binning, nbins): - return f"{prefix}_{ver}_blind={blind}_{binning}_nbins={nbins}.fits" +def _canonical(prefix, ver, binning, nbins): + return f"{prefix}_{ver}_{binning}_nbins={nbins}.fits" -def _run_and_tag(producer, prefix, ver, out, blind, binning, nbins, **kw): +def _run_and_tag(producer, prefix, ver, out, binning, nbins, **kw): """Run one per-version producer, then rename its native FITS to canonical.""" - producer( - version=ver, output_dir=out, blind=blind, binning=binning, nbins=nbins, **kw - ) + producer(version=ver, output_dir=out, binning=binning, nbins=nbins, **kw) native = os.path.join(out, f"{prefix}_{ver}.fits") - canonical = os.path.join(out, _canonical(prefix, ver, blind, binning, nbins)) + canonical = os.path.join(out, _canonical(prefix, ver, binning, nbins)) os.replace(native, canonical) print(f"[cl_sweep] {ver}: {os.path.basename(canonical)}") @@ -74,7 +72,6 @@ def _from_cli(argv=None): ) ap.add_argument("--nbins", type=int, default=32) ap.add_argument("--power", type=float, default=0.5) - ap.add_argument("--blind", choices=["A", "B", "C"], default="A") a = ap.parse_args(argv) with open(a.config) as f: config = yaml.safe_load(f) @@ -92,7 +89,6 @@ def _from_cli(argv=None): "pseudo_cl", ver, a.out, - a.blind, a.binning, a.nbins, **shared, @@ -102,7 +98,6 @@ def _from_cli(argv=None): "pseudo_cl_cov", ver, a.out, - a.blind, a.binning, a.nbins, **shared, diff --git a/papers/bmodes/scripts/run_cosebis_ptes_sweep.py b/papers/bmodes/scripts/run_cosebis_ptes_sweep.py index df3b9b79..b4b34041 100644 --- a/papers/bmodes/scripts/run_cosebis_ptes_sweep.py +++ b/papers/bmodes/scripts/run_cosebis_ptes_sweep.py @@ -9,7 +9,7 @@ driver reads that version's 1000-bin integration ξ_± from the xi_sweep output dir and its Gaussian integration covariance from the cov_sweep output dir (both by absolute path — lc does not wire cross-output deps, so run xi_sweep + cov_sweep -first), emitting the canonical ``cosebis_ptes_{ver}_{blind}.npz`` — the same +first), emitting the canonical ``cosebis_ptes_{ver}.npz`` — the same gathered layout the fiducial single-output writes, which config_space_pte_matrices.py adapts via ``_cosebis_matrix_from_npz`` — into ``--out``. Serial over versions (~25 min/version, 206 pairs). @@ -18,7 +18,7 @@ --config .../config.yaml \ --xi-sweep-dir \ --cov-sweep-dir \ - --out [--blind A] [--versions v1 v2 ...] + --out [--versions v1 v2 ...] """ import argparse @@ -47,8 +47,8 @@ def _xi_integration(xi_sweep_dir, ver): ) -def _cov_integration(cov_sweep_dir, ver, blind): - base = f"covariance_{ver}_{blind}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" +def _cov_integration(cov_sweep_dir, ver): + base = f"covariance_{ver}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" return os.path.join(cov_sweep_dir, base, f"{base}_processed.txt") @@ -70,7 +70,6 @@ def _from_cli(argv=None): help="cov_sweep output dir with per-version {base}/{base}_processed.txt cov", ) ap.add_argument("--out", required=True, help="Sweep output directory (lc {output})") - ap.add_argument("--blind", default="A", help="Blind tag (paper: A)") ap.add_argument("--versions", nargs="*", default=None, help="Explicit version keys") a = ap.parse_args(argv) @@ -81,15 +80,15 @@ def _from_cli(argv=None): for ver in versions: xi_int = _xi_integration(a.xi_sweep_dir, ver) - cov_int = _cov_integration(a.cov_sweep_dir, ver, a.blind) + cov_int = _cov_integration(a.cov_sweep_dir, ver) for f in (xi_int, cov_int): if not os.path.isfile(f): raise FileNotFoundError(f"MISSING upstream input for {ver}: {f}") print(f"[cosebis_ptes_sweep] {ver}", flush=True) - compute_cosebis_pte(config, xi_int, cov_int, a.out, version=ver, blind=a.blind) + compute_cosebis_pte(config, xi_int, cov_int, a.out, version=ver) print( f"[cosebis_ptes_sweep] {ver} -> " - f"{os.path.join(a.out, f'cosebis_ptes_{ver}_{a.blind}.npz')}", + f"{os.path.join(a.out, f'cosebis_ptes_{ver}.npz')}", flush=True, ) print(f"[cosebis_ptes_sweep] done -> {a.out}", flush=True) diff --git a/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh b/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh index 0e7ecdf5..2f4e4d91 100644 --- a/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh +++ b/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh @@ -7,26 +7,25 @@ # NPZ (data vectors + MC covariance) is read from the pure_eb_sweep output dir by # absolute path — lc does not wire cross-output deps, so the driver reads the # upstream sweep directly; run pure_eb_sweep before this. Each version emits the -# canonical ``{ver}_{blind}_pure_eb_ptes.npz`` — the exact name +# canonical ``{ver}_pure_eb_ptes.npz`` — the exact name # config_space_pte_matrices.py reconstructs from --pte-intermediate-dir — straight # into --out. Serial over versions; each version is fast (206-pair grid, ~seconds). # # Usage: # run_pure_eb_ptes_sweep.sh --config --cat-config \ # --pure-eb-sweep-dir \ -# --out [--blind A] [--versions "v1 v2 ..."] +# --out [--versions "v1 v2 ..."] set -euo pipefail . "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" -CONFIG=""; CATCONFIG=""; PUREEBSWEEP=""; OUT=""; BLIND="A"; VERSIONS="" +CONFIG=""; CATCONFIG=""; PUREEBSWEEP=""; OUT=""; VERSIONS="" while [ $# -gt 0 ]; do case "$1" in --config) CONFIG="$2"; shift 2;; --cat-config) CATCONFIG="$2"; shift 2;; --pure-eb-sweep-dir) PUREEBSWEEP="$2"; shift 2;; --out) OUT="$2"; shift 2;; - --blind) BLIND="$2"; shift 2;; --versions) VERSIONS="$2"; shift 2;; *) echo "unknown arg: $1" >&2; exit 2;; esac @@ -37,12 +36,12 @@ mkdir -p "$OUT" VERSIONS=$(sweep_versions "$CONFIG") for ver in $VERSIONS; do - pureeb="$PUREEBSWEEP/${ver}_${BLIND}_pure_eb_semianalytic.npz" + pureeb="$PUREEBSWEEP/${ver}_pure_eb_semianalytic.npz" [ -f "$pureeb" ] || { echo "MISSING upstream input for $ver: $pureeb" >&2; exit 1; } echo "[pure_eb_ptes_sweep] $ver" spv_python "$PSCRIPTS/calculate_pure_eb_ptes.py" \ - --version "$ver" --blind "$BLIND" \ + --version "$ver" \ --pure-eb-data "$pureeb" --n-samples 2000 --out "$OUT" - echo "[pure_eb_ptes_sweep] $ver -> $OUT/${ver}_${BLIND}_pure_eb_ptes.npz" + echo "[pure_eb_ptes_sweep] $ver -> $OUT/${ver}_pure_eb_ptes.npz" done echo "[pure_eb_ptes_sweep] done -> $OUT" diff --git a/papers/bmodes/scripts/run_pure_eb_semianalytic.sh b/papers/bmodes/scripts/run_pure_eb_semianalytic.sh index 472cd81d..1d8058e0 100644 --- a/papers/bmodes/scripts/run_pure_eb_semianalytic.sh +++ b/papers/bmodes/scripts/run_pure_eb_semianalytic.sh @@ -3,11 +3,11 @@ # # Runs the 20 independent MC chunks in parallel (each a fresh apptainer-exec # process, deterministic seed 42+chunk_id) throttled to the node core count, -# then gathers them into the per-(version,blind) semianalytic .npz. Bit-exact +# then gathers them into the per-version semianalytic .npz. Bit-exact # to the paper's scatter-gather (same per-chunk seeds/order); no nested dask. # # Usage: -# run_pure_eb_semianalytic.sh --version SP_v1.4.6.3_leak_corr --blind A \ +# run_pure_eb_semianalytic.sh --version SP_v1.4.6.3_leak_corr \ # --cat-config \ # --xi-reporting --xi-integration \ # --cov-integration \ @@ -16,7 +16,7 @@ set -euo pipefail . "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" -VERSION=""; BLIND="A"; CATCONFIG=""; XIREP=""; XIINT=""; COVINT=""; OUT="" +VERSION=""; CATCONFIG=""; XIREP=""; XIINT=""; COVINT=""; OUT="" NCHUNKS=20; NSAMPLES=2000; NPROC="${SLURM_CPUS_PER_TASK:-16}" MINSEP=1.0; MAXSEP=250.0; NBINS=20 MINSEPINT=0.5; MAXSEPINT=300.0; NBINSINT=1000; NPATCH=1 @@ -24,7 +24,6 @@ MINSEPINT=0.5; MAXSEPINT=300.0; NBINSINT=1000; NPATCH=1 while [ $# -gt 0 ]; do case "$1" in --version) VERSION="$2"; shift 2;; - --blind) BLIND="$2"; shift 2;; --cat-config) CATCONFIG="$2"; shift 2;; --xi-reporting) XIREP="$2"; shift 2;; --xi-integration) XIINT="$2"; shift 2;; @@ -46,13 +45,13 @@ done mkdir -p "$OUT/chunks" -echo "[pure_eb] $NCHUNKS chunks, $NSAMPLES samples, nproc=$NPROC, version=$VERSION blind=$BLIND" +echo "[pure_eb] $NCHUNKS chunks, $NSAMPLES samples, nproc=$NPROC, version=$VERSION" for i in $(seq 0 $((NCHUNKS-1))); do ( SPV_EXEC_EXTRA=$SINGLE_THREAD_ENV spv_python "$PSCRIPTS/precompute_pure_eb_chunk.py" \ --chunk-id "$i" --n-chunks "$NCHUNKS" --n-samples "$NSAMPLES" \ - --version "$VERSION" --blind "$BLIND" --cat-config "$CATCONFIG" \ + --version "$VERSION" --cat-config "$CATCONFIG" \ --xi-reporting "$XIREP" --xi-integration "$XIINT" --cov-integration "$COVINT" \ --min-sep "$MINSEP" --max-sep "$MAXSEP" --nbins "$NBINS" \ --min-sep-int "$MINSEPINT" --max-sep-int "$MAXSEPINT" --nbins-int "$NBINSINT" \ @@ -71,7 +70,7 @@ done echo "[pure_eb] all $NCHUNKS chunks done; gathering" spv_python "$PSCRIPTS/gather_pure_eb_chunks.py" \ - --version "$VERSION" --blind "$BLIND" \ + --version "$VERSION" \ --xi-reporting "$XIREP" --xi-integration "$XIINT" \ --chunks-dir "$OUT/chunks" \ --min-sep "$MINSEP" --max-sep "$MAXSEP" --nbins "$NBINS" \ diff --git a/papers/bmodes/scripts/run_pure_eb_sweep.sh b/papers/bmodes/scripts/run_pure_eb_sweep.sh index 8195f150..8b808c53 100755 --- a/papers/bmodes/scripts/run_pure_eb_sweep.sh +++ b/papers/bmodes/scripts/run_pure_eb_sweep.sh @@ -8,7 +8,7 @@ # integration covariance from the cov_sweep output dir (both by absolute path — # lc does not wire cross-output deps, so the driver reads upstream sweeps # directly; run xi_sweep + cov_sweep before this). The gathered -# ``{ver}_{blind}_pure_eb_semianalytic.npz`` — already the canonical name +# ``{ver}_pure_eb_semianalytic.npz`` — already the canonical name # pure_eb_version_comparison (--results-dir) reads — is moved into --out; the # per-version MC chunks stage in a scratch subdir that is removed after gather. # @@ -16,12 +16,12 @@ # run_pure_eb_sweep.sh --config --cat-config \ # --xi-sweep-dir \ # --cov-sweep-dir \ -# --out [--blind A] [--versions "v1 v2 ..."] +# --out [--versions "v1 v2 ..."] set -euo pipefail . "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" -CONFIG=""; CATCONFIG=""; XISWEEP=""; COVSWEEP=""; OUT=""; BLIND="A"; VERSIONS="" +CONFIG=""; CATCONFIG=""; XISWEEP=""; COVSWEEP=""; OUT=""; VERSIONS="" while [ $# -gt 0 ]; do case "$1" in --config) CONFIG="$2"; shift 2;; @@ -29,7 +29,6 @@ while [ $# -gt 0 ]; do --xi-sweep-dir) XISWEEP="$2"; shift 2;; --cov-sweep-dir) COVSWEEP="$2"; shift 2;; --out) OUT="$2"; shift 2;; - --blind) BLIND="$2"; shift 2;; --versions) VERSIONS="$2"; shift 2;; *) echo "unknown arg: $1" >&2; exit 2;; esac @@ -42,7 +41,7 @@ VERSIONS=$(sweep_versions "$CONFIG") for ver in $VERSIONS; do xirep="$XISWEEP/${ver}_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.txt" xiint="$XISWEEP/${ver}_xi_minsep=0.5_maxsep=300.0_nbins=1000_npatch=1.txt" - covbase="covariance_${ver}_${BLIND}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" + covbase="covariance_${ver}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" covint="$COVSWEEP/$covbase/${covbase}_processed.txt" for f in "$xirep" "$xiint" "$covint"; do [ -f "$f" ] || { echo "MISSING upstream input for $ver: $f" >&2; exit 1; } @@ -50,12 +49,12 @@ for ver in $VERSIONS; do echo "[pure_eb_sweep] $ver" stage="$OUT/_stage_${ver}" bash "$PSCRIPTS/run_pure_eb_semianalytic.sh" \ - --version "$ver" --blind "$BLIND" --cat-config "$CATCONFIG" \ + --version "$ver" --cat-config "$CATCONFIG" \ --xi-reporting "$xirep" --xi-integration "$xiint" --cov-integration "$covint" \ --out "$stage" - mv "$stage/${ver}_${BLIND}_pure_eb_semianalytic.npz" \ - "$OUT/${ver}_${BLIND}_pure_eb_semianalytic.npz" + mv "$stage/${ver}_pure_eb_semianalytic.npz" \ + "$OUT/${ver}_pure_eb_semianalytic.npz" rm -rf "$stage" - echo "[pure_eb_sweep] $ver -> $OUT/${ver}_${BLIND}_pure_eb_semianalytic.npz" + echo "[pure_eb_sweep] $ver -> $OUT/${ver}_pure_eb_semianalytic.npz" done echo "[pure_eb_sweep] done -> $OUT" diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 5d8416c6..8501bfbc 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -94,7 +94,6 @@ cosmo_val: # --------------------------------------------------------------------------- fiducial: version: SP_v1.4.6.3_leak_corr - blind: A npatch: 1 min_sep: 1.0 max_sep: 250.0 @@ -124,7 +123,6 @@ cl: # stamps into the filename, reconstructed by the inference consumer. harmonic: fiducial: - blind: A binning: powspace nbins: 32 diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 1ea75134..866e0cad 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -1,7 +1,6 @@ # %% import copy import os -import re from pathlib import Path import colorama @@ -256,7 +255,6 @@ def __init__( cell_seed=8192, path_onecovariance=None, cosmo_params=None, - blind=None, ): self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method @@ -285,7 +283,6 @@ def __init__( self.fiducial_input_inka = fiducial_input_inka self.nside_mask = nside_mask self.path_onecovariance = path_onecovariance - self.blind = blind assert self.cell_method in ["map", "catalog"], ( "cell_method must be 'map' or 'catalog'" @@ -443,20 +440,8 @@ def get_redshift(self, version): Redshift values nz : ndarray n(z) probability density - - Notes - ----- - If self.blind is set, the redshift path is modified to use the - specified blind (A, B, or C) by replacing the blind suffix in the - configured path. """ - redshift_path = self.cc[version]["shear"]["redshift_path"] - - # Override blind if specified - if self.blind is not None: - redshift_path = re.sub(r"_[ABC]\.txt$", f"_{self.blind}.txt", redshift_path) - - return np.loadtxt(redshift_path, unpack=True) + return np.loadtxt(self.cc[version]["shear"]["redshift_path"], unpack=True) def _write_catalog_config(self): with self.catalog_config_path.open("w") as file: diff --git a/src/sp_validation/tests/test_cv_init_params.py b/src/sp_validation/tests/test_cv_init_params.py index 240f9213..ffc87d8b 100644 --- a/src/sp_validation/tests/test_cv_init_params.py +++ b/src/sp_validation/tests/test_cv_init_params.py @@ -16,9 +16,7 @@ REPO = Path(__file__).resolve().parents[3] -EXEMPT = { - "blind": "None keeps the n(z) blind declared in the catalogue config", -} +EXEMPT = {} def _load_common(): diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index fd39c8e2..b5aa5e8a 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -597,7 +597,7 @@ def test_calculate_pseudo_cl_out_path_born_at_declared_name(cv): untagged native name — so the tagged and diagnostic rules stay disjoint.""" ver = cv._test_version cv._pseudo_cls = {} - tagged = cv._output_path(f"pseudo_cl_{ver}_blind=A_powspace_nbins=32.sacc") + tagged = cv._output_path(f"pseudo_cl_{ver}_powspace_nbins=32.sacc") native = cv._output_path(f"pseudo_cl_{ver}.sacc") cv.calculate_pseudo_cl(out_path=tagged) diff --git a/workflow/common.py b/workflow/common.py index b4844efe..d1b50abd 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -54,10 +54,6 @@ # The catalogue config of the launched checkout: the one file both the host # (CATALOG_CONFIG, loaded in configure) and every job read catalogues from. CAT_CONFIG = str(REPO_ROOT / "cosmo_val" / "cat_config.yaml") -# "blind" is the glass-mock A/B/C realisation convention, NOT Smokescreen -# blinding (a separate axis: the concealed=True SACC stamp). The name is baked -# into on-disk filenames we do not own (e.g. nz_{version}_{A|B|C}.txt). -BLINDS = ["A", "B", "C"] BLOCK_PAIRS = [("++", "1"), ("--", "2"), ("+-", "3")] # Fiducial cosmology: Planck 2018 (astropy Planck18, Table 2 + BAO) @@ -70,8 +66,7 @@ # Patterns must match all expected values; overly restrictive patterns cause # silent failures. Apply with: wildcard_constraints: **WILDCARD_CONSTRAINTS WILDCARD_CONSTRAINTS = { - "version": r"SP_v[\d.]+(_w_iv)?(_ecut\d+)?(_leak_corr)?", - "blind": r"[ABC]", + "version": r"SP_v[\d.]+(_[ABC])?(_w_iv)?(_ecut\d+)?(_leak_corr)?", "nbins": r"\d+", "min_sep": r"[0-9.]+", "max_sep": r"[0-9.]+", @@ -232,20 +227,13 @@ def fiducial_binning_suffix(fiducial=None): ) -def resolve_covariance_version(version): - """Map version to its covariance version.""" - return version - - def covariance_base( version, - blind, gaussian="ng", min_sep=None, max_sep=None, nbins=None, mask_suffix=None, - resolve_version=True, fiducial=None, default_mask_suffix=None, ): @@ -261,59 +249,31 @@ def covariance_base( DEFAULT_MASK_SUFFIX if default_mask_suffix is None else default_mask_suffix ) ) - cov_version = resolve_covariance_version(version) if resolve_version else version return ( - f"covariance_{cov_version}_{blind}_{gaussian}" + f"covariance_{version}_{gaussian}" f"_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}" ) def covariance_dir( - version, - blind, - gaussian="ng", - min_sep=None, - max_sep=None, - nbins=None, - mask_suffix=None, - resolve_version=True, + version, gaussian="ng", min_sep=None, max_sep=None, nbins=None, mask_suffix=None ): """Construct covariance directory path.""" - base = covariance_base( - version, - blind, - gaussian, - min_sep, - max_sep, - nbins, - mask_suffix, - resolve_version=resolve_version, - ) + base = covariance_base(version, gaussian, min_sep, max_sep, nbins, mask_suffix) return str(COSMO_INFERENCE / f"data/covariance/{base}") def covariance_path( version, - blind, gaussian="ng", min_sep=None, max_sep=None, nbins=None, mask_suffix=None, suffix="_processed.txt", - resolve_version=True, ): """Construct covariance file path.""" - base = covariance_base( - version, - blind, - gaussian, - min_sep, - max_sep, - nbins, - mask_suffix, - resolve_version=resolve_version, - ) + base = covariance_base(version, gaussian, min_sep, max_sep, nbins, mask_suffix) return str(COSMO_INFERENCE / f"data/covariance/{base}/{base}{suffix}") @@ -326,13 +286,19 @@ def base_version(version): return re.sub(r"_ecut\d+", "", re.sub(r"_leak_corr$", "", version)) -def build_redshift_path(version, blind): - """Construct n(z) filepath for given catalog version and blind.""" - base = base_version(version) - if "v1.4.11" in base: - base = "SP_v1.4.6" - version_dir = base.replace("SP_", "") - return f"/n17data/sguerrini/UNIONS/WL/nz/{version_dir}/nz_{base}_{blind}.txt" +def catalogue_entry(version): + """The catalogue-config entry describing ``version`` (its own, or the one + its ``_leak_corr`` variant derives from).""" + if version in CATALOG_CONFIG: + return CATALOG_CONFIG[version] + return CATALOG_CONFIG[re.sub(r"_leak_corr$", "", version)] + + +def redshift_path(version): + """The n(z) file of ``version``: its catalogue entry's ``shear.redshift_path``, + relative to the entry's ``subdir`` unless absolute.""" + entry = catalogue_entry(version) + return os.path.join(entry.get("subdir", ""), entry["shear"]["redshift_path"]) # --------------------------------------------------------------------------- @@ -409,7 +375,7 @@ def grid_of(grids, binning): def pseudo_cl_tag(config): """Fiducial harmonic-binning tag stamped into pseudo-Cl filenames.""" fiducial = config["harmonic"]["fiducial"] - return f"blind={fiducial['blind']}_{fiducial['binning']}_nbins={fiducial['nbins']}" + return f"{fiducial['binning']}_nbins={fiducial['nbins']}" def get_shear_catalog(wildcards): diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 34735a85..1bff5b43 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -106,7 +106,6 @@ def _grid_cov(version, grid, gaussian): binning = XI_GRIDS[grid] return covariance_path( version, - FIDUCIAL["blind"], gaussian=gaussian, min_sep=binning["min_sep"], max_sep=binning["max_sep"], diff --git a/workflow/rules/covariance.smk b/workflow/rules/covariance.smk index 9c26b775..6ca907f9 100644 --- a/workflow/rules/covariance.smk +++ b/workflow/rules/covariance.smk @@ -2,11 +2,8 @@ def get_cat_params(version): - """Extract covariance parameters (area, n_e, sigma_e) from catalog config.""" - base_version = version.replace("_leak_corr", "") - if base_version not in CATALOG_CONFIG: - raise KeyError(f"Catalog configuration not found for {base_version}") - cov_th = CATALOG_CONFIG[base_version]["cov_th"] + """Covariance parameters (area, n_e, sigma_e) of ``version``'s catalogue entry.""" + cov_th = catalogue_entry(version)["cov_th"] return cov_th["A"], cov_th["n_e"], cov_th["sigma_e"] @@ -20,16 +17,14 @@ MASK_CLS_FILES = { "footprint_starhalo": f"{MASK_CLS_BASE}/mask_cls_footprint_starhalo_nside_4096_norm.txt", } -# v1.4.8 uses the star-halo footprint; all other versions use the standard footprint -STARHALO_VERSIONS = {"v1.4.8"} +# SP_v1.4.8 uses the star-halo footprint; every other catalogue the standard one. +STARHALO_CATALOGUES = {"SP_v1.4.8"} def get_mask_cls_path(version): """Return absolute mask Cl path for the requested catalog version.""" - version_dir = version.replace('_leak_corr', '').replace('SP_', '') - version_dir = re.sub(r'_ecut\d+', '', version_dir) - key = "footprint_starhalo" if version_dir in STARHALO_VERSIONS else "footprint" - return MASK_CLS_FILES[key] + starhalo = base_version(version) in STARHALO_CATALOGUES + return MASK_CLS_FILES["footprint_starhalo" if starhalo else "footprint"] rule cosmology_params: @@ -59,14 +54,13 @@ with open('{output}', 'w') as f: rule covariance_ini: input: - nz_file=lambda w: build_redshift_path(w.version, w.blind), + nz_file=lambda w: redshift_path(w.version), mask=lambda w: [] if w.mask_suffix != "_masked" else [get_mask_cls_path(w.version)], output: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.ini") + str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.ini") params: outdir=lambda w: covariance_dir( - w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, - resolve_version=False + w.version, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix ), ng_value=lambda wildcards: "1" if wildcards.gaussian == "ng" else "0", omega_m=PLANCK18["Omega_m"], @@ -153,16 +147,15 @@ rule covariance_cosmocov: input: rules.covariance_ini.output, output: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/cov_tmp_ssss_{block_pm}_cov_Ntheta{nbins}_Ntomo1_{block_i}") + str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/cov_tmp_ssss_{block_pm}_cov_Ntheta{nbins}_Ntomo1_{block_i}") params: block_i="{block_i}", outdir=lambda w: covariance_dir( - w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, - resolve_version=False + w.version, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix ), ini_path=lambda w: covariance_path( - w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, - suffix=".ini", resolve_version=False + w.version, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, + suffix=".ini" ), cosmocov=config["tools"]["cosmocov_executable"], container: @@ -184,13 +177,13 @@ rule covariance_cosmocov: rule covariance_cat: input: cov_block=lambda w: [ - f"{covariance_dir(w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, resolve_version=False)}" + f"{covariance_dir(w.version, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix)}" f"/cov_tmp_ssss_{pm}_cov_Ntheta{w.nbins}_Ntomo1_{idx}" for pm, idx in BLOCK_PAIRS ], threads: 1 output: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.txt") + str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.txt") shell: """ cat {input} > {output} @@ -243,24 +236,24 @@ rule generate_glass_mock_rhotau_samples: rule covariance_process: """Post-process a raw CosmoCov matrix into the analysis-ready form.""" input: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.txt") + str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.txt") output: - matrix=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed.txt"), - gaussian=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_g.txt"), - plot=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_plot.pdf") + matrix=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed.txt"), + gaussian=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_g.txt"), + plot=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_plot.pdf") threads: 1 script: "../scripts/cosmocov_process.py" def fiducial_covariance_outputs(mask_suffix=""): - """Return processed covariance files for fiducial version/blind.""" + """Return processed covariance files for the fiducial version.""" ng_path = covariance_path( - FIDUCIAL["version"], FIDUCIAL["blind"], "ng", + FIDUCIAL["version"], "ng", FIDUCIAL["min_sep"], FIDUCIAL["max_sep"], FIDUCIAL["nbins"], mask_suffix ) g_path = covariance_path( - FIDUCIAL["version"], FIDUCIAL["blind"], "g", + FIDUCIAL["version"], "g", FIDUCIAL["min_sep_int"], FIDUCIAL["max_sep_int"], FIDUCIAL["nbins_int"], mask_suffix ) return [ng_path, g_path] diff --git a/workflow/rules/glass_mock.smk b/workflow/rules/glass_mock.smk index 243076d6..090e3b55 100644 --- a/workflow/rules/glass_mock.smk +++ b/workflow/rules/glass_mock.smk @@ -118,11 +118,7 @@ rule mock_cosebis_bias_test: mock_id=GLASS_MOCK_IDS, ), xi_ref=f"{MOCK_RESULTS}/gg_glass_mock_00001_nbins=1000.fits", - cov=str( - COSMO_INFERENCE / "data/covariance" - / "covariance_SP_v1.4.6_leak_corr_A_g_minsep=0.5_maxsep=500.0_nbins=1000_masked" - / "covariance_SP_v1.4.6_leak_corr_A_g_minsep=0.5_maxsep=500.0_nbins=1000_masked_processed.txt" - ), + cov=covariance_path("SP_v1.4.6_leak_corr", "g", 0.5, 500.0, 1000, "_masked"), params: nmodes=config["fiducial"]["nmodes"], theta_min=config["cosebis"]["theta_min"], diff --git a/workflow/rules/inference.smk b/workflow/rules/inference.smk index d2847976..f707e895 100644 --- a/workflow/rules/inference.smk +++ b/workflow/rules/inference.smk @@ -1,4 +1,4 @@ -# Imports from Snakefile: FIDUCIAL, COSMO_INFERENCE, COSMO_VAL, covariance_path, build_redshift_path, fiducial_binning_suffix +# Imports from Snakefile: FIDUCIAL, COSMO_INFERENCE, COSMO_VAL, covariance_path, redshift_path, fiducial_binning_suffix, pseudo_cl_tag # NOTE: dormant subsystem. The file-name plumbing (config-driven paths + the # producer-tagged pseudo-Cl names) is fixed and the DAG is valid, but it has not # been run end-to-end. Reviving it still needs the FITS-CONTENT plumbing @@ -34,14 +34,8 @@ GLASS_MOCK_CONFIG_PATTERN = str( # Fiducial harmonic-binning tag the pseudo-Cl producer (twopoint.smk) stamps # into the filename. These are NOT inference_prep wildcards, so the consumer -# reads them from config to reconstruct the exact name the producer emits -# (canonical: blind=A, powspace, nbins=32 — see twopoint.smk pseudo_cl_all). -HARMONIC_FIDUCIAL = config["harmonic"]["fiducial"] -PSEUDO_CL_TAG = ( - f"blind={HARMONIC_FIDUCIAL['blind']}" - f"_{HARMONIC_FIDUCIAL['binning']}" - f"_nbins={HARMONIC_FIDUCIAL['nbins']}" -) +# reads them from config to reconstruct the exact name the producer emits. +PSEUDO_CL_TAG = pseudo_cl_tag(config) def pseudo_cl_assets(version): @@ -58,12 +52,12 @@ def pseudo_cl_assets(version): rule inference_prep: input: # Processed covariance matrix - use centralized covariance_path() - cov_matrix=lambda w: covariance_path(w.version, w.blind, min_sep=w.min_sep, max_sep=w.max_sep, nbins=w.nbins), + cov_matrix=lambda w: covariance_path(w.version, min_sep=w.min_sep, max_sep=w.max_sep, nbins=w.nbins), # Xi FITS files xi_plus=str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), xi_minus=str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - # n(z) file (using new location with base version mapping) - nz_file=lambda w: build_redshift_path(w.version, w.blind), + # n(z) file: the catalogue entry's + nz_file=lambda w: redshift_path(w.version), # rho/tau stats rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), @@ -74,15 +68,15 @@ rule inference_prep: output: fits_file=str( COSMO_INFERENCE_PROD - / "data/{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}/cosmosis_{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits" + / "data/{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}/cosmosis_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits" ), config_file=str( COSMO_INFERENCE_PROD - / "cosmosis_config/output/cosmosis_pipeline_{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.ini" + / "cosmosis_config/output/cosmosis_pipeline_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.ini" ) params: - cosmosis_root="{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}", - data_dir=f"{CHAINS_DIR}/{{version}}_{{blind}}_minsep={{min_sep}}_maxsep={{max_sep}}_nbins={{nbins}}_npatch={{npatch}}", + cosmosis_root="{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}", + data_dir=f"{CHAINS_DIR}/{{version}}_minsep={{min_sep}}_maxsep={{max_sep}}_nbins={{nbins}}_npatch={{npatch}}", output_root=str(COSMO_INFERENCE_PROD), threads: 1 resources: @@ -113,12 +107,12 @@ rule inference_fiducial: input: # Use the same output patterns as inference_prep with FIDUCIAL params rules.inference_prep.output.fits_file.format( - version=FIDUCIAL["version"], blind=FIDUCIAL["blind"], + version=FIDUCIAL["version"], min_sep=FIDUCIAL["min_sep"], max_sep=FIDUCIAL["max_sep"], nbins=FIDUCIAL["nbins"], npatch=FIDUCIAL["npatch"] ), rules.inference_prep.output.config_file.format( - version=FIDUCIAL["version"], blind=FIDUCIAL["blind"], + version=FIDUCIAL["version"], min_sep=FIDUCIAL["min_sep"], max_sep=FIDUCIAL["max_sep"], nbins=FIDUCIAL["nbins"], npatch=FIDUCIAL["npatch"] ) @@ -133,9 +127,9 @@ rule inference_prep_glass_mock: input: xi=f"{GLASS_MOCK_DATA_DIR}/xi_glass_mock_{{mock_id}}_4096_nbins=20.fits", # Use centralized covariance_path() with fiducial mock version - cov_matrix=covariance_path(FIDUCIAL["mock_version"], "A"), + cov_matrix=covariance_path(FIDUCIAL["mock_version"]), # n(z) file - nz_file=build_redshift_path(FIDUCIAL["mock_version"], "A"), + nz_file=redshift_path(FIDUCIAL["mock_version"]), # Rho/tau stats: rho from real data, tau sampled rho_stats=str(COSMO_VAL / f"rho_tau_stats/rho_stats_{FIDUCIAL['mock_version']}{fiducial_binning_suffix()}.fits"), tau_stats="results/glass_mock_rhotau_samples/{mock_id}/tau_stats_sampled.fits", diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 4e8b75f6..0e3f90c1 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -68,7 +68,6 @@ rule rho_tau_stats: # Pseudo-Cl generation for harmonic-space data vectors and COSEBIS validation. -BASE_VERSIONS = [v.replace("_leak_corr", "") for v in config["versions"]] wildcard_constraints: binning="linear|logspace|powspace", @@ -77,12 +76,9 @@ wildcard_constraints: rule pseudo_cl: """Generate pseudo-Cl data vector (born as SACC) with configurable binning.""" output: - pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.sacc"), - wildcard_constraints: - blind="[ABC]", + pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_{binning}_nbins={nbins}.sacc"), params: version="{version}", - blind="{blind}", cat_config=CAT_CONFIG, nside=1024, npatch=1, @@ -101,12 +97,9 @@ rule pseudo_cl: rule pseudo_cl_cov: """Generate pseudo-Cl covariance with configurable binning.""" output: - pseudo_cl_cov=str(COSMO_VAL / "pseudo_cl_cov_{version}_blind={blind}_{binning}_nbins={nbins}.fits"), - wildcard_constraints: - blind="[ABC]", + pseudo_cl_cov=str(COSMO_VAL / "pseudo_cl_cov_{version}_{binning}_nbins={nbins}.fits"), params: version="{version}", - blind="{blind}", cat_config=CAT_CONFIG, nside=1024, npatch=1, @@ -129,7 +122,7 @@ rule pseudo_cl_all: """Generate pseudo-Cls for all versions.""" input: expand( - str(COSMO_VAL / "pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc"), + str(COSMO_VAL / "pseudo_cl_{version}_powspace_nbins=32.sacc"), version=PSEUDO_CL_VERSIONS, ), @@ -138,7 +131,7 @@ rule pseudo_cl_cov_all: """Generate pseudo-Cl covariances for all versions.""" input: expand( - str(COSMO_VAL / "pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits"), + str(COSMO_VAL / "pseudo_cl_cov_{version}_powspace_nbins=32.fits"), version=PSEUDO_CL_VERSIONS, ), @@ -147,7 +140,6 @@ rule pseudo_cl_fine_all: """Generate fine pseudo-Cls for COSEBIS.""" input: expand( - str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_linear_nbins=2040.sacc"), + str(COSMO_VAL / "pseudo_cl_{version}_linear_nbins=2040.sacc"), version=config["versions"], - blind=BLINDS, ), diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index e25f6ed9..e1b96da6 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -34,7 +34,6 @@ def generate_pseudo_cl( cat_config: str, nside: int = 1024, npatch: int = 1, - blind: str = None, cosmo_params: dict = None, binning: str = "linear", nbins: int = None, @@ -55,8 +54,6 @@ def generate_pseudo_cl( HEALPix nside for map-based estimation npatch : int Number of jackknife patches - blind : str, optional - Blind identifier (A, B, or C) to override n(z) path cosmo_params : dict, optional Cosmological parameters. Keys: Omega_m, sigma_8, n_s, h, Omega_b. If None, uses Planck 2018 defaults. @@ -75,7 +72,6 @@ def generate_pseudo_cl( output_dir = os.path.dirname(out_path) os.makedirs(output_dir, exist_ok=True) - blind_str = f" blind={blind}" if blind else "" if binning == "linear": # For linear binning, nbins determines ell_step such that we cover 2-2048 ell_step = max(1, (2048 - 2) // nbins) @@ -86,7 +82,7 @@ def generate_pseudo_cl( bin_str = f"nbins={nbins}, power={power}" print(f"\n{'=' * 60}") - print(f"Generating pseudo-Cl for {version}{blind_str}") + print(f"Generating pseudo-Cl for {version}") print(f"Binning: {binning} ({bin_str})") if cosmo_params: print( @@ -115,7 +111,6 @@ def generate_pseudo_cl( nside=nside, cell_method="catalog", nrandom_cell=100, - blind=blind, cosmo_params=cosmo_params, npatch=npatch, theta_min=1.0, @@ -150,7 +145,6 @@ def _from_snakemake(smk): cat_config=p["cat_config"], nside=int(p["nside"]), npatch=int(p["npatch"]), - blind=p.get("blind", None), cosmo_params=p.get("cosmo_params", None), binning=p["binning"], nbins=int(p["nbins"]), @@ -193,9 +187,6 @@ def _from_cli(argv=None): default=0.5, help="Power for powspace binning (0.5 = sqrt spacing)", ) - ap.add_argument( - "--blind", choices=["A", "B", "C"], default=None, help="Blind identifier" - ) ap.add_argument( "--cosmo-json", default=None, @@ -217,7 +208,6 @@ def _from_cli(argv=None): cat_config=a.cat_config, nside=a.nside, npatch=a.npatch, - blind=a.blind, cosmo_params=cosmo_params, binning=a.binning, nbins=a.nbins, diff --git a/workflow/scripts/generate_pseudo_cl_cov.py b/workflow/scripts/generate_pseudo_cl_cov.py index 08f2cf07..247c5f33 100644 --- a/workflow/scripts/generate_pseudo_cl_cov.py +++ b/workflow/scripts/generate_pseudo_cl_cov.py @@ -39,7 +39,6 @@ def generate_pseudo_cl_cov( cat_config: str, nside: int = 1024, npatch: int = 1, - blind: str = None, cosmo_params: dict = None, binning: str = "powspace", nbins: int = 32, @@ -61,8 +60,6 @@ def generate_pseudo_cl_cov( HEALPix nside for map-based estimation npatch : int Number of jackknife patches - blind : str, optional - Blind identifier (A, B, or C) to override n(z) path cosmo_params : dict, optional Cosmological parameters. Keys: Omega_m, sigma_8, n_s, h, Omega_b. If None, uses Planck 2018 defaults. @@ -80,7 +77,6 @@ def generate_pseudo_cl_cov( """ os.makedirs(output_dir, exist_ok=True) - blind_str = f" blind={blind}" if blind else "" if binning == "linear": ell_step = max(1, (2048 - 2) // nbins) bin_str = f"nbins={nbins} (ell_step={ell_step})" @@ -90,7 +86,7 @@ def generate_pseudo_cl_cov( bin_str = f"nbins={nbins}, power={power}" print(f"\n{'=' * 60}") - print(f"Generating pseudo-Cl covariance for {version}{blind_str}") + print(f"Generating pseudo-Cl covariance for {version}") print(f"Binning: {binning} ({bin_str})") if cosmo_params: print( @@ -119,7 +115,6 @@ def generate_pseudo_cl_cov( nside=nside, cell_method="catalog", nrandom_cell=100, - blind=blind, cosmo_params=cosmo_params, npatch=npatch, theta_min=1.0, @@ -158,7 +153,6 @@ def _from_snakemake(smk): cat_config=p["cat_config"], nside=int(p["nside"]), npatch=int(p["npatch"]), - blind=p.get("blind", None), cosmo_params=p.get("cosmo_params", None), binning=p["binning"], nbins=int(p["nbins"]), @@ -205,9 +199,6 @@ def _from_cli(argv=None): default=0.5, help="Power for powspace binning (0.5 = sqrt spacing)", ) - ap.add_argument( - "--blind", choices=["A", "B", "C"], default=None, help="Blind identifier" - ) ap.add_argument( "--cosmo-json", default=None, @@ -226,7 +217,6 @@ def _from_cli(argv=None): cat_config=a.cat_config, nside=a.nside, npatch=a.npatch, - blind=a.blind, cosmo_params=cosmo_params, binning=a.binning, nbins=a.nbins, diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py index 29ebf5c4..19153bae 100644 --- a/workflow/tests/conftest.py +++ b/workflow/tests/conftest.py @@ -216,7 +216,6 @@ def toy(tmp_path_factory): path = covariances[version, gaussian] = Path( common.covariance_path( version, - config["fiducial"]["blind"], gaussian, grid["min_sep"], grid["max_sep"], From 64b73ec47259fb646115f15fe333f31e987499d1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 05:05:21 +0200 Subject: [PATCH 127/160] cat_config: the leakage n(z) is a path template, not an A/B/C blind key nz.dndz.path names the file with {pipeline}; the same file as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- cosmo_val/cat_config.yaml | 4 ++-- src/sp_validation/cosmo_val/psf_systematics.py | 4 ++-- src/sp_validation/tests/test_config_paths_exist.py | 11 +---------- src/sp_validation/tests/test_cosmo_val.py | 4 ++-- src/sp_validation/tests/test_pseudo_cl.py | 2 +- 5 files changed, 8 insertions(+), 17 deletions(-) diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index edbc4e85..6914bdef 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -1156,9 +1156,9 @@ SP_v1.4.8_uncal: patch_number: 100 nz: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/nz + # The n(z) of the PSF-leakage theory, by pipeline. dndz: - blind: A - path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz + path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_{pipeline}_A.txt paths: output: ./output SP_v1.6.6: diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index f8002244..ad405714 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -313,8 +313,8 @@ def set_params_leakage_scale(self, ver): # Set parameters params_in["input_path_shear"] = self.cc[ver]["shear"]["path"] params_in["input_path_PSF"] = self.cc[ver]["star"]["path"] - params_in["dndz_path"] = ( - f"{self.cc['nz']['dndz']['path']}_{self.cc[ver]['pipeline']}_{self.cc['nz']['dndz']['blind']}.txt" + params_in["dndz_path"] = self.cc["nz"]["dndz"]["path"].format( + pipeline=self.cc[ver]["pipeline"] ) params_in["output_dir"] = f"{self.cc['paths']['output']}/leakage_{ver}" diff --git a/src/sp_validation/tests/test_config_paths_exist.py b/src/sp_validation/tests/test_config_paths_exist.py index 38a269ec..a3de6873 100644 --- a/src/sp_validation/tests/test_config_paths_exist.py +++ b/src/sp_validation/tests/test_config_paths_exist.py @@ -31,7 +31,6 @@ # files to stat. (The overlay's one real path, ``base:``, is deliberately not # listed, so it is still validated.) NON_PATH_KEYS = ("extra_output", "why", "replace", "with", "drop") -PATH_PREFIX_KEYS = ("nz.dndz.path",) TEXT_SUFFIXES = ( ".fits", ".fits.gz", @@ -78,10 +77,6 @@ def _non_path_key(key: str) -> bool: return key.lower().endswith(NON_PATH_KEYS) -def _path_prefix_key(key: str) -> bool: - return key.lower() in PATH_PREFIX_KEYS - - def _pathish_value(value: str) -> bool: if not value or any(part in value for part in SKIP_VALUE_PARTS): return False @@ -108,11 +103,7 @@ def _walk_yaml( yield from _walk_yaml(child, trail + (str(index),), base_dir) elif isinstance(value, str): key = ".".join(trail) - if ( - not _non_path_key(key) - and not _path_prefix_key(key) - and (_pathish_key(key) or _pathish_value(value)) - ): + if not _non_path_key(key) and (_pathish_key(key) or _pathish_value(value)): yield key, value, base_dir diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 813cf747..653976d5 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -73,7 +73,7 @@ def _make_seed_config(tmp_path, shear_filename): config_data = { "nz": { "subdir": str(nz_dir), - "dndz": {"blind": "A", "path": "dndz.txt"}, + "dndz": {"path": "dndz.txt"}, }, "paths": {"output": str(output_dir)}, base_version: { @@ -394,7 +394,7 @@ def _write_synthetic_catalogs( config_data = { "nz": { "subdir": str(nz_dir), - "dndz": {"blind": "A", "path": "dndz"}, + "dndz": {"path": "dndz_{pipeline}_A.txt"}, }, "paths": {"output": str(output_dir)}, version: version_cfg, diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index b5aa5e8a..ca6bbb1f 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -141,7 +141,7 @@ def _write_synthetic_config(tmp_path): "star_flag": "w", } config_data = { - "nz": {"subdir": str(nz_dir), "dndz": {"blind": "A", "path": "dndz"}}, + "nz": {"subdir": str(nz_dir), "dndz": {"path": "dndz_{pipeline}_A.txt"}}, "paths": {"output": str(output_dir)}, version: { "subdir": str(cat_dir), From 3f459b1eb37c4b8ca3602781097f3ce8abaa468f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 05:36:00 +0200 Subject: [PATCH 128/160] cat_config: SP_v1.4.11.2 reads the n(z) its covariance used Like SP_v1.4.11.3, its covariance was built from nz_SP_v1.4.6_A.txt. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- cosmo_val/cat_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index 6914bdef..618f3972 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -871,7 +871,7 @@ SP_v1.4.11.2: shear: R: 1.0 path: /n17data/UNIONS/WL/v1.4.x/v1.4.11.2/unions_shapepipe_cut_struc_2024_v1.4.11.2.fits - redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected From af20f6b489b0f0beeeb412b9e950c332f2211f3e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 05:47:41 +0200 Subject: [PATCH 129/160] blinding_theory: Omega_c from CCL's own massive-neutrino density CCL's total Omega_m is then exactly the blind axis, and a test says so. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- src/sp_validation/blinding_theory.py | 17 +++++++++++------ src/sp_validation/tests/test_blinding.py | 12 ++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/sp_validation/blinding_theory.py b/src/sp_validation/blinding_theory.py index 3c768035..629bd01b 100644 --- a/src/sp_validation/blinding_theory.py +++ b/src/sp_validation/blinding_theory.py @@ -91,12 +91,14 @@ def sigma8(self): def omega_c(self): """CCL cold-dark-matter density ``Omega_c = Omega_m − Omega_b − Ω_ν``. - The neutrino density ``Ω_ν h² = Σm_ν / 93.14 eV`` is subtracted so - the *total* matter density is exactly ``Omega_m`` (CCL treats massive - neutrinos as a separate species, not part of ``Omega_c``). + ``Ω_ν`` is CCL's own massive-neutrino density for this ``m_nu``, + ``mass_split``, ``h``, ``Neff`` and ``T_CMB`` (independent of + ``Omega_c``), so CCL's total ``Omega_m`` is exactly ``Omega_m``. """ - omega_nu = self.m_nu / (93.14 * self.h**2) - return self.Omega_m - self.Omega_b - omega_nu + import pyccl as ccl + + trial = ccl.Cosmology(**self._ccl_params(self.Omega_m - self.Omega_b)) + return self.Omega_m - self.Omega_b - trial["Omega_nu_mass"] def ccl_params(self): """This point as a plain CCL-native parameter mapping. @@ -105,8 +107,11 @@ def ccl_params(self): mass_split, w0, wa, Neff, T_CMB`` and no others, so no CCL default rides along; ``Neff``/``T_CMB`` are the fixed module constants. """ + return self._ccl_params(self.omega_c()) + + def _ccl_params(self, omega_c): return { - "Omega_c": self.omega_c(), + "Omega_c": omega_c, "Omega_b": self.Omega_b, "h": self.h, "n_s": self.n_s, diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 327890a6..4cfd3cb9 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -318,3 +318,15 @@ def test_the_audit_proves_the_shift(tmp_path, seed): cat_config=root / "cat_config.yaml", ) assert report["ok"] == (seed == "committed"), report + + +@pytest.mark.parametrize("S8, Omega_m", [(0.80, 0.30), (0.725, 0.20), (0.875, 0.40)]) +def test_ccl_total_matter_is_the_blind_axis(S8, Omega_m): + """CCL's Ωm and S8 at a point's parameters are the point's own.""" + ccl = pytest.importorskip("pyccl") + from sp_validation.blinding_theory import TheoryConfig + + point = TheoryConfig(S8=S8, Omega_m=Omega_m) + cosmo = ccl.Cosmology(**point.ccl_params()) + assert cosmo["Omega_m"] == pytest.approx(Omega_m, rel=1e-12) + assert cosmo["sigma8"] * np.sqrt(cosmo["Omega_m"] / 0.3) == pytest.approx(S8) From c27a504a0c18a48168526b734db3e0a9de862e98 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 05:52:17 +0200 Subject: [PATCH 130/160] workflow: no snakemake in the image, so any host Snakemake works A script: job unpickles the host's snakemake object with whichever snakemake it imports first, and the image's site-packages precede the host's appended sys.path. The image carried its own (the workflow extra, and the base image's jupyter extra), so the host had to match its version exactly. The Dockerfile now uninstalls every snakemake* package after the sync and the workflow extra drops snakemake; the job then reads the pickle with the package that wrote it. The launch check keeps only the Python minor (check_host_python): the host's snakemake and its compiled dependencies load into the image's interpreter. The README install line, CI's DAG job and the test harness no longer pin a Snakemake version. The container smoke job now reports which snakemake unpickled its object and asserts it is the host's version. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- .github/workflows/deploy-image.yml | 2 +- Dockerfile | 13 +- pyproject.toml | 4 +- src/sp_validation/container.py | 48 +-- uv.lock | 369 ------------------ workflow/README.md | 27 +- workflow/common.py | 39 +- workflow/tests/conftest.py | 15 +- .../data/container_smoke/container_smoke.py | 14 +- workflow/tests/test_container_smoke.py | 4 + workflow/tests/test_dag.py | 32 +- 11 files changed, 87 insertions(+), 480 deletions(-) diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 5867b9c2..f00619a6 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -25,7 +25,7 @@ jobs: - name: DAG tests run: >- uv run --isolated --no-project --python 3.12 - --with snakemake==9.23.1 --with snakemake-executor-plugin-slurm + --with snakemake --with snakemake-executor-plugin-slurm --with pytest --with numpy pytest workflow/tests -m "not candide" diff --git a/Dockerfile b/Dockerfile index 0437a929..c4fe4bac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,9 +72,8 @@ WORKDIR /sp_validation # our lock — instead of pruning them. Copy the lock + manifest first so this # layer caches independently of source edits. Extras: test (CI unit suite), # glass (GLASS map-level mock — pulls glass.ext.camb + the cosmology wrapper), -# workflow (Snakemake + mpi4py runners). cs_util 0.2.2 (with cs_util.size) and a -# numba-safe numpy 2.4.6 come straight from the lock, so the old ad-hoc snakemake -# and cs_util `--upgrade` layers are gone. +# workflow (mpi4py, CosmoSIS and the other rule-script runners). cs_util and a +# numba-safe numpy come straight from the lock. COPY pyproject.toml uv.lock /sp_validation/ # cosmosis builds MPI-enabled polychord/multinest only when MPIFC is set: its @@ -88,6 +87,14 @@ ENV MPIFC=/opt/ompi/bin/mpif90 RUN uv sync --frozen --inexact --no-install-project \ --extra test --extra glass --extra workflow +# No snakemake in the image. A `script:` job unpickles the host Snakemake's +# `snakemake` object with whichever snakemake it imports first, and the image's +# site-packages precede the host's; with none here, the job reads the pickle with +# the package that wrote it, so the host may run any Snakemake. The base image's +# jupyter extra brings one, which `--inexact` keeps. +RUN uv pip freeze | grep -io '^snakemake[a-z0-9_-]*' | xargs -r uv pip uninstall \ + && ! /app/.venv/bin/python -c "import snakemake" 2>/dev/null + # The CosmoSIS Standard Library: the module files (camb interface, projection, # 2pt likelihood, ...) the cosmo_inference pipelines name. The `workflow` extra # above installs cosmosis itself; CSL is a separate tree of modules that is not diff --git a/pyproject.toml b/pyproject.toml index 632793f4..37320d7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -157,9 +157,9 @@ glass = [ ] # Cosmo-inference workflow runners (workflow/scripts/*). Kept optional: the core # library resolves without them, but the container installs this extra so the -# Snakemake workflow and cross-validation runners are available. +# workflow's rule scripts and cross-validation runners are available. Snakemake +# itself is a host-side tool and stays out of the image (see the Dockerfile). workflow = [ - "snakemake", # Optional MPI runners (the container ships OpenMPI at /opt/ompi, so mpi4py # builds against it). "mpi4py", diff --git a/src/sp_validation/container.py b/src/sp_validation/container.py index ba37939a..08497ee2 100755 --- a/src/sp_validation/container.py +++ b/src/sp_validation/container.py @@ -130,37 +130,24 @@ def image_revision(sif): IMAGE_VENV = "/app/.venv" -def image_runtime(image): - """Return ``(python_minor, snakemake_version)`` of an image, or ``None``. - - Read from the venv's ``pyvenv.cfg`` and its ``snakemake-*.dist-info`` - directory: through ``apptainer exec`` for a SIF, straight off disk for a - sandbox. ``None`` when the image cannot be read -- a registry tag, a missing - ``apptainer``, a venv laid out differently. ``snakemake_version`` is - ``None`` when the image carries no snakemake. +def image_python(image): + """Return the Python minor (``"3.12"``) of an image's venv, or ``None``. + + Read from the venv's ``pyvenv.cfg``: through ``apptainer exec`` for a SIF, + straight off disk for a sandbox. ``None`` when the image cannot be read -- a + registry tag, a missing ``apptainer``, a venv laid out differently. """ image = str(image) - venv = IMAGE_VENV.lstrip("/") + cfg_path = Path(IMAGE_VENV) / "pyvenv.cfg" if Path(image).is_dir(): - root = Path(image) / venv try: - cfg = (root / "pyvenv.cfg").read_text() + cfg = (Path(image) / cfg_path.relative_to("/")).read_text() except OSError: return None - listing = [p.name for p in root.glob("lib/python*/site-packages/*")] elif Path(image).is_file() and shutil.which("apptainer") is not None: try: out = subprocess.run( - [ - "apptainer", - "exec", - "--cleanenv", - image, - "sh", - "-c", - f"cat {IMAGE_VENV}/pyvenv.cfg && " - f"ls {IMAGE_VENV}/lib/python*/site-packages", - ], + ["apptainer", "exec", "--cleanenv", image, "cat", str(cfg_path)], capture_output=True, text=True, timeout=120, @@ -169,23 +156,14 @@ def image_runtime(image): return None if out.returncode != 0: return None - cfg, listing = out.stdout, out.stdout.split() + cfg = out.stdout else: return None - fields = {} for line in cfg.splitlines(): key, sep, value = line.partition("=") - if sep: - fields[key.strip()] = value.strip() - version = fields.get("version_info") - if version is None: - return None - snakemake = [ - name[len("snakemake-") : -len(".dist-info")] - for name in listing - if name.startswith("snakemake-") and name.endswith(".dist-info") - ] - return ".".join(version.split(".")[:2]), (snakemake[0] if snakemake else None) + if sep and key.strip() == "version_info": + return ".".join(value.strip().split(".")[:2]) + return None def _require_apptainer(): diff --git a/uv.lock b/uv.lock index 1460cbc9..4acc2e41 100644 --- a/uv.lock +++ b/uv.lock @@ -94,15 +94,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/9a/4e5157d893ffc712b74dbd868c7f62365618266982b64accab26bab01edc/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99", size = 86777, upload-time = "2025-07-30T10:01:56.943Z" }, ] -[[package]] -name = "argparse-dataclass" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1a/ff/a2e4e328075ddef2ac3c9431eb12247e4ba707a70324894f1e6b4f43c286/argparse_dataclass-2.0.0.tar.gz", hash = "sha256:09ab641c914a2f12882337b9c3e5086196dbf2ee6bf0ef67895c74002cc9297f", size = 6395, upload-time = "2023-06-11T20:32:54.465Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/66/e6c0a808950ba5a4042e2fcedd577fc7401536c7db063de4d7c36be06f84/argparse_dataclass-2.0.0-py3-none-any.whl", hash = "sha256:3ffc8852a88d9d98d1364b4441a712491320afb91fb56049afd8a51d74bb52d2", size = 8762, upload-time = "2023-06-11T20:32:52.724Z" }, -] - [[package]] name = "array-api-compat" version = "1.15.0" @@ -480,33 +471,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, ] -[[package]] -name = "conda-inject" -version = "1.3.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/a8/8dc86113c65c949cc72d651461d6e4c544b3302a85ed14a5298829e6a419/conda_inject-1.3.2.tar.gz", hash = "sha256:0b8cde8c47998c118d8ff285a04977a3abcf734caf579c520fca469df1cd0aac", size = 3635, upload-time = "2024-05-27T12:20:58.873Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/4c/fc30b69fb4062aee57e3ab7ff493647c4220144908f0839c619f912045bf/conda_inject-1.3.2-py3-none-any.whl", hash = "sha256:6e641b408980c2814e3e527008c30749117909a21ff47392f07ef807da93a564", size = 4133, upload-time = "2024-05-27T12:20:57.332Z" }, -] - -[[package]] -name = "configargparse" -version = "1.7.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3f/0b/30328302903c55218ffc5199646d0e9d28348ff26c02ba77b2ffc58d294a/configargparse-1.7.5.tar.gz", hash = "sha256:e3f9a7bb6be34d66b2e3c4a2f58e3045f8dfae47b0dc039f87bcfaa0f193fb0f", size = 53548, upload-time = "2026-03-11T02:19:38.144Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/19/3ba5e1b0bcc7b91aeab6c258afd70e4907d220fed3972febe38feb40db30/configargparse-1.7.5-py3-none-any.whl", hash = "sha256:1e63fdffedf94da9cd435fc13a1cd24777e76879dd2343912c1f871d4ac8c592", size = 27692, upload-time = "2026-03-11T02:19:36.442Z" }, -] - -[[package]] -name = "connection-pool" -version = "0.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/df/c9b4e25dce00f6349fd28aadba7b6c3f7431cc8bd4308a158fbe57b6a22e/connection_pool-0.0.3.tar.gz", hash = "sha256:bf429e7aef65921c69b4ed48f3d48d3eac1383b05d2df91884705842d974d0dc", size = 3795, upload-time = "2020-09-17T02:48:28.824Z" } - [[package]] name = "contourpy" version = "1.3.3" @@ -858,15 +822,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, ] -[[package]] -name = "dpath" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/ce/e1fd64d36e4a5717bd5e6b2ad188f5eaa2e902fde871ea73a79875793fc9/dpath-2.2.0.tar.gz", hash = "sha256:34f7e630dc55ea3f219e555726f5da4b4b25f2200319c8e6902c394258dd6a3e", size = 28266, upload-time = "2024-06-12T22:08:03.686Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/d1/8952806fbf9583004ab479d8f58a9496c3d35f6b6009ddd458bdd9978eaf/dpath-2.2.0-py3-none-any.whl", hash = "sha256:b330a375ded0a0d2ed404440f6c6a715deae5313af40bbb01c8a41d891900576", size = 17618, upload-time = "2024-06-12T22:08:01.881Z" }, -] - [[package]] name = "ducc0" version = "0.41.0" @@ -1070,30 +1025,6 @@ dependencies = [ { name = "scipy" }, ] -[[package]] -name = "gitdb" -version = "4.0.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "smmap" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, -] - -[[package]] -name = "gitpython" -version = "3.1.50" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "gitdb" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, -] - [[package]] name = "glass" version = "2025.3" @@ -1138,44 +1069,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, ] -[[package]] -name = "greenlet" -version = "3.5.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e2/f1/fbbfef6af0bad0548f09bc28948ea3c275b4edb19e17fc5ca9900a6a634d/greenlet-3.5.3.tar.gz", hash = "sha256:a61efc018fd3eb317eeca31aba90ee9e7f26f22884a79b6c6ec715bf71bb62f1", size = 200270, upload-time = "2026-06-26T19:28:24.832Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/73/815dd90131c1b71ebdf53dbc7c276cafec2a1173b97559f97aba72724a87/greenlet-3.5.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efa9f765dd09f9d0cdac651ffdf631ee59ec5dc6ee7a73e0c012ba9c52fbdf5b", size = 604761, upload-time = "2026-06-26T19:07:10.114Z" }, - { url = "https://files.pythonhosted.org/packages/9f/57/079cfe76bcef36b153b25607ee91c6fcb58f17f8b23c86bbbeabe0c88d72/greenlet-3.5.3-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7faba15ac005376e02a0384504e0243be3370ce010296a44a820feb342b505ab", size = 617044, upload-time = "2026-06-26T19:10:07.25Z" }, - { url = "https://files.pythonhosted.org/packages/37/87/b4d095775a3fb1bcafbb483fc206b27ebb785724c83051447737085dc54e/greenlet-3.5.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:87142215824be6ac05e2e8e2786eec307ccbc27c36723c3881959df654af6861", size = 614244, upload-time = "2026-06-26T18:32:17.594Z" }, - { url = "https://files.pythonhosted.org/packages/8a/70/7559b609683650fa2b95b8ab84b4ab0b26556a635d19675e12aa832d826d/greenlet-3.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:215275b1b49320987352e6c1b054acca0064f965a2c66992bed9a6f7d913f149", size = 1574210, upload-time = "2026-06-26T19:09:03.077Z" }, - { url = "https://files.pythonhosted.org/packages/ae/73/be55392074c60fc37655ca40fa6022457bfbf6718e9e342a7b0b41f96dd2/greenlet-3.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6b1b0eed82364b0e32c4ea0f221452d33e6bb17ae094d9f72aed9851812747ea", size = 1638627, upload-time = "2026-06-26T18:31:44.748Z" }, - { url = "https://files.pythonhosted.org/packages/d6/fa/5401ac78021c826a25b6dde0c705e0a8f29b617509f9185a31dac15fbe1b/greenlet-3.5.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2d185dd1621757e70c3861cceffd5317ab4e7ed7eb09c82994828468527ade5", size = 607435, upload-time = "2026-06-26T19:07:11.412Z" }, - { url = "https://files.pythonhosted.org/packages/e9/76/1dc144a2e56e65d36405078ed774224375ea520a1870a6e46e08bb4ac7bf/greenlet-3.5.3-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1c514a468149bf8fbbab874188a3535cd8a48a3e353eb53a3d424296f8dbacd3", size = 619787, upload-time = "2026-06-26T19:10:08.396Z" }, - { url = "https://files.pythonhosted.org/packages/bf/87/c298cee62df1de4ad7fec32abda73526cff347fd143a6ed4ac369246668a/greenlet-3.5.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:915f887cf2682b66419b879423a2e072634aa7b7dce6f3ada4957cfced3f1e9a", size = 616786, upload-time = "2026-06-26T18:32:19.128Z" }, - { url = "https://files.pythonhosted.org/packages/9e/2e/e6f009885ed0705ccf33fe0583c117cfd03cde77e31a596dd5785a30762b/greenlet-3.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:766cfd421c13e450feb340cd472a3ed9957d438727b7b4593ad7c76c5d2b0deb", size = 1574316, upload-time = "2026-06-26T19:09:04.273Z" }, - { url = "https://files.pythonhosted.org/packages/ef/fe/43fd110b01e40da0adb7c90ac7ea744bef2d43dca00de5095fd2351c2a68/greenlet-3.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2ecda9ec22edf38fa389369eaed8c3d37c05f3c54e69f69438dbb2cc1de1458b", size = 1638614, upload-time = "2026-06-26T18:31:46.297Z" }, - { url = "https://files.pythonhosted.org/packages/82/2f/146d218299046a43d1f029fd544b3d110d0f175a09c715c7e8da4a4a345d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df0a0628d1597eb0897b62f55d1343f772405fd25f3b2a796c76874b0c2e22e8", size = 654096, upload-time = "2026-06-26T19:07:12.71Z" }, - { url = "https://files.pythonhosted.org/packages/a0/cc/04738cafb3f45fa991ea44f9de94c47dcec964f5a972300988a6751f49d9/greenlet-3.5.3-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ebd933a6adabc298bab47731a130fe6bfb888bd934eee37810f151159544540d", size = 666304, upload-time = "2026-06-26T19:10:09.503Z" }, - { url = "https://files.pythonhosted.org/packages/ce/aa/4e0dad5e605c270c784ab911c43da6adb136ccd4d81180f763ca429a723d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b9d501b40e80b70e32323c799dd9b420a5577a9601469d362ae1ffb690f3a7c", size = 663635, upload-time = "2026-06-26T18:32:20.802Z" }, - { url = "https://files.pythonhosted.org/packages/d1/50/13efdbea246fe3d3b735e191fec08fb50809f53cd2383ebe123d0809e44b/greenlet-3.5.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a1fad1d11e7d6aab184107baa8e4ece11ccba3ec9599cd7efa5ff4d70d43256a", size = 1621252, upload-time = "2026-06-26T19:09:05.647Z" }, - { url = "https://files.pythonhosted.org/packages/f7/22/c0a336ae4a1410fd5f5121098e5bfbf1865f64c5ef80b4b5412886c4a332/greenlet-3.5.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fad5aec764399f1b5cc347ad250a59660f20c8f8888ea6bae1f93b769cce1154", size = 1684824, upload-time = "2026-06-26T18:31:47.738Z" }, - { url = "https://files.pythonhosted.org/packages/00/92/715c44721abe2b4d1ae9abde4179411868a5bff312479f54e105d372f131/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19131729ae0ddc3c2e1ef85e650169b5e37ee32e400f215f78b94d7b0d567310", size = 653382, upload-time = "2026-06-26T19:07:14.209Z" }, - { url = "https://files.pythonhosted.org/packages/a0/83/37a10372a1090a6624cca8e74c12df1a36c2dc36429ed0255b7fb1aeee23/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1540dd8e5fc2a5aec40fbb98ef8e149fa47c89a4b4a1cf2575a14d3d1869d7a8", size = 659401, upload-time = "2026-06-26T19:10:10.876Z" }, - { url = "https://files.pythonhosted.org/packages/db/e2/d1509cad4207da559cc42986ecdd8fc67ad0d1bba2bf03023c467fd5e0f3/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e81fa194a1d20967877bdf9c7794db2bc99063e5be36aee710c08f04c5bb087f", size = 656969, upload-time = "2026-06-26T18:32:22.272Z" }, - { url = "https://files.pythonhosted.org/packages/86/7d/eaf70de20aadca3a5884aec58362861c64ce45e7b277f47ed026926a3b89/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:55cf4d777485d43110e47133cbba6d74a8885a87ec1227ef0267f9ee80c5aa21", size = 1617822, upload-time = "2026-06-26T19:09:06.893Z" }, - { url = "https://files.pythonhosted.org/packages/8a/f9/414d38fc400ae4350d4185eaad1827676f7cf5287b9136e0ed1cbbe20a7f/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:12a248ba75f6a9a236375f52296c498c89ff1d8badf32deb9eca7abd5853f7da", size = 1677983, upload-time = "2026-06-26T18:31:49.396Z" }, - { url = "https://files.pythonhosted.org/packages/4f/ca/69db42d447a1378043e2c8f19c09cbbd1263371505053c496b49066d3d16/greenlet-3.5.3-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78dbef602fda6d97d957eb7937f70c9ce9e9527330347f8f6b6f9e554a9e7a47", size = 659747, upload-time = "2026-06-26T19:07:15.565Z" }, - { url = "https://files.pythonhosted.org/packages/a8/0b/af7ac2ef8dd41e3da1a40dda6305c23b9a03e13ba975ec916357b50f8575/greenlet-3.5.3-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f73857adb8fee13fa56c172bd11262f888c0c648f9fea113e777bb2c7904a81", size = 670419, upload-time = "2026-06-26T19:10:12.293Z" }, - { url = "https://files.pythonhosted.org/packages/51/1e/1d51640cacbfc455dbe9f9a9f594c49e4e244f63b9971a2f4764e46cc53d/greenlet-3.5.3-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:232fec92e823addaf02d9472cf7381e24a1d046a6ced1103c5caa4c21b9dfc1d", size = 668787, upload-time = "2026-06-26T18:32:24.298Z" }, - { url = "https://files.pythonhosted.org/packages/21/66/4030d5b0b5894500023f003bb054d9bb354dfbd1e186c3a296759172f5f5/greenlet-3.5.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2421c3564da9429d5586d46ca31ebb26516b5498a802cf65c041a8e8a8980d34", size = 1626305, upload-time = "2026-06-26T19:09:08.281Z" }, - { url = "https://files.pythonhosted.org/packages/0e/50/5221371c7550108dfa3c378debc41d032aa9c78e89abb01d8011cfc93289/greenlet-3.5.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e0f0d160f0b2e558e6c75f7930967183255dc9735e5f5b8cae58ee09c9576d8b", size = 1688631, upload-time = "2026-06-26T18:31:51.278Z" }, - { url = "https://files.pythonhosted.org/packages/57/66/b3bfae3e220a9b63ea539a0eea681800c69ab1aada757eae8789f183e7ce/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:629b614d2b786e89c50440e246f33eea78f58a962d0bdbbcc809e6d13605903f", size = 657221, upload-time = "2026-06-26T19:07:16.973Z" }, - { url = "https://files.pythonhosted.org/packages/7b/81/b6d4d73a709684fc77e7fa034d7c2fe82cffa9fc920fadcaa659c2626213/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2b2e857ae16f5f72142edf75f9f176fe7526ba19a2841df1420516f83831c9f2", size = 663226, upload-time = "2026-06-26T19:10:13.723Z" }, - { url = "https://files.pythonhosted.org/packages/f5/07/e210b02b589f16e74ff48b730690e4a34ffe984219fce4f3c1a0e7ec8545/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e515757e2e36bcbf1fad09a46e1557e8b1ae1797d4b44d09da7deed88ad28608", size = 660802, upload-time = "2026-06-26T18:32:26.081Z" }, - { url = "https://files.pythonhosted.org/packages/eb/2e/5303eb3fa06bca089060f479707182a93e360683bc252acf846c3090d34e/greenlet-3.5.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b363d46ed1ea431825fdb01471bb024fc08399bad1572a616e853c7684415adb", size = 1622157, upload-time = "2026-06-26T19:09:09.527Z" }, - { url = "https://files.pythonhosted.org/packages/54/70/50de47a488f14df260b50ae34fb5d56016e308b098eab02c878b5223c26a/greenlet-3.5.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e44da2f5bbdaabaf7d80b73dbb430c7035771e9f244e3c8b769715c9d8fa0a16", size = 1681159, upload-time = "2026-06-26T18:31:52.986Z" }, -] - [[package]] name = "gsl" version = "0.0.3" @@ -1354,15 +1247,6 @@ http2 = [ { name = "h2" }, ] -[[package]] -name = "humanfriendly" -version = "10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794, upload-time = "2021-09-17T21:40:39.897Z" }, -] - [[package]] name = "hyperframe" version = "6.1.0" @@ -1403,22 +1287,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/53/fb7122b71361a0d121b669dcf3d31244ef75badbbb724af388948de543e2/imagesize-2.0.0-py2.py3-none-any.whl", hash = "sha256:5667c5bbb57ab3f1fa4bc366f4fbc971db3d5ed011fd2715fd8001f782718d96", size = 9441, upload-time = "2026-03-03T14:18:27.892Z" }, ] -[[package]] -name = "immutables" -version = "0.21" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/69/41/0ccaa6ef9943c0609ec5aa663a3b3e681c1712c1007147b84590cec706a0/immutables-0.21.tar.gz", hash = "sha256:b55ffaf0449790242feb4c56ab799ea7af92801a0a43f9e2f4f8af2ab24dfc4a", size = 89008, upload-time = "2024-10-10T00:55:01.434Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/52/cb9e2bb7a69338155ffabbd2f993c968c750dd2d5c6c6eaa6ebb7bfcbdfa/immutables-0.21-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b48b116aaca4500398058b5a87814857a60c4cb09417fecc12d7da0f5639b73d", size = 104270, upload-time = "2024-10-10T00:54:21.912Z" }, - { url = "https://files.pythonhosted.org/packages/0f/a4/25df835a9b9b372a4a869a8a1ac30a32199f2b3f581ad0e249f7e3d19eed/immutables-0.21-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dad7c0c74b285cc0e555ec0e97acbdc6f1862fcd16b99abd612df3243732e741", size = 104864, upload-time = "2024-10-10T00:54:22.956Z" }, - { url = "https://files.pythonhosted.org/packages/4a/51/b548fbc657134d658e179ee8d201ae82d9049aba5c3cb2d858ed2ecb7e3f/immutables-0.21-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e44346e2221a5a676c880ca8e0e6429fa24d1a4ae562573f5c04d7f2e759b030", size = 99733, upload-time = "2024-10-10T00:54:23.99Z" }, - { url = "https://files.pythonhosted.org/packages/47/db/d7b1e0e88faf07fe9a88579a86f58078a9a37fff871f4b3dbcf28cad9a12/immutables-0.21-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b10139b529a460e53fe8be699ebd848c54c8a33ebe67763bcfcc809a475a26f", size = 101698, upload-time = "2024-10-10T00:54:25.734Z" }, - { url = "https://files.pythonhosted.org/packages/d2/d0/a5fb7c164ddb298ec37537e618b70dfa30c7cae9fac01de374c36489cbc9/immutables-0.21-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:583d2a63e444ce1538cc2bda56ae1f4a1a11473dbc0377c82b516bc7eec3b81e", size = 104334, upload-time = "2024-10-10T00:54:31.284Z" }, - { url = "https://files.pythonhosted.org/packages/f3/a5/5fda0ee4a261a85124011ac0750fec678f00e1b2d4a5502b149a3b4d86d9/immutables-0.21-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b274a52da9b106db55eceb93fc1aea858c4e6f4740189e3548e38613eafc2021", size = 104898, upload-time = "2024-10-10T00:54:32.295Z" }, - { url = "https://files.pythonhosted.org/packages/93/fa/d46bfe92f2c66d35916344176ff87fa839aac9c16849652947e722b7a15f/immutables-0.21-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:338bede057250b33716a3e4892e15df0bf5a5ddbf1d67ead996b3e680b49ef9e", size = 99966, upload-time = "2024-10-10T00:54:34.046Z" }, - { url = "https://files.pythonhosted.org/packages/d7/f5/2a19e2e095f7a39d8d77dcc10669734d2d99773ce00c99bdcfeeb7d714e6/immutables-0.21-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8781c89583b68f604cf30f0978b722165824c3075888639fde771bf1a3e12dc0", size = 101773, upload-time = "2024-10-10T00:54:35.851Z" }, -] - [[package]] name = "importlib-metadata" version = "9.0.0" @@ -2853,15 +2721,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, ] -[[package]] -name = "pulp" -version = "3.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4e/70/69be07a67621ad804d6cf347965eb4e0d7786a97330d99c31d735aaa6c5a/pulp-3.3.2.tar.gz", hash = "sha256:d0904700c207ac11e25e3b1213b70eae1d6fb25faa719d75f3f15054901258c0", size = 16305346, upload-time = "2026-05-25T09:41:26.207Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/6e/d674f1dde91c71e2ac19e5e5cf1ee6d5845e3aefecd9c39ed9c4b0c9a696/pulp-3.3.2-py3-none-any.whl", hash = "sha256:631b166f72086971a9597f7a0233ababa99bb8d50a01cd543f7758be5a9f86c0", size = 16391742, upload-time = "2026-05-25T09:41:22.2Z" }, -] - [[package]] name = "pure-eval" version = "0.2.3" @@ -3646,27 +3505,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e8/ae/fa6cd331b364ad2bbc31652d025f5747d89cbb75576733dfdf8efe3e4d62/slicerator-1.1.0-py3-none-any.whl", hash = "sha256:167668d48c6d3a5ba0bd3d54b2688e81ee267dc20aef299e547d711e6f3c441a", size = 10274, upload-time = "2022-04-07T18:54:07.029Z" }, ] -[[package]] -name = "smart-open" -version = "7.7.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/db/c6/22e7a2acd5d27941e85e0d7ede398da5abe2e4677d2265c924157247c32e/smart_open-7.7.1.tar.gz", hash = "sha256:9414ba5733e28309f29b28a303b0f1054ad23fe0275f1a1b600c80a724f4bd1a", size = 54952, upload-time = "2026-06-26T07:56:35.309Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/c5/14732c140fa0af0e59345949eaf9016c75ad3f076586f2413e3d3566a8da/smart_open-7.7.1-py3-none-any.whl", hash = "sha256:cb62dc45f519bf39b612564d326d2a17556f3a6056e3c3a86a07215d670d45bc", size = 65061, upload-time = "2026-06-26T07:56:33.685Z" }, -] - -[[package]] -name = "smmap" -version = "5.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1f/ea/49c993d6dfdd7338c9b1000a0f36817ed7ec84577ae2e52f890d1a4ff909/smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c", size = 22506, upload-time = "2026-03-09T03:43:26.1Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, -] - [[package]] name = "smokescreen" version = "1.5.6" @@ -3682,128 +3520,6 @@ dependencies = [ { name = "scipy" }, ] -[[package]] -name = "snakemake" -version = "9.23.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "conda-inject" }, - { name = "configargparse" }, - { name = "connection-pool" }, - { name = "docutils" }, - { name = "dpath" }, - { name = "gitpython" }, - { name = "humanfriendly" }, - { name = "immutables" }, - { name = "jinja2" }, - { name = "jsonschema" }, - { name = "nbformat" }, - { name = "packaging" }, - { name = "platformdirs" }, - { name = "psutil" }, - { name = "pulp" }, - { name = "pyyaml" }, - { name = "referencing" }, - { name = "requests" }, - { name = "smart-open" }, - { name = "snakemake-interface-common" }, - { name = "snakemake-interface-executor-plugins" }, - { name = "snakemake-interface-logger-plugins" }, - { name = "snakemake-interface-report-plugins" }, - { name = "snakemake-interface-scheduler-plugins" }, - { name = "snakemake-interface-storage-plugins" }, - { name = "sqlmodel" }, - { name = "tabulate" }, - { name = "tenacity" }, - { name = "throttler" }, - { name = "wrapt" }, - { name = "yte" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9e/c2/45aa858e55edbb7dbc243ed60859f4f3d92d63ecdd67a80632aa3659fe6b/snakemake-9.23.1.tar.gz", hash = "sha256:ef8d698bfce66a6669cc29df7e344b0b367fd90956c725e5ae0a79a556e8e93f", size = 6802587, upload-time = "2026-06-18T09:42:57.653Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/65/2cb1c99117746bea8aff34dc434703f8f23d3876860e2ff3ebeccb754b0f/snakemake-9.23.1-py3-none-any.whl", hash = "sha256:9fdbfff25f1b442cdf7906ca06b465fc24b9bf72e2d51514fb93d83e3f8aeb35", size = 1162133, upload-time = "2026-06-18T09:42:55.541Z" }, -] - -[[package]] -name = "snakemake-interface-common" -version = "1.23.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argparse-dataclass" }, - { name = "configargparse" }, - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/89/c3/592f832f6e5d2d31f749392e48e8401b7625dec668d3d365d8d28f2b6c30/snakemake_interface_common-1.23.0.tar.gz", hash = "sha256:6ed14531a461417659364a0dd0acc51b786af4e26fc15cc5e00ff3d9fcaffacc", size = 13960, upload-time = "2026-03-08T21:54:29.251Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/28/90/a4a3652e832667917383f8d34112bdb37b78a6d14efc049e22a2c8192524/snakemake_interface_common-1.23.0-py3-none-any.whl", hash = "sha256:43bb10102d07149f27787dfc65a7342b86ff772973e7f9ca4940c89cb412a11e", size = 17162, upload-time = "2026-03-08T21:54:28.074Z" }, -] - -[[package]] -name = "snakemake-interface-executor-plugins" -version = "9.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argparse-dataclass" }, - { name = "snakemake-interface-common" }, - { name = "throttler" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/54/50/de06b284c45a8e94fb8e4a12d5235065e78b49b8f84329dc10fe39f4b7dd/snakemake_interface_executor_plugins-9.4.0.tar.gz", hash = "sha256:9d4138897beacbaadaedad94b63f948eaeb604b7fc78f9cf65ac57f090f2c066", size = 16549, upload-time = "2026-03-08T17:04:02.644Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/2c/4422eb14884bb4955ce65819799ff6452dcd3aa5f8e158540a287877bbcf/snakemake_interface_executor_plugins-9.4.0-py3-none-any.whl", hash = "sha256:d335c093185897e4e3654a39bab3a6f1c57dc1a54f998c92bd2d88640ab4a05e", size = 22567, upload-time = "2026-03-08T17:04:01.296Z" }, -] - -[[package]] -name = "snakemake-interface-logger-plugins" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "snakemake-interface-common" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a7/0c/3fa5d592663c65669a867526604aadc6fbc235fb9284e94b49c0ef59aa41/snakemake_interface_logger_plugins-2.1.0.tar.gz", hash = "sha256:c89a00d2a398490cecd91b6dc6db8049cba93712d82e1d8f3000f3040bf3791c", size = 15917, upload-time = "2026-05-20T15:12:35.259Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/43/cfdaf562303012024c8f48efebdfe258eef3d895665bd7e0000fdeb2411d/snakemake_interface_logger_plugins-2.1.0-py3-none-any.whl", hash = "sha256:601e3e0770dd1b460c1dcb7bd33f4fc2eb8af5016634b80d985cb993f674ef46", size = 14313, upload-time = "2026-05-20T15:12:34.186Z" }, -] - -[[package]] -name = "snakemake-interface-report-plugins" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "snakemake-interface-common" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/d6/6160ed98de665d6871dd356597dbf726688cc786e88668359ca37b7d9f54/snakemake_interface_report_plugins-1.3.0.tar.gz", hash = "sha256:fc9495298bec4e69721ab8afe6d6d88a86966fda2eeb003db56b9a88b86d5934", size = 4283, upload-time = "2025-10-31T10:52:36.55Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/f0/df73f6abc9b5910e43612ae28c7b6f666af80c4edd46a216ef47599ab6cb/snakemake_interface_report_plugins-1.3.0-py3-none-any.whl", hash = "sha256:78da3931f70e79eef51e5645a40b172929e555fe4d86ff45d6b856e521a379db", size = 7251, upload-time = "2025-10-31T10:52:35.474Z" }, -] - -[[package]] -name = "snakemake-interface-scheduler-plugins" -version = "2.0.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "snakemake-interface-common" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/88/d9/d480807d2cfc2d132bc760d877d45ec8fbe620a24200ec4d2697c4a26031/snakemake_interface_scheduler_plugins-2.0.2.tar.gz", hash = "sha256:2797e8fa9019d983132c2b403f14d6fcd3c5ad4c8d8a66b984b4740a71cacc46", size = 8642, upload-time = "2025-10-20T13:58:12.988Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/d0/f4e9894c8aaf37efe3bf1afe15ee3cf0546d82b2713a589e266ee47bf2ef/snakemake_interface_scheduler_plugins-2.0.2-py3-none-any.whl", hash = "sha256:b9ddfa508bd480711de1770dfb24f3b813cfa3cd0f862f0127ef721ae5346915", size = 10766, upload-time = "2025-10-20T13:58:11.898Z" }, -] - -[[package]] -name = "snakemake-interface-storage-plugins" -version = "4.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "humanfriendly" }, - { name = "snakemake-interface-common" }, - { name = "tenacity" }, - { name = "throttler" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/93/6e/f3c5b2d621fd6a6b78d8cfc01fef6b926fe2c277f5ed77c5e4deeacb94eb/snakemake_interface_storage_plugins-4.4.1.tar.gz", hash = "sha256:b2b5bf05318af36955ebf2ce76c921c0fb06904ca98fb30e1657d88b7b7b6945", size = 14924, upload-time = "2026-03-16T11:16:01.075Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/49/1f72d8c05d8588f06a61f1af3deb125a4ff8169d2a41f1b839d62c824d1c/snakemake_interface_storage_plugins-4.4.1-py3-none-any.whl", hash = "sha256:6e075f448543cc3d0cb1ee5e6e5a83af9761e7085f3ce0922011685d098f231e", size = 18256, upload-time = "2026-03-16T11:15:59.582Z" }, -] - [[package]] name = "snowballstemmer" version = "3.1.1" @@ -3903,7 +3619,6 @@ workflow = [ { name = "cosmosis" }, { name = "fast-pt" }, { name = "mpi4py" }, - { name = "snakemake" }, ] [package.metadata] @@ -3957,7 +3672,6 @@ requires-dist = [ { name = "shear-psf-leakage", git = "https://github.com/CosmoStat/shear_psf_leakage.git?rev=develop" }, { name = "skyproj" }, { name = "smokescreen", git = "https://github.com/UNIONS-WL/Smokescreen?rev=main" }, - { name = "snakemake", marker = "extra == 'workflow'" }, { name = "sp-validation", extras = ["test", "docs"], marker = "extra == 'develop'" }, { name = "sphinx", marker = "extra == 'docs'", specifier = ">=8.0" }, { name = "sphinxawesome-theme", marker = "extra == 'docs'", specifier = ">=5.3,!=6.0.3" }, @@ -4078,48 +3792,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, ] -[[package]] -name = "sqlalchemy" -version = "2.0.51" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/02/f1/a7a892f18d4d224e6b26f706531eafccc41e37594d37d304786969ee13cb/sqlalchemy-2.0.51.tar.gz", hash = "sha256:804dccd8a4a6242c4e30ad961e540e18a588f6527202f2d6791b01845d59fdc9", size = 9912201, upload-time = "2026-06-15T15:41:20.012Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/1c/71ee0f8a6b9d7316a1ccd30430b4c62b6c2e36adc96017a4e3a72dce49d6/sqlalchemy-2.0.51-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581921d849d6e6f994d560389192955e80e2950e18fcdfe2ccea863e01158e6e", size = 3343835, upload-time = "2026-06-15T16:19:42.613Z" }, - { url = "https://files.pythonhosted.org/packages/2b/7c/7ab9f9aadc5944fdd06612484ed7918fe376ad871a5f50404dc1536e0194/sqlalchemy-2.0.51-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d21ce524ab86c23046e992a5b81cb54c21079c6df6e78b8fc77d77cac70a6b9", size = 3358470, upload-time = "2026-06-15T16:26:38.011Z" }, - { url = "https://files.pythonhosted.org/packages/d0/7d/ff77169fee6186de145a7f2b87006c39638391130abbab2b1f63ac6ea583/sqlalchemy-2.0.51-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c5d98a2709840027f5a347c3af0a7c3d5f6c1ff93af2ca1c54494e23cba8f389", size = 3289874, upload-time = "2026-06-15T16:19:45.212Z" }, - { url = "https://files.pythonhosted.org/packages/6f/3b/6c505903710d781b55bc3141ee34a062bf9745a6b5bc7333305b9ed63b33/sqlalchemy-2.0.51-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1181256e0f16479691b5616d36375dc2620ad8332b25978763c3d206ad3f3f1d", size = 3321692, upload-time = "2026-06-15T16:26:39.747Z" }, - { url = "https://files.pythonhosted.org/packages/17/6b/2dce8369b199cb855110e056032f94a9f66dacc2237d3d39c115a86eac56/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:483b11bd46bf35fc14c52faf338b04300c9e6ce554bce9b11be85bfec3bc3195", size = 3284735, upload-time = "2026-06-15T16:19:46.934Z" }, - { url = "https://files.pythonhosted.org/packages/53/ff/dbc495b8a14da840faffb353857a72d4190113cac33727906fb997047f0f/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1bed1ee8b01da6088210aa9412023326fb98a599ba502e6118308601dcbef77f", size = 3302756, upload-time = "2026-06-15T16:26:41.336Z" }, - { url = "https://files.pythonhosted.org/packages/cf/d5/fde8f4dddcf518ee15ab35a7c6a28acc32c8ba548d1d2aa451f96e6dbb0b/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:72ca54c952107ba5cd58854b67a5a6268631289d21651a1235396f3b98b47400", size = 3232055, upload-time = "2026-06-15T16:19:49.286Z" }, - { url = "https://files.pythonhosted.org/packages/67/d1/43d3a0ac955a58601c24fa23038b1c55ee3a1ec02c0f96ebb1eae2bcf614/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b3e693d15533a45cd5906f0589f9c35090bef6ef45bf1e8195c424aa0ae06a8d", size = 3269850, upload-time = "2026-06-15T16:26:43.017Z" }, - { url = "https://files.pythonhosted.org/packages/23/6b/2e0e38cf75c8780eca78d9b2e78164f8bcfd70125e5caa588ff5cbb9c9f4/sqlalchemy-2.0.51-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c45a496d6bc05dec41dcd4c3a2b183723f47473255c159cd80b503c8f246424d", size = 3282539, upload-time = "2026-06-15T16:19:51.065Z" }, - { url = "https://files.pythonhosted.org/packages/dd/a1/e77854cb5336fd37dc3c6ae3b71de242c98caac5725120be0b526b31cbd0/sqlalchemy-2.0.51-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4004ada0aafe8ae1991b2cd1d99c6d9146126e123bd6f883c260d974aa012e54", size = 3287545, upload-time = "2026-06-15T16:26:44.735Z" }, - { url = "https://files.pythonhosted.org/packages/f6/ab/9e17272fd4dac8df3b83c4fbe52b998a1c9d89a843c8c35ff29b74ff7364/sqlalchemy-2.0.51-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0f6bcad487aee1c638d707235682fc96f741de00663619881ab235400d03289e", size = 3230929, upload-time = "2026-06-15T16:19:52.625Z" }, - { url = "https://files.pythonhosted.org/packages/02/3c/52f408ea701781caee975606beccc48845f2aee8711ac29843d612c0306c/sqlalchemy-2.0.51-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:39a76529db6305693d8d4affa58ad5b5e2e18edd62daea628b29b97930b3513d", size = 3252888, upload-time = "2026-06-15T16:26:46.454Z" }, - { url = "https://files.pythonhosted.org/packages/bf/90/a1bb5c7cbba76b7bc1fbd586d0a5479a7bc9c27b4a8298f22ec9423b2bb3/sqlalchemy-2.0.51-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c6b36ed71f41942bdcd2ad2522be46bfce09d5705be5640ecf19bbc7660e4b7", size = 3566534, upload-time = "2026-06-15T15:58:35.024Z" }, - { url = "https://files.pythonhosted.org/packages/15/4b/481f1fed30e0e9e8dd24aecbb49f29eb57fe7657ece5cf06ee9b84bb97d8/sqlalchemy-2.0.51-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c2c62877097e1a0db401fba5cb4debee33265e5b2a55c4ccb489c02c53b4f72", size = 3535844, upload-time = "2026-06-15T16:02:43.973Z" }, - { url = "https://files.pythonhosted.org/packages/02/71/0aa64aeda645510af0a43f7d9ee70932f0d1dc4263aed34c50ee891d9df3/sqlalchemy-2.0.51-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0378d055e9e8cd6ce4d8dff683bdd3d7d413533c4ee51d67a2b1e0f9eacc0f23", size = 3475355, upload-time = "2026-06-15T15:58:36.592Z" }, - { url = "https://files.pythonhosted.org/packages/05/db/6061db32316446135a3abae5f308d144ab988a34234726042da3e58b1c63/sqlalchemy-2.0.51-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6e46fc36029eff666391e0531e5387b62ce6c4f1d8e50b3fb3099eaca1b42522", size = 3486591, upload-time = "2026-06-15T16:02:45.346Z" }, - { url = "https://files.pythonhosted.org/packages/e2/22/dbf013a12ec759e54a34a119e9e217435b3f71b2dd5c61a7ade0a25dae87/sqlalchemy-2.0.51-py3-none-any.whl", hash = "sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5", size = 1944334, upload-time = "2026-06-15T16:09:22.418Z" }, -] - -[[package]] -name = "sqlmodel" -version = "0.0.37" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "sqlalchemy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fb/26/1d2faa0fd5a765267f49751de533adac6b9ff9366c7c6e7692df4f32230f/sqlmodel-0.0.37.tar.gz", hash = "sha256:d2c19327175794faf50b1ee31cc966764f55b1dedefc046450bc5741a3d68352", size = 85527, upload-time = "2026-02-21T16:39:47.038Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/e1/7c8d18e737433f3b5bbe27b56a9072a9fcb36342b48f1bef34b6da1d61f2/sqlmodel-0.0.37-py3-none-any.whl", hash = "sha256:2137a4045ef3fd66a917a7717ada959a1ceb3630d95e1f6aaab39dd2c0aef278", size = 27224, upload-time = "2026-02-21T16:39:47.781Z" }, -] - [[package]] name = "stack-data" version = "0.6.3" @@ -4234,24 +3906,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/68/025307ebd3c54c65e540d2a4ced11d0e5aa370c8ddb6d0e83066e5a39054/tables_io-1.1.2-py3-none-any.whl", hash = "sha256:4361e04dfcc10471bf9a3940d9ee6a1f903ac76ece51d32ef7fec7ea572b20c0", size = 39206, upload-time = "2026-06-18T16:21:19.577Z" }, ] -[[package]] -name = "tabulate" -version = "0.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, -] - -[[package]] -name = "tenacity" -version = "9.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, -] - [[package]] name = "terminado" version = "0.18.1" @@ -4274,15 +3928,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, ] -[[package]] -name = "throttler" -version = "1.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ce/3f/47baf510c31e0e52ac0d80d9071e5e166ca069167fee4a6c13841f9d5f5f/throttler-1.2.3.tar.gz", hash = "sha256:d2f5b0b499d62f1fc984dcac8043450b606549b0097753a9c8a707f7427c27e1", size = 11792, upload-time = "2026-01-27T00:48:11.552Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/70/42d8796acc57c8bcd9ae395b1a6a0bbc833f738492a8ed192a44ccd58035/throttler-1.2.3-py3-none-any.whl", hash = "sha256:241ea3e97438dec4dc2f31ddc56dbd96262787a9b1d0598adfcc0bada1134b66", size = 9704, upload-time = "2026-01-27T00:48:09.544Z" }, -] - [[package]] name = "tifffile" version = "2026.6.1" @@ -4550,20 +4195,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, ] -[[package]] -name = "yte" -version = "1.9.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argparse-dataclass" }, - { name = "dpath" }, - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/44/f5/7e44620e6e077bfe624b9a17c329b8e0d0159e176e1f1a93c2790428ab2c/yte-1.9.4.tar.gz", hash = "sha256:86a47e6d722cec9419a7ac88be57d0d6c4ce28f02860393b71a66f2c674069f6", size = 8101, upload-time = "2025-11-27T12:55:00.85Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/63/6a44729fdc60eb255a7b156a84e7552290174a9bf151e3b6c18e83d6fbfa/yte-1.9.4-py3-none-any.whl", hash = "sha256:5dac63303d3e6bc2ebadc36ece3c3fb09343772fe6e25e9356d9baf8f9dfaf6d", size = 10618, upload-time = "2025-11-27T12:55:01.685Z" }, -] - [[package]] name = "zarr" version = "3.2.1" diff --git a/workflow/README.md b/workflow/README.md index b6058026..6bcb186c 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -139,20 +139,20 @@ already uses the plain form; keep new paths the same. ### Run Snakemake from the host, never from inside the container -`snakemake` is a thin host-side tool, pinned once per machine: +`snakemake` is a thin host-side tool, installed once per machine on the +image's Python: ```bash -uv tool install --python 3.12 snakemake==9.23.1 --with snakemake-executor-plugin-slurm +uv tool install --python 3.12 snakemake --with snakemake-executor-plugin-slurm ``` -The Python minor and the version must match the image's (the image's Python -and the `snakemake` in this repo's `uv.lock`): a `script:` job appends the -host's `sys.path` — standard library included — to its own so that it can -unpickle the host's `snakemake` object, so a host on another Python loads -modules the image lacks from the host's stdlib, and another Snakemake writes a -pickle the job cannot read. Every launch checks this (`common.check_host_parity`) -and stops with the reinstall command on a mismatch; an image it cannot read, -such as a registry tag, is named in one line and passes. +Any Snakemake version works; the Python minor must be the image's. A `script:` +job appends the host's `sys.path` to its own and unpickles the host's +`snakemake` object with the host's own package (the image carries none), so +that package and its compiled dependencies load into the image's interpreter. +Every launch checks the minor (`common.check_host_python`) and stops with the +reinstall command on a mismatch; an image it cannot read, such as a registry +tag, is named in one line and passes. Run every `snakemake` command directly on the host — do not `apptainer shell` first. @@ -162,11 +162,6 @@ wrapping from the profile (see above), so the container is where the science code runs, not where the orchestrator runs — one container per job, never a nested one. -Check for a stray `~/.local/bin/snakemake` (any host-side `pip install --user -snakemake` leaves one): Apptainer passes your `PATH` and mounts your `$HOME` by -default, so it can silently shadow the one `uv tool install` set up. `which -snakemake` should resolve under `uv tool dir`, not `~/.local/bin`. - ### The container image — one per person Everything runs one image, published by CI as a registry tag: @@ -298,7 +293,7 @@ visible. checkout and — on candide — on the real papers: ```bash -uv run --isolated --no-project --python 3.12 --with snakemake==9.23.1 \ +uv run --isolated --no-project --python 3.12 --with snakemake \ --with snakemake-executor-plugin-slurm --with pytest --with numpy \ pytest workflow/tests ``` diff --git a/workflow/common.py b/workflow/common.py index b4844efe..597f3954 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -30,7 +30,7 @@ compare_revision = _container.compare_revision image_revision = _container.image_revision -image_runtime = _container.image_runtime +image_python = _container.image_python resolve_image = _container.resolve_image # Every job inherits this launch's environment (the slurm executor submits with @@ -126,10 +126,10 @@ def resolve_container(override=None): ``override`` wins if set (a ``docker://`` tag, a ``.sif`` path or a sandbox directory -- Snakemake's ``container:`` accepts all three); otherwise ``resolve_image()``, so jobs run what interactive ``spv-container`` work - runs. The image returned has passed ``check_host_parity``. + runs. The image returned has passed ``check_host_python``. """ image = str(override) if override else resolve_image()[0] - check_host_parity(image) + check_host_python(image) return image @@ -169,40 +169,33 @@ def warn_if_image_stale(): @functools.cache -def check_host_parity(image): - """Stop the launch unless this Snakemake matches the image's Python and Snakemake. +def check_host_python(image): + """Stop the launch unless this Snakemake runs on the image's Python minor. - A ``script:`` job appends the host's ``sys.path`` -- standard library - included -- to its own, as the fallback that lets it unpickle the host's - ``snakemake`` object. So the image and the host Snakemake must share a - Python minor (else a module the image lacks loads from the host's stdlib) - and a Snakemake version (else the pickle does not match its reader). + A ``script:`` job unpickles the host's ``snakemake`` object by appending the + host's ``sys.path`` to its own; the image carries no snakemake, so the host's + package and its compiled dependencies load into the image's interpreter. An image that cannot be read (a registry tag, no apptainer) is named in one line and passes. """ - import snakemake from snakemake.exceptions import WorkflowError - runtime = image_runtime(image) - if runtime is None: + python = image_python(image) + if python is None: print( - f"[container] cannot read the Python of {image}; host/image parity " + f"[container] cannot read the Python of {image}; host/image Python " "unchecked.", file=sys.stderr, ) return - python, image_snakemake = runtime - host_python = ".".join(str(v) for v in sys.version_info[:2]) - host = (host_python, snakemake.__version__) - if (python, image_snakemake or snakemake.__version__) == host: + host = ".".join(str(v) for v in sys.version_info[:2]) + if python == host: return raise WorkflowError( - f"host Snakemake {snakemake.__version__} on Python {host_python} does not " - f"match the image {image} (Snakemake {image_snakemake}, Python {python}).\n" - "Reinstall the host Snakemake to match:\n" - f" uv tool install --force --python {python} " - f"snakemake=={image_snakemake or snakemake.__version__} " + f"host Snakemake runs on Python {host}, the image {image} on Python " + f"{python}.\nReinstall the host Snakemake on the image's Python:\n" + f" uv tool install --force --python {python} snakemake " "--with snakemake-executor-plugin-slurm" ) diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py index 29ebf5c4..08db8e9d 100644 --- a/workflow/tests/conftest.py +++ b/workflow/tests/conftest.py @@ -2,7 +2,7 @@ These tests run under the host launcher, never inside the image:: - uv run --isolated --no-project --python 3.12 --with snakemake==9.23.1 \\ + uv run --isolated --no-project --python 3.12 --with snakemake \\ --with snakemake-executor-plugin-slurm --with pytest --with numpy \\ pytest workflow/tests @@ -14,8 +14,8 @@ ``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, a one-catalogue ``cosmo_val/cat_config.yaml``, a touched catalogue file, the processed CosmoCov covariances already in place (their inputs live on candide), and both output -roots in tmp. Its runs use a fake image whose Python and Snakemake match the -running ones, so the launch-time parity check passes without apptainer. +roots in tmp. Its runs use a fake image whose Python matches the running one, +so the launch-time Python check passes without apptainer. """ import dataclasses @@ -28,7 +28,6 @@ from pathlib import Path import pytest -import snakemake import yaml REPO = Path(__file__).resolve().parents[2] @@ -39,12 +38,10 @@ HOST_PYTHON = ".".join(str(v) for v in sys.version_info[:3]) -def fake_image(root, python=HOST_PYTHON, snakemake_version=snakemake.__version__): - """A sandbox-shaped directory carrying only what the parity check reads.""" +def fake_image(root, python=HOST_PYTHON): + """A sandbox-shaped directory carrying only what the Python check reads.""" venv = Path(root) / "app" / ".venv" - minor = ".".join(python.split(".")[:2]) - site = venv / "lib" / f"python{minor}" / "site-packages" - (site / f"snakemake-{snakemake_version}.dist-info").mkdir(parents=True) + venv.mkdir(parents=True) (venv / "pyvenv.cfg").write_text( f"home = /usr/local/bin\nimplementation = CPython\nversion_info = {python}\n" ) diff --git a/workflow/tests/data/container_smoke/container_smoke.py b/workflow/tests/data/container_smoke/container_smoke.py index effc0b0b..73a0cebf 100644 --- a/workflow/tests/data/container_smoke/container_smoke.py +++ b/workflow/tests/data/container_smoke/container_smoke.py @@ -2,13 +2,15 @@ Cheap sanity check of the profile-driven container path -- same executor (slurm), same software-deployment-method (apptainer), same apptainer-args -binds, same container image every real rule uses. Four things it proves, each +binds, same container image every real rule uses. Five things it proves, each written to the output YAML: * the job really ran inside the image (``APPTAINER_CONTAINER``, set by apptainer itself -- without it the rest could all pass on the bare host); * which ``sp_validation`` the job imports (file + version, not just import success); + * which ``snakemake`` package unpickled the injected ``snakemake`` object + (file + version); * the numeric stack works (numpy eigh on a small fixed matrix). ``OMP_NUM_THREADS`` is recorded but not asserted -- see the assertions; * which commit of this checkout is running (git rev-parse from inside the @@ -21,6 +23,7 @@ import os import platform import subprocess +import sys import numpy as np import yaml @@ -38,6 +41,14 @@ "file": sp_validation.__file__, } +# --- the snakemake package the preamble unpickled with --------------------- +# Read from sys.modules: importing it here would rebind the injected global. +snakemake_package = sys.modules["snakemake"] +snakemake_info = { + "version": snakemake_package.__version__, + "file": snakemake_package.__file__, +} + # --- numeric stack + threading ----------------------------------------- rng = np.random.default_rng(seed=42) a = rng.standard_normal((8, 8)) @@ -78,6 +89,7 @@ { "container": container_info, "sp_validation": sp_validation_info, + "snakemake": snakemake_info, "numeric": numeric_info, "provenance": provenance, }, diff --git a/workflow/tests/test_container_smoke.py b/workflow/tests/test_container_smoke.py index faae81ad..5454d8d3 100644 --- a/workflow/tests/test_container_smoke.py +++ b/workflow/tests/test_container_smoke.py @@ -22,6 +22,7 @@ import numpy as np import pytest +import snakemake import yaml from conftest import REPO, container, on_candide @@ -86,6 +87,9 @@ def test_container_smoke(): module_file ) + # The job read the pickle with the Snakemake that wrote it. + assert report["snakemake"]["version"] == snakemake.__version__, report["snakemake"] + # The numeric stack agrees with the same computation run here. np.testing.assert_allclose( report["numeric"]["eigenvalues"], diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index e2218f1e..e6dbfc82 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -7,7 +7,7 @@ import pytest import yaml -from conftest import HOST_PYTHON, REPO, VERSIONS, fake_image, on_candide, parse_jobs +from conftest import REPO, VERSIONS, fake_image, on_candide, parse_jobs def test_assemble_resolves(toy): @@ -93,22 +93,12 @@ def test_outputs_stay_in_the_output_roots(toy, named): assert not strays, strays -@pytest.mark.parametrize( - "python, snakemake_version", - [("3.13.1", None), (None, "9.0.0")], - ids=["python-minor", "snakemake"], -) -def test_image_parity_is_checked_at_launch(toy, tmp_path, python, snakemake_version): - """An image whose Python minor or Snakemake differs stops the launch.""" - image = fake_image( - tmp_path / "image", - **({"python": python} if python else {}), - **({"snakemake_version": snakemake_version} if snakemake_version else {}), - ) +def test_image_python_is_checked_at_launch(toy, tmp_path): + """An image on another Python minor stops the launch.""" + image = fake_image(tmp_path / "image", python="3.13.1") result = toy.snakemake("-n", "assemble_sacc_all", container=image) assert result.returncode != 0, result.stdout - minor = ".".join((python or HOST_PYTHON).split(".")[:2]) - assert f"uv tool install --force --python {minor} snakemake==" in result.stdout + assert "uv tool install --force --python 3.13 snakemake " in result.stdout assert "rule assemble_sacc" not in result.stdout @@ -118,7 +108,7 @@ def test_unreadable_image_is_named_not_fatal(toy): "-n", "assemble_sacc_all", container="docker://example.org/image:tag" ) assert result.returncode == 0, result.stdout - assert "parity unchecked" in result.stdout + assert "Python unchecked" in result.stdout def test_launch_reads_the_image_under_home(toy, tmp_path): @@ -134,7 +124,7 @@ def test_launch_reads_the_image_under_home(toy, tmp_path): env.update(HOME=str(home), XDG_CACHE_HOME=str(tmp_path / "node-local")) result = toy.snakemake("-n", "assemble_sacc_all", container=False, env=env) assert result.returncode != 0, result.stdout - assert "uv tool install --force --python 3.13 snakemake==" in result.stdout + assert "uv tool install --force --python 3.13 snakemake " in result.stdout def _apptainer_stub(tmp_path): @@ -232,7 +222,7 @@ def launch(profile_dir): assert refused.returncode != 0 and "--jobs" in refused.stdout, refused.stdout -def test_image_sims_checks_parity_at_launch(toy, tmp_path): +def test_image_sims_checks_python_at_launch(toy, tmp_path): """The standalone image-sims workflow stops on a mismatched image too.""" run = { "image_sims": { @@ -257,7 +247,7 @@ def test_image_sims_checks_parity_at_launch(toy, tmp_path): cwd=toy.root, ) assert result.returncode != 0, result.stdout - assert "uv tool install --force --python 3.13 snakemake==" in result.stdout + assert "uv tool install --force --python 3.13 snakemake " in result.stdout def _real_dry_run(paper, targets): @@ -297,10 +287,10 @@ def test_papers_resolve_on_candide(paper, targets): """The real paper DAGs resolve against the real catalogues and your image. The e-cut catalogue reads its parent's catalogue entry. Your image (the SIF - or sandbox `spv-container` manages) is read, so parity was checked. + or sandbox `spv-container` manages) is read, so its Python was checked. """ result = _real_dry_run(paper, targets) assert result.returncode == 0, result.stdout - assert "parity unchecked" not in result.stdout, ( + assert "Python unchecked" not in result.stdout, ( "no local image was read; run `spv-container pull`\n" + result.stdout ) From 299ab65d9af98fcca2b295dffff93d45c34a63b2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 05:52:18 +0200 Subject: [PATCH 131/160] profiles: every key needed and non-default, each with its reason Drops the candide LD_LIBRARY_PATH to /softs/openmpi: /softs is not bound, so the path does not exist inside the container, and no containerized rule uses MPI. Drops the default profile's --bind /home (apptainer mounts $HOME already). States the real reasons for the rest: rerun-triggers leaves out software-env because it hashes the per-person image path; shared-fs-usage leaves out source-cache because jobs would be handed the launch's node-local cache path; slurm_account because the executor's guess fails on candide; retries and kept logs for fan-outs and their printed output. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- workflow/profiles/candide/config.yaml | 82 ++++++++++++--------------- workflow/profiles/default/config.yaml | 47 ++++++--------- 2 files changed, 53 insertions(+), 76 deletions(-) diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 466d7cd2..74569d64 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -1,62 +1,49 @@ -# Committed SLURM profile for the candide cluster (IAP). -# -# Drive any target with +# SLURM profile for the candide cluster (IAP). Drive any target with # # snakemake --profile workflow/profiles/candide \ # -s workflow/image_sims/Snakefile \ # --configfile # -# Snakemake owns scheduling and the container wrapping; run it host-side, never -# inside an ``apptainer shell``. workflow/README.md is the full story. +# from the host, never inside an ``apptainer shell``; workflow/README.md is the +# full story. executor: slurm -# --- GENERIC: mirrored in workflow/profiles/default/config.yaml ------------- +# --- mirrored in workflow/profiles/default/config.yaml ---------------------- software-deployment-method: apptainer -# Rerun a job when its code / params / inputs change, not only on mtime. -# Caveat: "code" watches rule bodies and ``script:`` files, not ``src/`` -- -# editing a module under src/ does not mark outputs stale on its own. +# Every trigger but software-env, which hashes the image *path*: that path is +# per person (~/.cache/sp_validation/...) and switches between SIF and sandbox, +# so it would rerun everything without the stack having changed. +# "code" watches rule bodies and ``script:`` files, not ``src/``. rerun-triggers: ["mtime", "params", "input", "code"] -# Give an appearing output file a moment on networked filesystems before -# Snakemake calls a job failed for a missing output. A job's write can take -# more than 5 s to show on the launching host (candide's /home does); a -# shorter wait reads a finished job as failed and re-runs it. +# A job's write can take more than 5 s to show on the launching host (candide's +# /home does); a shorter wait reads a finished job as failed and re-runs it. latency-wait: 60 -# --- end GENERIC ------------------------------------------------------------ +# --- end mirrored ----------------------------------------------------------- -# No ``apptainer-prefix``: the entry Snakefiles resolve ``container:`` to this -# user's own image path, so there is nothing for Snakemake to cache. -# -# candide's disks, plus the one machine-specific env var: the host OpenMPI libs -# MPI rules need to find libmpi inside the container (only rules importing -# mpi4py care; harmless for the rest). The bind list matches ``spv-container -# exec``'s default; keep the two in step. +# candide's disks; the same list as ``spv-container exec``'s default. apptainer-args: >- --cleanenv --bind /home,/scratch,/automnt,/n17data,/n23data1,/n09data - --env LD_LIBRARY_PATH=/softs/openmpi/5.0.5-slurm-CentOS8/lib -# Cluster policy applied to every job unless a rule overrides it. Excludes are -# the flaky/no-internet candide nodes (n17 mount issues, n09 no internet, n36). +# * ``runtime`` MUST carry a unit (``60m``, ``6h``, ``2d``): Snakemake reads a +# bare number here as SECONDS. (In a rule's own ``resources:`` a bare +# integer is minutes.) 60m is the partitions' default; setting it keeps the +# slurm executor from warning on every job that names none. # -# * ``runtime`` MUST carry a unit (``60m``, ``6h``, ``2d``). Snakemake's -# resource parser reads a bare number as SECONDS, so ``runtime: 60`` would -# silently give every job a 60-second wall clock and kill it on start. -# (A bare integer in a rule's own ``resources: runtime=720`` is fine -- -# rule-level numeric runtime is read as minutes; the seconds trap is only -# the CLI/default-resources parser.) +# * ``cpus_per_task`` is 12 to CAP JOBS PER NODE, not because a job needs 12 +# cores: past ~4 concurrent jobs on a 48-core node, candide's per-user +# process limit (``ulimit -u 1200``) crashes apptainer ("can't start new +# thread"). After launching a fan-out, ``squeue -u $USER -o "%C %l"`` +# should show 12 CPUs and the intended wall clock. # -# * ``cpus_per_task`` is pinned to 12 to CAP JOBS PER NODE, not because a job -# needs 12 cores. candide's per-user process limit is ``ulimit -u 1200`` -# per node, and apptainer crashes ("can't start new thread") beyond ~4 -# concurrent jobs on a 48-core node; 12 CPUs/job holds SLURM to ~4 jobs per -# node. +# * ``slurm_account``: the slurm executor's guess fails on candide (its +# ``sacct`` call errors) and warns on every launch. # -# * After launching a real fan-out, verify the request landed: -# ``squeue -u $USER -o "%C %l"`` should show 12 (CPUs) and the intended -# wall clock. +# * The excludes are the flaky nodes: n17 (mount issues), n09 (no internet), +# n36. default-resources: slurm_account: "cusers" slurm_partition: "comp,pscomp" @@ -64,14 +51,15 @@ default-resources: cpus_per_task: 12 slurm_extra: "'--exclude=n17,n09,n36'" -# How many jobs may be queued or running at once. Snakemake requires a bound -# for any remote executor and refuses a real run without one. +# Snakemake refuses a real run on a remote executor without a job bound. jobs: 100 -# Snakemake's source cache lives under the launching shell's XDG_CACHE_HOME, -# which may be node-local. Leaving source-cache out of the shared-filesystem -# usages keeps each job from reusing the launch's cache and puts the job's own -# in its tmp dir. +# Every usage but source-cache. Jobs inherit the launching shell's environment +# (--export=ALL), and a login shell may point XDG_CACHE_HOME at node-local +# /scratch, which some nodes lack; with the cache shared, each job is handed the +# launch's cache path under it. common.py drops XDG_CACHE_HOME for the Snakemake +# each job step starts, which the slurm-jobstep executor forces onto a shared +# cache regardless. shared-fs-usage: - persistence - input-output @@ -80,7 +68,9 @@ shared-fs-usage: - storage-local-copies - software-deployment-cache -# Retry a job once on transient node failure, and keep the SLURM logs of -# successful jobs (candide debugging). +# One retry absorbs a node failure in a large fan-out, which would otherwise +# stop the launch scheduling new jobs. retries: 1 + +# A successful job's SLURM log is the only record of what its script printed. slurm-keep-successful-logs: true diff --git a/workflow/profiles/default/config.yaml b/workflow/profiles/default/config.yaml index cd42355a..b7fc1b22 100644 --- a/workflow/profiles/default/config.yaml +++ b/workflow/profiles/default/config.yaml @@ -1,41 +1,28 @@ -# Machine-independent profile: the container model, and nothing else. -# -# Use it anywhere that is not candide -- a laptop, a workstation, another -# cluster's interactive node: +# Machine-independent profile: the container model, and nothing else. Use it +# anywhere that is not candide (a laptop, a workstation, another cluster): # # snakemake --profile workflow/profiles/default -s workflow/Snakefile \ # --configfile -j 4 # -# On candide -- where the analysis actually runs -- use -# `--profile workflow/profiles/candide` instead: the GENERIC block below plus -# the SLURM executor and candide's machine layer. -# -# Requirements are the same everywhere: `apptainer` on PATH, and `snakemake` -# installed host-side (`uv tool install ...`, see workflow/README.md) -- never -# run from inside an apptainer shell. +# On candide use workflow/profiles/candide. Either way, `apptainer` on PATH and +# `snakemake` installed host-side (workflow/README.md); never run from inside +# an apptainer shell. -# --- GENERIC: mirrored in workflow/profiles/candide/config.yaml ------------- -# Wrap each job's `shell:`/`script:` command in `apptainer exec`, using the -# image named by the entry Snakefile's `container:` directive. +# --- mirrored in workflow/profiles/candide/config.yaml ---------------------- software-deployment-method: apptainer -# Rerun a job when its code / params / inputs change, not only on mtime. -# Caveat: "code" watches rule bodies and `script:` files, not `src/` -- editing -# a module under src/ does not mark outputs stale on its own. +# Every trigger but software-env, which hashes the image *path*: that path is +# per person (~/.cache/sp_validation/...) and switches between SIF and sandbox, +# so it would rerun everything without the stack having changed. +# "code" watches rule bodies and `script:` files, not `src/`. rerun-triggers: ["mtime", "params", "input", "code"] -# Give an appearing output file a moment on networked filesystems before -# Snakemake calls a job failed for a missing output. A job's write can take -# more than 5 s to show on the launching host (candide's /home does); a -# shorter wait reads a finished job as failed and re-runs it. +# A job's write can take more than 5 s to show on the launching host (candide's +# /home does); a shorter wait reads a finished job as failed and re-runs it. latency-wait: 60 -# --- end GENERIC ------------------------------------------------------------ - -# Binds are the one thing you almost certainly need to edit for your machine: -# whatever paths your inputs, outputs and checkout live under. `--cleanenv` so a -# job's environment is the image's, not your shell's. If $HOME and the working -# directory cover everything (apptainer mounts both by default), drop `--bind`. -apptainer-args: "--cleanenv --bind /home" +# --- end mirrored ----------------------------------------------------------- -# No `apptainer-prefix`, here or on candide: the entry Snakefiles resolve -# `container:` to this user's own image path (workflow/README.md). +# `--cleanenv` so a job's environment is the image's, not your shell's. +# Apptainer mounts $HOME and the working directory; add `--bind` for any other +# disk your inputs, outputs or checkout live on. +apptainer-args: "--cleanenv" From a833eace5b78108d75bb0703f4ebb2ea82a3ae22 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 05:56:04 +0200 Subject: [PATCH 132/160] workflow/tests: keep the sweep drivers' image-resolution test It guards the baseline's image resolution, not blinding, and runs on any host. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- workflow/tests/test_image_resolution.py | 36 +++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 workflow/tests/test_image_resolution.py diff --git a/workflow/tests/test_image_resolution.py b/workflow/tests/test_image_resolution.py new file mode 100644 index 00000000..66687468 --- /dev/null +++ b/workflow/tests/test_image_resolution.py @@ -0,0 +1,36 @@ +"""Every launcher runs the image ``sp_validation/container.py`` resolves.""" + +import os +import subprocess + +import pytest +from conftest import REPO, container + +SWEEP_ENV = REPO / "papers" / "bmodes" / "scripts" / "container_env.sh" + + +@pytest.mark.parametrize("sandbox", [False, True], ids=["sif", "sandbox"]) +def test_sweep_drivers_run_the_resolved_image(tmp_path, monkeypatch, sandbox): + """The Paper II sweep drivers' CONTAINER is the image container.py resolves. + + Their shell copy of the resolution reads the same cache, whatever + XDG_CACHE_HOME says, and prefers the sandbox the same way. + """ + cache = tmp_path / "home" / ".cache" / "sp_validation" + cache.mkdir(parents=True) + (cache / "sp_validation.sif").touch() + if sandbox: + (cache / "sandbox").mkdir() + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "node-local")) + for var in ("SPV_CONTAINER", "SPV_SANDBOX"): + monkeypatch.delenv(var, raising=False) + + shell = subprocess.run( + ["bash", "-c", f'. "{SWEEP_ENV}" && printf %s "$CONTAINER"'], + env=dict(os.environ), + capture_output=True, + text=True, + check=True, + ) + assert shell.stdout == container.resolve_image()[0] From c63e732c9d0e45d8402e58717522319337432da6 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 05:56:48 +0200 Subject: [PATCH 133/160] workflow: jobs read the repo's matplotlibrc, never the launching user's Apptainer binds $HOME, so a job read the launcher's own matplotlibrc, and a LaTeX preamble there the image cannot typeset stopped every figure rule. common.py points each job's MATPLOTLIBRC (through APPTAINERENV_, past --cleanenv) at an empty workflow/matplotlibrc. The container smoke job reports the matplotlibrc it would read and the test asserts it is the repo's. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- workflow/README.md | 4 ++++ workflow/common.py | 7 +++++++ workflow/matplotlibrc | 2 ++ workflow/tests/data/container_smoke/container_smoke.py | 9 +++++++-- workflow/tests/test_container_smoke.py | 6 ++++++ 5 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 workflow/matplotlibrc diff --git a/workflow/README.md b/workflow/README.md index 6bcb186c..8c3ccc31 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -70,6 +70,10 @@ source cache off the shared filesystem), so a login shell that points it at `/scratch` is fine; keep any other path you export on a shared disk. Per-rule `mem_mb` / `runtime` stay on the rules. +Jobs draw their figures with matplotlib's defaults: `common.py` points each +job's `MATPLOTLIBRC` at `workflow/matplotlibrc`, so your own matplotlibrc +shapes your interactive work in the container but never a rule's figure. + ### Off candide — the default profile Candide is where the analysis runs, so the candide profile is the one to reach diff --git a/workflow/common.py b/workflow/common.py index 597f3954..7fd2ce08 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -39,6 +39,13 @@ # Without it, jobs use the home directory's cache, which every node mounts. os.environ.pop("XDG_CACHE_HOME", None) +# Jobs draw their figures with matplotlib's defaults, whoever launches them. +# Apptainer binds $HOME, so a job would otherwise read the launching user's +# ~/.config/matplotlib/matplotlibrc, and a LaTeX preamble there that the image +# cannot typeset stops every figure rule. MATPLOTLIBRC outranks that file, and +# APPTAINERENV_ carries it past --cleanenv. +os.environ["APPTAINERENV_MATPLOTLIBRC"] = str(REPO_ROOT / "workflow/matplotlibrc") + # Output roots are env-overridable so a reproduction run can write into a # fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL diff --git a/workflow/matplotlibrc b/workflow/matplotlibrc new file mode 100644 index 00000000..6a13d270 --- /dev/null +++ b/workflow/matplotlibrc @@ -0,0 +1,2 @@ +# The matplotlibrc every workflow job reads (common.py names it): empty, so a +# job's figures take matplotlib's defaults. diff --git a/workflow/tests/data/container_smoke/container_smoke.py b/workflow/tests/data/container_smoke/container_smoke.py index 73a0cebf..e6bcf63f 100644 --- a/workflow/tests/data/container_smoke/container_smoke.py +++ b/workflow/tests/data/container_smoke/container_smoke.py @@ -2,7 +2,7 @@ Cheap sanity check of the profile-driven container path -- same executor (slurm), same software-deployment-method (apptainer), same apptainer-args -binds, same container image every real rule uses. Five things it proves, each +binds, same container image every real rule uses. What it proves, each written to the output YAML: * the job really ran inside the image (``APPTAINER_CONTAINER``, set by @@ -14,7 +14,8 @@ * the numeric stack works (numpy eigh on a small fixed matrix). ``OMP_NUM_THREADS`` is recorded but not asserted -- see the assertions; * which commit of this checkout is running (git rev-parse from inside the - container -- proves /home is bound and usable, not just readable). + container -- proves /home is bound and usable, not just readable); + * which matplotlibrc the job's figures would read. Driven by the co-located Snakefile; the assertions on the output YAML live in workflow/tests/test_container_smoke.py (candide only). @@ -77,6 +78,9 @@ except (subprocess.CalledProcessError, FileNotFoundError) as exc: commit = f"unavailable ({exc})" +# --- the matplotlibrc a figure rule would read ----------------------------- +import matplotlib # noqa: E402 + provenance = { "repo_dir": repo_dir, "commit": commit, @@ -92,6 +96,7 @@ "snakemake": snakemake_info, "numeric": numeric_info, "provenance": provenance, + "matplotlibrc": matplotlib.matplotlib_fname(), }, f, sort_keys=False, diff --git a/workflow/tests/test_container_smoke.py b/workflow/tests/test_container_smoke.py index 5454d8d3..915d1a9a 100644 --- a/workflow/tests/test_container_smoke.py +++ b/workflow/tests/test_container_smoke.py @@ -90,6 +90,12 @@ def test_container_smoke(): # The job read the pickle with the Snakemake that wrote it. assert report["snakemake"]["version"] == snakemake.__version__, report["snakemake"] + # Figures read the workflow's matplotlibrc, never the user's. + assert ( + Path(report["matplotlibrc"]).resolve() + == (REPO / "workflow/matplotlibrc").resolve() + ), report["matplotlibrc"] + # The numeric stack agrees with the same computation run here. np.testing.assert_allclose( report["numeric"]["eigenvalues"], From 8d32fea892138039c8a613fa0624fdc76fe45b4b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 05:57:06 +0200 Subject: [PATCH 134/160] CONTRACTS: the host-side workflow imports only stdlib and snakemake workflow.common runs in the host Snakemake with no sp_validation installed, and loads container.py by path; container.py imports only the standard library, since it also runs as the spv-container CLI before any image exists. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- src/sp_validation/CONTRACTS | 4 ++++ workflow/CONTRACTS | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 src/sp_validation/CONTRACTS create mode 100644 workflow/CONTRACTS diff --git a/src/sp_validation/CONTRACTS b/src/sp_validation/CONTRACTS new file mode 100644 index 00000000..05409483 --- /dev/null +++ b/src/sp_validation/CONTRACTS @@ -0,0 +1,4 @@ +@sc container-is-stdlib +The image model is loaded by path by the host Snakemake and runs as the +spv-container CLI before any image exists. +only: sp_validation.container may import stdlib diff --git a/workflow/CONTRACTS b/workflow/CONTRACTS new file mode 100644 index 00000000..dee2fa88 --- /dev/null +++ b/workflow/CONTRACTS @@ -0,0 +1,4 @@ +@sc host-importable +The host Snakemake imports workflow.common with no sp_validation installed; +common loads the stdlib-only project modules it needs by file path. +only: workflow.common may import stdlib, snakemake, snakemake.* From 5020f13211bc805b24c8cd4a13337585e06b9995 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 05:59:40 +0200 Subject: [PATCH 135/160] profiles/default: bind /home; Snakemake's --home keeps apptainer from mounting it A job's home is its working directory, so a checkout or output tree under the launching user's home was invisible to the job: the toy run's jobs imported the image's sp_validation instead of the checkout's src/. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- workflow/profiles/default/config.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/workflow/profiles/default/config.yaml b/workflow/profiles/default/config.yaml index b7fc1b22..419b88a4 100644 --- a/workflow/profiles/default/config.yaml +++ b/workflow/profiles/default/config.yaml @@ -23,6 +23,7 @@ latency-wait: 60 # --- end mirrored ----------------------------------------------------------- # `--cleanenv` so a job's environment is the image's, not your shell's. -# Apptainer mounts $HOME and the working directory; add `--bind` for any other +# Snakemake runs each job with `--home `, so apptainer mounts +# that directory and not your home: bind /home, and add `--bind` for any other # disk your inputs, outputs or checkout live on. -apptainer-args: "--cleanenv" +apptainer-args: "--cleanenv --bind /home" From 6073486aeb147645872176821ed4c45e7a2db857 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 00:01:19 +0200 Subject: [PATCH 136/160] =?UTF-8?q?test=5Fcosmo=5Fval:=20=CE=BE=C2=B1=20do?= =?UTF-8?q?es=20not=20depend=20on=20TreeCorr's=20thread=20count?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_calculate_2pcf_does_not_depend_on_thread_count guards the min_top pin in treecorr_config: calculate_2pcf's ξ± part, measured on 4 and on 48 threads from fresh Catalogs split at the same persisted patch centres, must agree below 1e-6σ at production binning. Without the pin it moves by 0.038σ. Co-Authored-By: Claude Opus 5.5 --- src/sp_validation/tests/test_cosmo_val.py | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 5cdc25c5..a06ecda4 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -310,6 +310,40 @@ def test_calculate_2pcf_runs_on_synthetic_catalog(self, tmp_path): # The additive-bias subtraction in the pipeline must have run. assert version in cv.c1 and version in cv.c2 + def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): + """calculate_2pcf's ξ± is the same on 4 and on 48 TreeCorr threads. + + Production binning (default bin_slop/angle_slop), both runs on the same + persisted jackknife patches, each from a fresh Catalog; they must agree + to far below the jackknife σ. + """ + import treecorr + + params, version = write_synthetic_catalogs( + tmp_path, n_gal=4000, coherent_shear=True + ) + cv = CosmologyValidation( + versions=[version], + npatch=8, + theta_min=15.0, + theta_max=70.0, + nbins=6, + **params, + ) + cv.write_patch_centers(version, 8) + + xi = {} + for n_threads in (4, 48): + gg = sacc_io.xi_correlation( + cv.calculate_2pcf(version, num_threads=n_threads) + ) + assert treecorr.get_omp_threads() == n_threads # the count took effect + xi[n_threads] = np.concatenate([gg.xip, gg.xim]) + sigma = np.sqrt(np.concatenate([gg.varxip, gg.varxim])) + + shift = np.max(np.abs(xi[48] - xi[4]) / sigma) + assert shift < 1e-6, f"ξ± moves by {shift:.3g}σ between 4 and 48 threads" + def test_calculate_scale_dependent_leakage_runs_on_synthetic_catalog( self, tmp_path ): From e6e51d75af70469cf2efc5636f33afd7d7333b71 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 02:11:38 +0200 Subject: [PATCH 137/160] Patch centres: develop's reuse-or-write, written atomically calculate_2pcf splits at the output tree's {ver}_patches_npatch=N.dat and writes it when missing, as develop does, now through a temporary file and os.replace so a concurrent reader never sees a torn file. The hand-drawn per-base centres go: patch_centers.py, write_patch_centers, the patch_centers_sha256 part key, rule xi's patches input (common.patches_path and patches_input), run_2pcf's patch_centers, and their README section, tests and toy-run step. So do common._plain and the affinity num_threads default. All three live on feat/jackknife-patch-centres, which seeds the k-means so a fresh tree, a racing job and a variant split alike. Co-Authored-By: Claude Opus 5.5 --- src/sp_validation/blinding.py | 5 +- src/sp_validation/cosmo_val/core.py | 14 +--- src/sp_validation/cosmo_val/patch_centers.py | 33 -------- src/sp_validation/cosmo_val/pure_eb.py | 2 +- src/sp_validation/cosmo_val/real_space.py | 80 +++----------------- src/sp_validation/tests/test_cosmo_val.py | 8 +- src/sp_validation/tests/test_custody.py | 12 +-- workflow/README.md | 29 +------ workflow/common.py | 61 +-------------- workflow/rules/twopoint.smk | 5 +- workflow/scripts/run_2pcf.py | 9 +-- workflow/tests/conftest.py | 12 +-- workflow/tests/test_dag.py | 47 ++---------- workflow/tests/test_toy_run.py | 19 +---- 14 files changed, 40 insertions(+), 296 deletions(-) delete mode 100644 src/sp_validation/cosmo_val/patch_centers.py diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 34852dd6..852d0a56 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -618,9 +618,6 @@ def _audit_part(blinded, true, fiducial, hidden, tolerance): cov = _covariance(true) if not _same_covariance(_covariance(blinded), cov): problems.append("covariances differ") - centres = [x.metadata.get("patch_centers_sha256") for x in (blinded, true)] - if centres[0] != centres[1]: - problems.append(f"patch centres differ: {centres}") values = np.asarray(true.mean) delta = np.asarray(blinded.mean) - values @@ -656,7 +653,7 @@ def audit(name, *, archive, true_root, cat_config, out=None): The published seed must be the committed one. For each archived file, the live file at the same relative path must be stamped unblinded for the same catalogue. Its signal rows must match the archived ones in tags, tracers, - covariance and patch centres (numbers to a re-measurement's float noise), + covariance (numbers to a re-measurement's float noise), and blinded − true must equal the seed's shift on every ξ± and Cℓ_EE block to :data:`SHIFT_TOLERANCE` of the block's largest shift (:data:`SHIFT_TOLERANCE_OTHER_STACK` under another theory stack); Cℓ_BB and Cℓ_EB may not move, and diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index aa4b7bd6..65650b84 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -13,7 +13,7 @@ _get_pte_from_scale_cut, find_conservative_scale_cut_key, ) -from ..custody import base_catalogue, custody_of, registry_of, seed_path +from ..custody import custody_of, registry_of, seed_path from ..statistics import chi2_and_pte from ..version import __version__ from .catalog_characterization import CatalogCharacterizationMixin @@ -257,9 +257,6 @@ def __init__( # thread count (max(3, ceil(log2 n))) and ξ± depends on the machine. # 6 is what TreeCorr derives on candide's 48- and 64-CPU nodes. "min_top": 6, - # The CPUs this process may use; TreeCorr's own default is the - # node's count, whatever share of it the job holds. - "num_threads": len(os.sched_getaffinity(0)), } self.catalog_config_path = Path(catalog_config) @@ -460,15 +457,6 @@ def custody(self, version): self._declared, version, registry=registry_of(self.catalog_config_path) ) - def patch_centers_path(self, version, npatch): - """The jackknife patch centres ``version`` is measured on with ``npatch``. - - One file per base catalogue, under the output directory's ``patches/``: - a catalogue and its variants split at the same centres. - """ - base = base_catalogue(self._declared, version) - return self._output_path("patches", f"{base}_npatch={int(npatch)}.dat") - def basename(self, version, treecorr_config=None, npatch=None): cfg = treecorr_config or self.treecorr_config patches = npatch or self.npatch diff --git a/src/sp_validation/cosmo_val/patch_centers.py b/src/sp_validation/cosmo_val/patch_centers.py deleted file mode 100644 index 54275be7..00000000 --- a/src/sp_validation/cosmo_val/patch_centers.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Draw a base catalogue's jackknife patch centres into an output tree, once. - - python -m sp_validation.cosmo_val.patch_centers \\ - --cat-config cosmo_val/cat_config.yaml --output-dir - -writes ``/patches/_npatch=.dat`` -(:meth:`CosmologyValidation.write_patch_centers`). It reads the whole -catalogue, so run it on a compute node. -""" - -import argparse - -from .core import CosmologyValidation - - -def main(argv=None): - parser = argparse.ArgumentParser( - prog="python -m sp_validation.cosmo_val.patch_centers", - description=__doc__.split("\n")[0], - ) - parser.add_argument("catalogue", help="a base catalogue of the catalogue config") - parser.add_argument("npatch", type=int) - parser.add_argument("--cat-config", required=True) - parser.add_argument("--output-dir", required=True, help="the output tree") - a = parser.parse_args(argv) - cv = CosmologyValidation( - versions=[a.catalogue], catalog_config=a.cat_config, output_dir=a.output_dir - ) - print(cv.write_patch_centers(a.catalogue, a.npatch)) - - -if __name__ == "__main__": - main() diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index cc6515a5..52ca5e45 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -98,7 +98,7 @@ def calculate_pure_eb( Notes ----- - Both binnings are the version's sealed ξ± parts - (:meth:`calculate_2pcf`), split at its persisted patch centres, so a + (:meth:`calculate_2pcf`), split at the same patch centres, so a blinded catalogue's modes are concealed. The jackknife covariance is the integration part's, pushed through the kernel (:func:`~sp_validation.b_modes.pure_eb_covariance_from_xi`): the diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 6f19ee24..62518b75 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -6,8 +6,8 @@ and plots. It depends on TreeCorr. """ -import hashlib import os +import uuid import matplotlib.pyplot as plt import matplotlib.ticker as mticker @@ -16,7 +16,6 @@ from cs_util import plots as cs_plots from .. import sacc_io -from ..custody import base_catalogue from .sacc_writers import xi_to_sacc @@ -27,7 +26,6 @@ def calculate_2pcf( *, grid="reporting", npatch=None, - patch_centers=None, out=None, **treecorr_config, ): @@ -40,14 +38,9 @@ def calculate_2pcf( blind is opened first, so a blind that cannot open fails before TreeCorr runs. - @sc patch-centres-are-inputs - With patches, the catalogue splits at persisted centres, one file per - base catalogue (:meth:`patch_centers_path`), never at centres drawn - here: the full-sample ξ± depends on the patch layout, and TreeCorr's - k-means cannot be reproduced. So the centres are drawn once, by hand - (``python -m sp_validation.cosmo_val.patch_centers``, which never - replaces a file), and never by a workflow rule, which a forced or - re-triggered run would re-draw. The part names the file by its sha256. + With patches, the catalogue splits at the output tree's + ``{ver}_patches_npatch={npatch}.dat``, drawn by TreeCorr's k-means and + written there when missing. Parameters: ver (str): The catalogue version to measure. @@ -55,8 +48,6 @@ def calculate_2pcf( npatch (int, optional): Jackknife patches; the instance's ``npatch`` by default. With patches the part carries the jackknife covariance, without them the shot-noise diagonal. - patch_centers (str, optional): The centres file; by default - :meth:`patch_centers_path`. out (str, optional): Where to write the part as well. **treecorr_config: Overrides of the instance's ``treecorr_config``, e.g. ``min_sep=1``. @@ -73,11 +64,8 @@ def calculate_2pcf( blinding.open_blind(custody) metadata = {**self.sacc_metadata(ver), "npatch": npatch} - patch_centers = self._patch_centers(ver, npatch, patch_centers) - if patch_centers is not None: - with open(patch_centers, "rb") as f: - metadata["patch_centers_sha256"] = hashlib.sha256(f.read()).hexdigest() jackknife = npatch > 1 + patch_file = self._output_path(f"{ver}_patches_npatch={npatch}.dat") gg = treecorr.GGCorrelation( { **self._binning(**treecorr_config), @@ -95,8 +83,14 @@ def calculate_2pcf( ra_units=self.treecorr_config["ra_units"], dec_units=self.treecorr_config["dec_units"], npatch=npatch, - patch_centers=patch_centers, + patch_centers=patch_file if os.path.exists(patch_file) else None, ) + # Through a temporary file, so a concurrent reader never sees a + # torn one. + if jackknife and not os.path.exists(patch_file): + tmp = f"{patch_file}.{uuid.uuid4().hex}.tmp" + catalogue.write_patch_centers(tmp) + os.replace(tmp, patch_file) gg.process(catalogue) s = xi_to_sacc( @@ -119,56 +113,6 @@ def calculate_2pcf( self.print_done("Done 2PCF") return part - def _patch_centers(self, ver, npatch, path=None): - """The existing centres file ``ver`` splits at, or None without patches.""" - if npatch <= 1: - return None - path = path or self.patch_centers_path(ver, npatch) - if not os.path.exists(path): - base = base_catalogue(self._declared, ver) - raise FileNotFoundError( - f"{ver} splits at {base}'s patch centres, and {path} does not " - "exist. Draw them once: python -m " - f"sp_validation.cosmo_val.patch_centers {base} {npatch} " - f"--cat-config {os.path.abspath(self.catalog_config_path)} " - f"--output-dir {self._output_path()}" - ) - return path - - def write_patch_centers(self, catalogue, npatch, path=None): - """Draw ``npatch`` jackknife patch centres for a base catalogue, once. - - TreeCorr's k-means over the catalogue's positions and weights, written - to ``path`` (default :meth:`patch_centers_path`). An existing file is - never replaced. - """ - base = base_catalogue(self._declared, catalogue) - if base != catalogue: - raise ValueError( - f"patch centres belong to {base}, the base catalogue of " - f"{catalogue}; write them from {base}" - ) - path = path or self.patch_centers_path(catalogue, npatch) - if os.path.exists(path): - raise FileExistsError( - f"{path} exists: {catalogue}'s patch centres are drawn once, and " - "its patched ξ± split at them. Delete the file to draw new ones." - ) - # A Catalog's k-means runs on TreeCorr's process-wide thread count. - treecorr.set_omp_threads(self.treecorr_config["num_threads"]) - with self.results[catalogue].temporarily_read_data(): - positions = treecorr.Catalog( - ra=self.results[catalogue].dat_shear["RA"], - dec=self.results[catalogue].dat_shear["Dec"], - w=self._read_shear_cols(catalogue, "w_col"), - ra_units=self.treecorr_config["ra_units"], - dec_units=self.treecorr_config["dec_units"], - npatch=int(npatch), - ) - os.makedirs(os.path.dirname(path), exist_ok=True) - positions.write_patch_centers(path) - return path - def _reporting_xi(self, ver): """``ver``'s reporting-grid ξ±, from its part, measured if not yet held.""" part = self.xi_parts.get((ver, "reporting")) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index a06ecda4..8d18b890 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -313,9 +313,9 @@ def test_calculate_2pcf_runs_on_synthetic_catalog(self, tmp_path): def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): """calculate_2pcf's ξ± is the same on 4 and on 48 TreeCorr threads. - Production binning (default bin_slop/angle_slop), both runs on the same - persisted jackknife patches, each from a fresh Catalog; they must agree - to far below the jackknife σ. + Production binning (default bin_slop/angle_slop), both runs on the + jackknife patches the first one writes, each from a fresh Catalog; they + must agree to far below the jackknife σ. """ import treecorr @@ -330,7 +330,6 @@ def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): nbins=6, **params, ) - cv.write_patch_centers(version, 8) xi = {} for n_threads in (4, 48): @@ -422,7 +421,6 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( **params, ) cv.treecorr_config.update(bin_slop=0, angle_slop=0) - cv.write_patch_centers(version, npatch) import treecorr diff --git a/src/sp_validation/tests/test_custody.py b/src/sp_validation/tests/test_custody.py index 65049bbb..76165d9a 100644 --- a/src/sp_validation/tests/test_custody.py +++ b/src/sp_validation/tests/test_custody.py @@ -178,12 +178,8 @@ def _toy_checkout(tmp_path): def test_host_and_job_resolve_the_same_custody(tmp_path): - """The host and a job agree on each version's custody and patch centres. - - `common.custody_token(v)` equals `CosmologyValidation.custody(v).token`, and - the centres the xi rule declares are the file the job splits at, one per - base catalogue. - """ + """The host and a job agree on each version's custody: + `common.custody_token(v)` equals `CosmologyValidation.custody(v).token`.""" from sp_validation.cosmo_val import CosmologyValidation root = _toy_checkout(tmp_path) @@ -202,8 +198,4 @@ def test_host_and_job_resolve_the_same_custody(tmp_path): ) for version in versions: assert common.custody_token(version) == cv.custody(version).token, version - assert common.patches_path(version, 100) == cv.patch_centers_path( - version, 100 - ), version assert common.custody_token("TOY").startswith("blinded:TOY:toy:") - assert len({common.patches_path(v, 100) for v in versions}) == 3 diff --git a/workflow/README.md b/workflow/README.md index 9aceeabd..2153f64f 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -166,21 +166,6 @@ mkdir -p /data ln -s /n17data/cdaley/unions/code/sp_validation/cosmo_inference/data/mask /data/ ``` -Jackknife patch centres are drawn once per base catalogue into -`COSMO_VAL/patches/`, and every patched ξ± measurement of the catalogue and its -variants splits at them. TreeCorr's k-means cannot be reproduced, so no rule -draws them: `-F` re-measures on the same centres, and a launch that needs -centres its tree lacks stops with the command that draws them, to run once on a -compute node: - -```bash -APPTAINERENV_PYTHONPATH=$PWD/src spv-container exec python -m sp_validation.cosmo_val.patch_centers \ - --cat-config cosmo_val/cat_config.yaml --output-dir -``` - -Or copy the file from the tree whose ξ± yours should match. The command never -replaces a file; to re-draw, delete it on purpose. - **Caveat:** `rerun-triggers: code` watches rule bodies and `script:` files, not `src/`. Editing a module under `src/` does not by itself mark outputs stale — force with `-F` or `--forcerun `. @@ -191,7 +176,8 @@ To reproduce a run from the image alone, opt out: snakemake --profile workflow/profiles/candide --config checkout_pythonpath=false ``` -Either way the checkout has to sit on a disk the jobs see (next section). +Either way the checkout has to sit under one of the profile's bind mounts to be +visible inside the job. Most of the time this default is all you need. Reach for a different *image* only when the dependency stack changed — a new package, a lockfile bump — not @@ -205,15 +191,8 @@ directory. `/automnt/nXXdataN` works only from a node that does *not* own that disk. On the owning node the disk is mounted directly at `/nXXdataN` and there is no `/automnt/nXXdataN` entry at all, so a job that lands there dies about one second after the allocation starts, before any log file is written. This is why -`n17` is in the profile's exclude list. `common.py` spells the launched -checkout, `COSMO_VAL` and `COSMO_INFERENCE` in the plain form whatever spelling -it is given (a symlink, a relative path, `/automnt`), and Snakemake matches a -target by its path string, so name file targets in the plain form too; keep new -paths the same. A job sees a plain path only on a disk the profile binds by -name — `/home`, `/n17data`, `/n23data1`, `/n09data` (the `/automnt` bind does -not serve it) — so the checkout and both output roots sit on one of those. To -work on another disk, add it to both bind lists: the candide profile's -`apptainer-args` and `container.DEFAULT_BINDS`. +`n17` is in the profile's exclude list. Every canonical path in `common.py` +already uses the plain form; keep new paths the same. ### Run Snakemake from the host, never from inside the container diff --git a/workflow/common.py b/workflow/common.py index 539c61e9..55c214e0 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -12,25 +12,8 @@ import sys from pathlib import Path - -def _plain(path): - """``path`` resolved, in the spelling every candide node can reach. - - A data disk is mounted at ``/nXXdataN`` on the node that owns it and - reached from every other node through ``/nXXdataN -> /automnt/nXXdataN``; - the owning node has no ``/automnt/nXXdataN``. A resolved path under - ``/automnt/`` is therefore spelled back under ``/``, whatever - the host: a job step re-derives its paths on its own node, and must - derive the launch's. - """ - path = Path(path).resolve() - if len(path.parts) > 2 and path.parts[1] == "automnt": - return Path("/", *path.parts[2:]) - return path - - # The checkout this workflow was launched from: workflow/common.py -> . -REPO_ROOT = _plain(__file__).parents[1] +REPO_ROOT = Path(__file__).resolve().parent.parent REPO_SRC = REPO_ROOT / "src" @@ -74,12 +57,9 @@ def _load_checkout_module(name): # fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL # defaults to the launched checkout's own (gitignored) cosmo_val/output, so a # launch writes into another checkout's products only when it names that tree; -# COSMO_INFERENCE defaults to the shared tree on candide. Snakemake keys its -# persistence records (params, input, code) and matches targets by path string, -# so both roots are spelled plain whatever spelling a launch gives, and a file -# target is named in that plain form. -COSMO_VAL = _plain(os.environ.get("COSMO_VAL", REPO_ROOT / "cosmo_val" / "output")) -COSMO_INFERENCE = _plain( +# COSMO_INFERENCE defaults to the shared tree on candide. +COSMO_VAL = Path(os.environ.get("COSMO_VAL", REPO_ROOT / "cosmo_val" / "output")) +COSMO_INFERENCE = Path( os.environ.get( "COSMO_INFERENCE", "/n17data/cdaley/unions/code/sp_validation/cosmo_inference" ) @@ -320,39 +300,6 @@ def base_version(version): return _custody.base_catalogue(CATALOG_CONFIG, version) -def patches_path(version, npatch): - """The jackknife patch centres ``version`` is measured on with ``npatch``. - - One file per base catalogue, so a catalogue and its variants split alike. - """ - name = f"{base_version(version)}_npatch={int(npatch)}.dat" - return str(COSMO_VAL / "patches" / name) - - -def patches_input(version, npatch): - """Rule xi's centres input for ``version`` with ``npatch``; none unpatched. - - No rule draws centres (``patch-centres-are-inputs``), so a missing file - stops the launch with the command that draws it. - """ - if int(npatch) <= 1: - return [] - path = patches_path(version, npatch) - if os.path.exists(path): - return path - from snakemake.exceptions import WorkflowError - - base = base_version(version) - raise WorkflowError( - f"{version} splits at {base}'s jackknife patch centres, {path}, which do " - "not exist; no rule draws them. Draw them once, on a compute node:\n" - f" APPTAINERENV_PYTHONPATH={REPO_SRC} spv-container exec python -m " - f"sp_validation.cosmo_val.patch_centers {base} {int(npatch)} " - f"--cat-config {CAT_CONFIG} --output-dir {COSMO_VAL}\n" - f"or copy {base}'s centres from the output tree whose ξ± yours should match." - ) - - def catalogue_entry(version): """The catalogue-config entry describing ``version`` (its own, or the one its ``_leak_corr``/``_seed`` variant chain reaches).""" diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 42364eeb..82a43614 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -17,13 +17,10 @@ rule xi: """TreeCorr ξ±(θ) for one version on one angular grid, as its sealed SACC part. One rule for every grid: outputs are named by their binning, so a request - binds the wildcards and the grid label resolves from them. With patches, - the measurement splits at the base catalogue's persisted centres, drawn by - hand (common.patches_input). + binds the wildcards and the grid label resolves from them. """ input: catalog=lambda w: shear_catalog(w.version), - patches=lambda w: patches_input(w.version, w.npatch), output: sacc=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 24 diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index ca431a66..c79ee191 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -15,9 +15,7 @@ The measurement is binning-agnostic: the reporting and the fine integration grids are the same compute with different ``--min-sep/--max-sep/--nbins``. The ξ± is born as a SACC part, named by its binning, tagged with its ``--grid`` and -sealed under the catalogue's custody by ``CosmologyValidation.calculate_2pcf``; -nothing else is written. With patches, the measurement splits at the base -catalogue's persisted centres (``python -m sp_validation.cosmo_val.patch_centers``). +sealed under the catalogue's custody by ``CosmologyValidation.calculate_2pcf``. ``output_dir`` is passed explicitly so lc can point each run at its own ``{output}`` tree. @@ -41,7 +39,6 @@ def run_2pcf( sacc_out=None, grid="reporting", custody=None, - patch_centers=None, ): """Measure ξ±(θ) for ``ver`` and write its sealed SACC part. @@ -54,8 +51,6 @@ def run_2pcf( name under the resolved output directory for the CLI path. ``custody`` is the custody token Snakemake resolved for ``ver`` (the rule's ``params.custody``); the part is not written under any other. - ``patch_centers`` is the centres file to split at, by default the base - catalogue's under ``output_dir``. Returns ------- @@ -75,7 +70,6 @@ def run_2pcf( ver, grid=grid, npatch=npatch, - patch_centers=patch_centers, out=out_path, min_sep=min_sep, max_sep=max_sep, @@ -98,7 +92,6 @@ def _from_snakemake(smk): grid=p.get("grid", "reporting"), sacc_out=smk.output["sacc"], custody=p["custody"], - patch_centers=smk.input.get("patches") or None, ) diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py index 7d8edfd2..6d00ae4f 100644 --- a/workflow/tests/conftest.py +++ b/workflow/tests/conftest.py @@ -12,8 +12,8 @@ The ``toy`` fixture is a disposable checkout: copies of ``workflow/`` and ``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, one catalogue per custody state, hand-written blind records (the host never decrypts), stand-ins -for the processed CosmoCov covariances (their inputs live on candide) and the -patch centres, and both output roots in tmp. +for the processed CosmoCov covariances (their inputs live on candide), and both +output roots in tmp. """ import dataclasses @@ -244,14 +244,6 @@ def toy(tmp_path_factory): path.parent.mkdir(parents=True, exist_ok=True) path.touch() - # The toy catalogue's patch centres, drawn by hand, in both output roots a - # launch may name. - npatch = grids["reporting"]["npatch"] - for cosmo_val in (Path(env["COSMO_VAL"]), root / "cosmo_val" / "output"): - centres = cosmo_val / "patches" / f"{VERSIONS[0]}_npatch={npatch}.dat" - centres.parent.mkdir(parents=True) - centres.touch() - return Toy( root=root, rundir=rundir, diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 3e8077a7..494c9d90 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -13,7 +13,6 @@ STALE, UNCOVERED, VERSIONS, - _load_module, on_candide, parse_jobs, ) @@ -61,22 +60,6 @@ def test_one_integration_grid(toy, forced, grids): assert {part, covariance} <= by_rule["cv_pure_eb"], by_rule["cv_pure_eb"] -def test_patch_centres_are_an_input_no_rule_draws(toy, forced, grids): - """[P13] Every patched ξ± job, the variant's too, splits at its base - catalogue's centres, and even a forced run writes none.""" - jobs = forced[1] - npatch = grids["reporting"]["npatch"] - assert npatch > 1 - centres = str(toy.cosmo_val / "patches" / f"{VERSIONS[0]}_npatch={npatch}.dat") - for job in [j for j in jobs if j.rule == "xi"]: - patched = int(job.wildcards["npatch"]) > 1 - assert [f for f in job.input if "/patches/" in f] == ( - [centres] if patched else [] - ), job - drawn = [f for j in jobs for f in j.output if "/patches/" in f] - assert not drawn, drawn - - def test_custody_is_the_checkouts_and_no_rule_touches_a_blind(toy, forced, tmp_path): """[P9, P10, P8] A catalogue and its variant share one custody line; even a forced run schedules nothing that reads or writes the registry; and a @@ -208,26 +191,9 @@ def test_a_job_needs_no_launch_cache(toy, tmp_path): assert (tmp_path / "env.txt").read_text() == "" -def _stand_in_centres(paper, cosmo_val): - """Touch the centres ``paper``'s patched ξ± grids read; a dry-run reads none.""" - common = _load_module( - REPO / "workflow" / "common.py", f"{paper}_common", {"COSMO_VAL": cosmo_val} - ) - common.CATALOG_CONFIG = yaml.safe_load(Path(common.CAT_CONFIG).read_text()) - config = yaml.safe_load( - (REPO / "papers" / paper / "config" / "config.yaml").read_text() - ) - for grid in common.xi_grids(config, config["fiducial"]).values(): - for version in config["versions"] if grid["npatch"] > 1 else (): - centres = Path(common.patches_path(version, grid["npatch"])) - centres.parent.mkdir(parents=True, exist_ok=True) - centres.touch() - - -def _real_dry_run(paper, targets, cosmo_val): +def _real_dry_run(paper, targets): env = {k: v for k, v in os.environ.items() if k != "SNAKEMAKE_PROFILE"} - env.update(PYTHONUNBUFFERED="1", PYTHONNOUSERSITE="1", COSMO_VAL=cosmo_val) - _stand_in_centres(paper, cosmo_val) + env.update(PYTHONUNBUFFERED="1", PYTHONNOUSERSITE="1") return subprocess.run( [ sys.executable, @@ -258,10 +224,7 @@ def _real_dry_run(paper, targets, cosmo_val): ], ids=["cosmo_val", "bmodes"], ) -def test_papers_resolve_on_candide(paper, targets, tmp_path): - """The real paper DAGs resolve against the real catalogues and your image. - - The products land in a tree of their own, holding stand-in patch centres. - """ - result = _real_dry_run(paper, targets, str(tmp_path)) +def test_papers_resolve_on_candide(paper, targets): + """The real paper DAGs resolve against the real catalogues and your image.""" + result = _real_dry_run(paper, targets) assert result.returncode == 0, result.stdout diff --git a/workflow/tests/test_toy_run.py b/workflow/tests/test_toy_run.py index 9d4a9912..b81050a4 100644 --- a/workflow/tests/test_toy_run.py +++ b/workflow/tests/test_toy_run.py @@ -4,15 +4,13 @@ host Snakemake, the image `spv-container` manages, jobs on this node. A toy checkout under your home directory (which the profile binds) carries copies of workflow/, papers/cosmo_val/ and src/, and one synthetic catalogue declared -unblinded. The first launch stops for want of its patch centres and names the -command that draws them; after it has run, its reporting parts and their figure -are made; then the catalogue is declared blinded, a blind is drawn for it, and -the same launch re-measures the parts concealed. +unblinded. The first launch makes its reporting parts and their figure; then +the catalogue is declared blinded, a blind is drawn for it, and the same launch +re-measures the parts concealed. """ import json import os -import shlex import shutil import subprocess import sys @@ -143,17 +141,6 @@ def launch(*args): check=False, ) - # No centres: the launch names the command that draws them, which works. - result = launch() - assert result.returncode != 0, result.stdout - (command,) = [ - line.split("spv-container exec ", 1)[1] - for line in result.stdout.splitlines() - if "sp_validation.cosmo_val.patch_centers" in line - ] - _in_image(image, root, *shlex.split(command)) - assert (out / "patches" / "SP_v0.1_npatch=4.dat").is_file() - # Declared unblinded: parts stamped unblinded, and their figure drawn. result = launch() assert result.returncode == 0, result.stdout From 507e68da770352e636d397dbdcc2f4affed000f0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 02:40:08 +0200 Subject: [PATCH 138/160] Custody is one `blind:` per catalogue entry; a blind is one record outside git Every cat_config entry declares `blind: none | mock | `, and a missing declaration is refused. Entries reading one shear file declare one blind, with the repository config authoritative for the files it names, and a version set may show a blinded catalogue only beside mocks and its own blind (checked at launch and in CosmologyValidation). The registry is `paths.blinds`, refused inside a git worktree unless it is that worktree's root; `*.blind.json` is gitignored. A blind is `.blind.json`: seed, envelope, fiducial and draw scheme, written once at 0440, with a commitment over the whole canonical record. Jobs take custody from their params token (`CosmologyValidation( custody=)`, `assemble_sacc`), and `open_blind` checks the record against the token's commitment and the fork's draw scheme. `Blind` holds a name and a path; `_hidden()` returns a mapping whose repr is ``, and a theory failure at the hidden point is reported by exception type alone. `conceal` refuses a zero or non-finite shift. Gone: `bases`/`share`/`base:` and the twin checks, Fernet and the key, BlindingConfig and its digest, init staging, `confirm`, `verify`, `reveal`, `audit`, the `cosmo_val.type` refusal and the `cryptography` dependency. `xip_xim.py` refuses a blinded shear file. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 3 + CLAUDE.md | 6 + cosmo_val/cat_config.yaml | 72 +- pyproject.toml | 5 - scripts/xip_xim.py | 9 + src/sp_validation/blinding.py | 877 ++++-------------- src/sp_validation/cosmo_val/core.py | 32 +- src/sp_validation/cosmo_val/real_space.py | 2 +- src/sp_validation/custody.py | 579 ++++-------- src/sp_validation/sacc_io.py | 14 +- src/sp_validation/tests/_synthetic.py | 23 +- src/sp_validation/tests/test_assemble_sacc.py | 2 +- src/sp_validation/tests/test_blinding.py | 224 ++--- .../tests/test_config_paths_exist.py | 5 +- src/sp_validation/tests/test_cosmo_val.py | 50 +- src/sp_validation/tests/test_custody.py | 290 +++--- src/sp_validation/tests/test_custody_e2e.py | 82 +- .../tests/test_cv_init_params.py | 4 +- src/sp_validation/tests/test_pseudo_cl.py | 2 +- src/sp_validation/tests/test_sacc_io.py | 2 +- src/sp_validation/tests/test_sacc_writers.py | 2 +- uv.lock | 2 - workflow/README.md | 70 +- workflow/common.py | 52 +- workflow/rules/cosmo_val.smk | 1 - workflow/scripts/assemble_sacc.py | 17 +- workflow/scripts/generate_pseudo_cl.py | 6 +- workflow/scripts/run_2pcf.py | 10 +- workflow/scripts/run_rho_tau.py | 3 +- workflow/tests/conftest.py | 49 +- workflow/tests/test_dag.py | 55 +- workflow/tests/test_toy_run.py | 52 +- 32 files changed, 863 insertions(+), 1739 deletions(-) diff --git a/.gitignore b/.gitignore index 9c07b05b..388b38f7 100644 --- a/.gitignore +++ b/.gitignore @@ -206,3 +206,6 @@ report.md check2.txt format2.txt residual.md + +# Blind records hold a secret seed and never enter git. +*.blind.json diff --git a/CLAUDE.md b/CLAUDE.md index b4447758..48db4a4e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -79,6 +79,12 @@ Main configuration in `scripts/calibration/params.py` with parameters: - emcee for MCMC sampling - pyccl for cosmological calculations +## Blinded catalogues +Each `cosmo_val/cat_config.yaml` entry declares `blind: none`, `mock` or a blind's name. +- For an entry with `blind: `, measure signal (ξ±, Cℓ, γt, COSEBIs, M_ap, maps) only through `CosmologyValidation` or the workflow, never from the file directly. +- Never set `blind: none` to get a run through. +- Never print, paste or commit a `.blind.json`. + ## Container Usage Nothing is hand-built. CI publishes `ghcr.io/cosmostat/sp_validation:` on every push, and each person keeps their own copy at diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index bcab108c..a19357f2 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -1,9 +1,7 @@ -# One entry per catalogue. `blinding:` declares a base catalogue's custody: -# blinded, unblinded or mock; an entry that declares none is blinded. A variant -# stored as its own entry names its parent with `base:` and shares the parent's -# custody and blind. See src/sp_validation/custody.py. +# One entry per catalogue. `blind:` is its custody: `none` (public), `mock`, or +# the name of the blind its signal is concealed under. See custody.py. DES: - blinding: unblinded + blind: none subdir: /n17data/mkilbing/astro/data/DES pipeline: SP colour: orange @@ -45,7 +43,7 @@ DES: path: psf_y3a1-v29.fits patch_number: 120 SP_axel_v0.0: - blinding: unblinded + blind: none subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 pipeline: SP colour: cyan @@ -86,7 +84,7 @@ SP_axel_v0.0: path: star_cat.fits patch_number: 120 SP_v0.1.1: - blinding: unblinded + blind: none subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 pipeline: SP colour: cyan @@ -121,7 +119,7 @@ SP_v0.1.1: path: star_cat.fits patch_number: 150 SP_v1.3: - blinding: unblinded + blind: none subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe pipeline: SP colour: green @@ -158,7 +156,7 @@ SP_v1.3: path: unions_shapepipe_star_2022_v1.0.3.fits patch_number: 150 SP_v1.3.6: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.3.x pipeline: SP colour: coral @@ -203,7 +201,7 @@ SP_v1.3.6: path: unions_shapepipe_star_2022_v1.0.3.fits patch_number: 150 SP_v1.4.5: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: dodgerblue @@ -248,7 +246,7 @@ SP_v1.4.5: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.5.A: - blinding: unblinded + blind: none subdir: /n17data/guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5 pipeline: SP colour: red @@ -290,7 +288,7 @@ SP_v1.4.5.A: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.5_bright: - blinding: unblinded + blind: none subdir: /n17data/murray/unions_cats pipeline: SP colour: mediumblue @@ -332,7 +330,7 @@ SP_v1.4.5_bright: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.5_faint: - blinding: unblinded + blind: none subdir: /n17data/murray/unions_cats pipeline: SP colour: forestgreen @@ -374,7 +372,7 @@ SP_v1.4.5_faint: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6_glass_mock: - blinding: mock + blind: mock subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: darkgreen @@ -418,7 +416,7 @@ SP_v1.4.6_glass_mock: path: unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.5_intermediate: - blinding: unblinded + blind: none subdir: /n17data/murray/unions_cats pipeline: SP colour: orange @@ -460,7 +458,7 @@ SP_v1.4.5_intermediate: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: green @@ -505,7 +503,7 @@ SP_v1.4.6: path: unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.3: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: orange @@ -550,7 +548,7 @@ SP_v1.4.6.3: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.3_B: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: green @@ -595,7 +593,7 @@ SP_v1.4.6.3_B: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.3_C: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: green @@ -640,7 +638,7 @@ SP_v1.4.6.3_C: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.3_A: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: green @@ -685,7 +683,7 @@ SP_v1.4.6.3_A: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6_ecut07: - base: SP_v1.4.6 + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: darkgreen @@ -730,7 +728,7 @@ SP_v1.4.6_ecut07: path: unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.1: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: red @@ -775,7 +773,7 @@ SP_v1.4.6.1: path: unions_shapepipe_star_2024_v1.4.a.fits patch_number: 100 SP_v1.4.7: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: darkorchid @@ -822,7 +820,7 @@ SP_v1.4.7: path: unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.8: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: darkorchid @@ -867,7 +865,7 @@ SP_v1.4.8: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.11.2: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: orange @@ -912,7 +910,7 @@ SP_v1.4.11.2: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.11.3: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: red @@ -957,7 +955,7 @@ SP_v1.4.11.3: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.12.3: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: lightblue @@ -1003,7 +1001,7 @@ SP_v1.4.12.3: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.13.3: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: cyan @@ -1049,7 +1047,7 @@ SP_v1.4.13.3: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.11.3_ecut07: - base: SP_v1.4.11.3 + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: darkblue @@ -1094,7 +1092,7 @@ SP_v1.4.11.3_ecut07: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6.3_uncal: - blinding: unblinded + blind: none pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: @@ -1110,7 +1108,7 @@ SP_v1.4.6.3_uncal: hdu: 1 patch_number: 100 SP_v1.4.6.3_uncal_w_iv: - blinding: unblinded + blind: none pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: @@ -1126,7 +1124,7 @@ SP_v1.4.6.3_uncal_w_iv: hdu: 1 patch_number: 100 SP_v1.4.6.3_uncal_w_1: - blinding: unblinded + blind: none pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: @@ -1142,7 +1140,7 @@ SP_v1.4.6.3_uncal_w_1: hdu: 1 patch_number: 100 SP_v1.4.5_uncal: - blinding: unblinded + blind: none pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: @@ -1158,7 +1156,7 @@ SP_v1.4.5_uncal: hdu: 1 patch_number: 100 SP_v1.4.7_uncal: - blinding: unblinded + blind: none pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: @@ -1174,7 +1172,7 @@ SP_v1.4.7_uncal: hdu: 1 patch_number: 100 SP_v1.4.8_uncal: - blinding: unblinded + blind: none pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: @@ -1196,8 +1194,10 @@ nz: path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_{pipeline}_A.txt paths: output: ./output + # The blind registry: .blind.json records, outside any git worktree. + blinds: /n17data/UNIONS/WL/blinds SP_v1.6.6: - blinding: unblinded + blind: none subdir: /n17data/UNIONS/WL/v1.6.x pipeline: SP colour: violet diff --git a/pyproject.toml b/pyproject.toml index 3e69c174..98166f2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,11 +28,6 @@ dependencies = [ "camb>=1.6", "clmm", "colorama", - # Blinding closure (core, no extra): cryptography + sacc here, plus the - # Smokescreen fork pin below; pyccl (the theory backend) is already core. - # cryptography and sacc are declared explicitly so the core runtime - # closure is self-documenting and independent of fork-metadata drift. - "cryptography", # Track cs_util's develop branch directly (git dependency) rather than a # PyPI pin: the two repos are iterating together heavily and cs_util # releases are infrequent. This PR's cosmology repoint needs get_cosmo / diff --git a/scripts/xip_xim.py b/scripts/xip_xim.py index 1345f01a..9713745c 100755 --- a/scripts/xip_xim.py +++ b/scripts/xip_xim.py @@ -15,6 +15,8 @@ from astropy.io import fits from cs_util import logging +from sp_validation.custody import MOCK, NONE, of_file + def params_default(): @@ -121,6 +123,13 @@ def main(argv=None): # Save calling command logging.log_command(argv) + blinds = of_file(params["input_path"]) - {NONE, MOCK} + if blinds: + raise SystemExit( + f"{params['input_path']} is blinded under {', '.join(sorted(blinds))}: " + "measure its signal through CosmologyValidation or the workflow" + ) + # Open input catalogue if params["verbose"]: print(f"Reading catalogue {params['input_path']}...") diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 852d0a56..42af6520 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -1,129 +1,51 @@ -"""The blind: one secret seed, hence one hidden cosmology, and its custody. - -:Name: blinding.py - -:Description: A blind conceals a catalogue's cosmological signal (Muir et al. - 2019): every ξ± and pseudo-Cℓ_EE row is shifted by t(hidden) − t(fiducial), - the theory difference between a hidden cosmology and the fiducial, before - the row is first written (:func:`sp_validation.sacc_io.seal`). The hidden - point is drawn uniformly in S8 and Ωm about the fiducial from a secret seed, - with the Smokescreen fork's per-key RNG. COSEBIs and pure-E/B computed from - concealed ξ± are concealed with it; the shift is pure E-mode, so B-mode null - tests stay valid. - - The commands of ``python -m sp_validation.blinding`` are the only writers - of the blind registry, ``cosmo_val/blinds//``, which is committed to - git and which no Snakemake rule reads, writes or draws, so no run can - re-draw a blind and every checkout resolves the same one. ``init`` draws a - blind, once, by a person, writing read-only ``commitment.json`` (the public - record: the seed's commitment, the BlindingConfig and its digest, the draw - scheme and the theory stack), ``seed.fernet`` (the seed, encrypted) and - ``key``, and ``bases``, the base catalogues it covers; ``share`` appends a - catalogue to ``bases``; ``reveal`` writes ``revealed.json`` (the published - seed) once and moves the concealed files aside. ``audit`` checks the - re-measured true files against that archive, and ``verify`` checks a - file's stamp against its catalogue's custody, seedless. +"""The blind: a secret seed, drawn once, and the shift it conceals signal by. + +@sc blind-record +A blind is one read-only file, ``/.blind.json``, outside +any git worktree: its seed (unencrypted: registry access is blind access), the +envelope the hidden point is drawn in, the fiducial and the fork's draw scheme. +Nothing here prints the seed or the hidden cosmology: :class:`Blind` holds a +name and a path, :func:`_hidden` returns a mapping whose repr is +````, and a theory failure at the hidden point is reported +by exception type alone. + +@sc hidden-draw-uniform-s8-om +The hidden point is the fiducial moved by the fork's per-key draw from the +seed, uniform within the envelope in S8 and Ωm. + +``python -m sp_validation.blinding init `` draws a blind; ``show `` +prints its public record (everything but the seed). """ import argparse import dataclasses -import functools -import hashlib import json import os import secrets -import shutil -import sys -from datetime import datetime, timezone +import warnings +from collections.abc import Mapping from pathlib import Path import numpy as np -import yaml from . import custody as _custody from . import sacc_io from .blinding_theory import TheoryConfig, cl_ee, xi_ccl -_BLIND_NAME = "abcdefghijklmnopqrstuvwxyz0123456789-_." +ENVELOPE = {"S8": 0.075, "Omega_m": 0.1} +SECRET = "secret: never print, paste or commit" -# --------------------------------------------------------------------------- # -# The config a blind is drawn under, and its digest -# --------------------------------------------------------------------------- # -@dataclasses.dataclass(frozen=True) -class BlindingConfig: - """The envelope of the hidden draw and the theory the shift is computed in. - - ``envelope`` maps a :class:`TheoryConfig` field to the half-width of its - uniform draw about the fiducial ``theory``. - """ - - envelope: dict = dataclasses.field( - default_factory=lambda: {"S8": 0.075, "Omega_m": 0.1} - ) - theory: TheoryConfig = dataclasses.field(default_factory=TheoryConfig) - - def record(self): - """The config as plain data: every field, numbers as floats.""" - return _canonical(self) - - def digest(self): - """sha256 of the canonical record; int and float literals agree.""" - return record_digest(self.record()) - - -def record_digest(record): - """sha256 of a config record's canonical JSON, whatever the code's schema.""" - text = json.dumps(record, sort_keys=True, separators=(",", ":")) - return hashlib.sha256(text.encode("utf-8")).hexdigest() - - -def _canonical(value): - if dataclasses.is_dataclass(value): - return { - f.name: _canonical(getattr(value, f.name)) - for f in dataclasses.fields(value) - } - if isinstance(value, dict): - return {str(k): _canonical(v) for k, v in value.items()} - if isinstance(value, bool) or isinstance(value, str): - return value - if isinstance(value, (int, float, np.integer, np.floating)): - return float(value) - raise TypeError(f"cannot serialise {value!r} into a blind's record") - - -# A blind's record opens under this code only if it names every TheoryConfig -# field, or omits one listed here with the value under which that field changes -# no shift: a record drawn without it then reproduces its shift. -NEUTRAL = {} +class BlindingError(RuntimeError): + """Concealment failed; the message carries no hidden value.""" -def _recorded_config(name, record): - """The :class:`BlindingConfig` of blind ``name``, from its config record. +@dataclasses.dataclass(frozen=True) +class Blind: + """An opened blind: its name and the path of its record.""" - The one reader of the record format. Refuses, naming the fields, a record - whose shift this code cannot reproduce: one naming a field the code lacks, - or lacking a code field that has no value in :data:`NEUTRAL`. - """ - fields = {f.name for f in dataclasses.fields(TheoryConfig)} - theory = {**NEUTRAL, **record.get("theory", {})} - envelope = record.get("envelope", {}) - foreign = sorted(set(record) - {"envelope", "theory"}) - foreign += sorted((set(theory) | set(envelope)) - fields) - unset = sorted(fields - set(theory)) + ["envelope"] * ("envelope" not in record) - if foreign or unset: - raise _custody.CustodyError( - f"blind {name}'s config does not fit this code. Fields the record " - f"names and this code lacks: {foreign or 'none'}. Fields this code " - "has and the record lacks, with no value in blinding.NEUTRAL: " - f"{unset or 'none'}. A drawn blind opens under the code it was " - "drawn with, or once each new field has its neutral value." - ) - return BlindingConfig( - envelope={k: float(v) for k, v in envelope.items()}, - theory=TheoryConfig(**theory), - ) + name: str + path: Path def draw_scheme(): @@ -133,664 +55,193 @@ def draw_scheme(): return int(DRAW_SCHEME) -def hidden_theory(seed, config): - """The hidden point: the fiducial theory moved by the seed's draw. - - @sc hidden-draw-uniform-s8-om - The draw is uniform within the envelope in the physical coordinates it - names (S8 and Ωm), by the fork's per-key RNG; CCL's σ8 and Ω_c follow from - the drawn point through ``TheoryConfig.ccl_params``, never the reverse. - """ - from smokescreen.param_shifts import draw_param_shifts - - shift = draw_param_shifts(dict(config.envelope), seed) - return dataclasses.replace( - config.theory, - **{key: getattr(config.theory, key) + delta for key, delta in shift.items()}, - ) - - -# --------------------------------------------------------------------------- # -# Opening a blind -# --------------------------------------------------------------------------- # -@dataclasses.dataclass(frozen=True) -class Blind: - """An opened blind. Its ``repr`` hides the seed.""" - - name: str - seed: str = dataclasses.field(repr=False) - config: BlindingConfig - hidden: TheoryConfig - - -@functools.cache -def _open(registry, name): - path = Path(registry) / name - if not path.is_dir(): - raise _custody.CustodyError(f"no blind {name} in {registry}") - return _read_blind(path) - - -def _read_blind(path): - """The blind stored at directory ``path``, its sealed seed checked against - its public record.""" - from cryptography.fernet import Fernet, InvalidToken - - name = path.name - c = _custody.read_record(path).commitment +def open_blind(custody): + """The blind a blinded custody names, its record matching the commitment.""" + path = Path(custody.registry or "") / f"{custody.blind}.blind.json" try: - key = (path / "key").read_bytes() - ciphertext = (path / "seed.fernet").read_bytes() - payload = json.loads(Fernet(key).decrypt(ciphertext)) - except (OSError, ValueError, InvalidToken) as err: + record = json.loads(path.read_text()) + except (OSError, ValueError): raise _custody.CustodyError( - f"blind {name}: the seed does not decrypt with its key " - f"({type(err).__name__})" + f"cannot read blind {custody.blind} at {path}" ) from None - for what, values in { - "blind name": (payload["blind"], c["blind"], name), - "config digest": ( - payload["config_digest"], - c["config_digest"], - record_digest(c["config"]), - ), - "draw scheme": (payload["draw_scheme"], c["draw_scheme"]), - "seed commitment": ( - _custody.seed_commitment(payload["seed"]), - c["seed_commitment"], - ), - }.items(): - if len(set(values)) != 1: - raise _custody.CustodyError( - f"blind {name}: the {what} disagrees between the sealed seed and " - "the record; the record was edited" - ) - if c["draw_scheme"] != draw_scheme(): + if _custody.commitment(record) != custody.commitment: raise _custody.CustodyError( - f"blind {name} was drawn under draw scheme {c['draw_scheme']}; this " - f"install's smokescreen draws under {draw_scheme()}" + f"blind {custody.blind}'s record no longer matches the commitment this " + "run was launched under; launch again" ) - config = _recorded_config(name, c["config"]) - return Blind(name, payload["seed"], config, hidden_theory(payload["seed"], config)) - - -def open_blind(custody): - """The verified blind a blinded custody names (cached per process).""" - if custody.status != "blinded": - raise _custody.CustodyError(f"{custody.catalogue} is {custody.status}") - return _open(Path(custody.registry), custody.blind) - - -# --------------------------------------------------------------------------- # -# Blocks, from the content, and the concealment -# --------------------------------------------------------------------------- # -@dataclasses.dataclass(frozen=True) -class Block: - """Rows of one statistic and tracer pair, and their theory at a point.""" + if record["draw_scheme"] != draw_scheme(): + raise _custody.CustodyError( + f"blind {custody.blind} was drawn under draw scheme " + f"{record['draw_scheme']}; this smokescreen draws under {draw_scheme()}" + ) + return Blind(custody.blind, path) - name: str - rows: np.ndarray - theory: object # (ccl params, TheoryConfig) -> values aligned to rows +class _Hidden(Mapping): + def __init__(self, values): + self._values = values -def _pair_nz(s, tracers): - nzs = [] - for name in tracers: - tracer = s.tracers[name] - if not hasattr(tracer, "nz"): - raise ValueError(f"shiftable rows name tracer {name}, which has no n(z)") - nzs.append((np.asarray(tracer.z, float), np.asarray(tracer.nz, float))) - return nzs + def __getitem__(self, key): + return self._values[key] + def __iter__(self): + return iter(self._values) -def _xi_block(s, tracers, rows): - nz_i, nz_j = _pair_nz(s, tracers) - theta = np.array([s.data[i].tags["theta"] for i in rows], float) - plus = np.array([s.data[i].data_type == sacc_io.XI_PLUS for i in rows]) - grid = np.unique(theta) - at = np.searchsorted(grid, theta) + def __len__(self): + return len(self._values) - def theory(params, config): - xip, xim = xi_ccl(params, config, nz_i, nz_j, grid) - return np.where(plus, np.asarray(xip)[at], np.asarray(xim)[at]) + def __repr__(self): + return "" - return Block(f"ξ± {tracers[0]}×{tracers[1]}", np.asarray(rows), theory) + __str__ = __repr__ -def _cl_block(s, tracers, rows): - nz_i, nz_j = _pair_nz(s, tracers) - window = s.get_bandpower_windows(rows) - ell = np.asarray(window.values, float) - weight = np.asarray(window.weight, float) +def _hidden(blind): + """The blind's hidden point: its fiducial moved by the seed's draw.""" + from smokescreen.param_shifts import draw_param_shifts - def theory(params, config): - return weight.T @ cl_ee(params, config, nz_i, nz_j, ell) + record = json.loads(blind.path.read_text()) + shift = draw_param_shifts(dict(record["envelope"]), record["seed"]) + fiducial = record["fiducial"] + return _Hidden({k: v + shift[k] if k in shift else v for k, v in fiducial.items()}) - return Block(f"Cℓ_EE {tracers[0]}×{tracers[1]}", np.asarray(rows), theory) +def _nz(s, name): + tracer = s.tracers[name] + if not hasattr(tracer, "nz"): + raise ValueError(f"shiftable rows name tracer {name}, which has no n(z)") + return np.asarray(tracer.z, float), np.asarray(tracer.nz, float) -def shiftable_blocks(s): - """Every ξ± and Cℓ_EE block of ``s``: by tracer pair (and window, for Cℓ). - @sc shift-from-content - Blocks are discovered from the data types and tracers alone, whatever the - ``grid`` tags or file names; each ξ± row is evaluated at its own ``theta`` - from its pair's two n(z), each Cℓ_EE row through its bandpower window. A - shiftable row without a ``theta`` tag or a window is refused. - """ +def _blocks(s): + """Each ξ± tracer pair and Cℓ_EE (pair, window): rows, and theory(params, config).""" xi, cl = {}, {} for i, dp in enumerate(s.data): if dp.data_type in (sacc_io.XI_PLUS, sacc_io.XI_MINUS): - if "theta" not in dp.tags: - raise ValueError(f"ξ± row {i} has no theta tag to evaluate theory at") xi.setdefault(tuple(dp.tracers), []).append(i) elif dp.data_type == sacc_io.CL_EE: - if dp.tags.get("window") is None: - raise ValueError(f"Cℓ_EE row {i} has no bandpower window") - key = (tuple(dp.tracers), id(dp.tags["window"])) + key = (tuple(dp.tracers), id(dp.tags.get("window"))) cl.setdefault(key, []).append(i) - return [_xi_block(s, t, rows) for t, rows in xi.items()] + [ - _cl_block(s, t, rows) for (t, _), rows in cl.items() - ] - - -def _factor(block, fiducial, hidden): - """t(hidden) − t(fiducial) on ``block``'s rows, by the fork.""" - from smokescreen import factor_from_params - - return np.asarray( - factor_from_params( - fiducial.ccl_params(), - hidden.ccl_params(), - theory_fn=lambda params: block.theory(params, fiducial), - ), - float, - ) + blocks = [] + for pair, rows in xi.items(): + theta = np.array([s.data[i].tags["theta"] for i in rows], float) + grid, at = np.unique(theta, return_inverse=True) + plus = np.array([s.data[i].data_type == sacc_io.XI_PLUS for i in rows]) + nzs = [_nz(s, t) for t in pair] + def theory(p, c, nzs=nzs, grid=grid, at=at, plus=plus): + xip, xim = xi_ccl(p, c, *nzs, grid) + return np.where(plus, np.asarray(xip)[at], np.asarray(xim)[at]) -def factors(s, fiducial, hidden): - """Each shiftable block of ``s`` and its factor.""" - out = [] - for block in shiftable_blocks(s): - factor = _factor(block, fiducial, hidden) - if factor.shape != block.rows.shape or not np.all(np.isfinite(factor)): - raise ValueError( - f"{block.name}: the theory left rows unfilled or non-finite" - ) - out.append((block, factor)) - return out - + blocks.append((np.array(rows), theory)) + for (pair, _), rows in cl.items(): + window = s.get_bandpower_windows(rows) + nzs = [_nz(s, t) for t in pair] -def conceal(s, blind): - """A copy of ``s`` with every ξ± and Cℓ_EE row shifted by the blind. + def theory(p, c, nzs=nzs, w=window): + return np.asarray(w.weight).T @ cl_ee(p, c, *nzs, np.asarray(w.values)) - Every block's factor is computed before any value changes, and only the - copy changes. - """ - shifts = factors(s, blind.config.theory, blind.hidden) - out = s.copy() - for block, factor in shifts: - for row, delta in zip(block.rows, factor): - out.data[int(row)].value = out.data[int(row)].value + float(delta) - return out + blocks.append((np.array(rows), theory)) + return blocks -# --------------------------------------------------------------------------- # -# Custody commands: init, share, reveal, audit, verify -# --------------------------------------------------------------------------- # -def _catalogues(cat_config): - return yaml.safe_load(Path(cat_config).read_text()) +def _at_hidden(blocks, fiducial, blind): + """Each block's theory at the hidden point; a failure names only its type.""" + def evaluate(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + point = dataclasses.replace(fiducial, **_hidden(blind)) + return [theory(point.ccl_params(), fiducial) for _, theory in blocks] -def declared_custody(cat_config, version): - """The custody ``version`` is declared under in the catalogue config at - ``cat_config``, with the blind registry beside it.""" - return _custody.custody_of( - _catalogues(cat_config), version, registry=_custody.registry_of(cat_config) - ) - - -def _theory_stack(): - from importlib.metadata import PackageNotFoundError, version - - stack = {} - for package in ("pyccl", "camb", "smokescreen"): - try: - stack[package] = version(package) - except PackageNotFoundError: - stack[package] = None - return stack - - -def _may_cover(catalogues, recs, name, bases): - """Refuse ``bases`` blind ``name`` cannot cover: a variant, a catalogue - declared unblinded or mock or covered already, or one whose shear file - another catalogue would go on reading otherwise concealed.""" - for base in bases: - if _custody.base_catalogue(catalogues, base) != base: - raise _custody.CustodyError( - f"{base} is a variant of " - f"{_custody.base_catalogue(catalogues, base)}; a blind covers base " - "catalogues" - ) - declared = _custody.declaration(catalogues, base) - if declared in ("unblinded", "mock"): - raise _custody.CustodyError( - f"{base} is declared {declared}; declare it blinded first" - ) - covering = [r.name for r in recs.values() if base in r.bases] - if covering: - raise _custody.CustodyError( - f"{base} is already covered by blind {covering[0]}" - ) - covered = recs[name].bases if name in recs else () - after = {**recs, name: _custody.Record(name, {}, (*covered, *bases), None)} - for base in bases: - _custody.refuse_twins(catalogues, after, base) - - -def _conceal_one_row(blind): - """Conceal one ξ± row under ``blind``, as every blinded birth does; raise - if it cannot.""" - z = np.linspace(0.0, 2.0, 101) - s = sacc_io.new_sacc({0: (z, np.exp(-0.5 * ((z - 0.7) / 0.2) ** 2))}) - sacc_io.add_xi(s, (0, 0), [10.0], [0.0], [0.0], grid="probe") try: - factors(s, blind.config.theory, blind.hidden) - except Exception as err: - raise _custody.CustodyError( - f"no row conceals under blind {blind.name}'s config: {type(err).__name__}: {err}" - ) from err + return evaluate() + except Exception as err: # the message must carry no value + failure = type(err).__name__ + raise BlindingError( + f"the theory failed at blind {blind.name}'s hidden point ({failure})" + ) -def init(name, bases, *, cat_config, config=None): - """Draw blind ``name`` for ``bases`` and write its record; return its directory. +def conceal(s, blind): + """A copy of ``s`` with every ξ± and Cℓ_EE row shifted by the blind. - The one place a seed is drawn. Existing state is refused, never replaced, - and the seed is held in memory and written only as ciphertext. The record - is written to a staging directory, opened from there as jobs open it, and - made to conceal a row before it takes its name, so every blind in the - registry is one its jobs can conceal under. + The shift is t(hidden) − t(fiducial), evaluated at the fiducial first; a + block the blind leaves unmoved, or moves to a non-finite value, is refused. """ - from cryptography.fernet import Fernet + record = json.loads(blind.path.read_text()) + fiducial = TheoryConfig(**record["fiducial"]) + blocks = _blocks(s) + at_fiducial = [theory(fiducial.ccl_params(), fiducial) for _, theory in blocks] + out = s.copy() + for (rows, _), t_fid, t_hid in zip( + blocks, at_fiducial, _at_hidden(blocks, fiducial, blind) + ): + delta = np.asarray(t_hid, float) - np.asarray(t_fid, float) + if ( + delta.shape != rows.shape + or not np.all(np.isfinite(delta)) + or not delta.any() + ): + raise BlindingError( + f"blind {blind.name} leaves {len(rows)} shiftable rows unmoved or " + "non-finite; the theory must depend on the cosmology" + ) + for row, d in zip(rows, delta): + out.data[int(row)].value += float(d) + return out + - config = config or BlindingConfig() - registry = _custody.registry_of(cat_config) - catalogues = _catalogues(cat_config) - record, staging = registry / name, registry / f".{name}.tmp" / name - if not name or name[0] in "-_." or set(name) - set(_BLIND_NAME): +def init(name, catalogues, *, fiducial=None): + """Draw blind ``name`` into the catalogue config's registry, once.""" + if not _custody.BLIND_NAME.fullmatch(name) or name in ( + _custody.NONE, + _custody.MOCK, + ): raise _custody.CustodyError(f"blind name {name!r}: use a-z, 0-9, - _ .") - for path in (record, staging.parent): - if path.exists(): - raise _custody.CustodyError( - f"{path} exists; a blind is drawn once (delete a staging " - "directory an interrupted init left)" - ) - _may_cover(catalogues, _custody.records(registry), name, bases) - - seed = secrets.token_hex(16) - key = Fernet.generate_key() - payload = { - "blind": name, - "seed": seed, - "config_digest": config.digest(), + where = _custody.registry(catalogues) + where.mkdir(mode=0o700, parents=True, exist_ok=True) + record = { + "_": SECRET, + "seed": secrets.token_hex(32), + "envelope": ENVELOPE, + "fiducial": fiducial or dataclasses.asdict(TheoryConfig()), "draw_scheme": draw_scheme(), } - ciphertext = Fernet(key).encrypt(json.dumps(payload).encode("utf-8")) - commitment = { - "blind": name, - "seed_commitment": _custody.seed_commitment(seed), - "config": config.record(), - "config_digest": config.digest(), - "draw_scheme": draw_scheme(), - "theory_stack": _theory_stack(), - "created": datetime.now(timezone.utc).isoformat(timespec="seconds"), - } - - staging.mkdir(parents=True) - (staging / "commitment.json").write_text(json.dumps(commitment, indent=2) + "\n") - (staging / "seed.fernet").write_bytes(ciphertext) - (staging / "key").write_bytes(key) - (staging / "bases").write_text("".join(f"{b}\n" for b in bases)) - for part in ("commitment.json", "seed.fernet", "key"): - os.chmod(staging / part, 0o444) + path = where / f"{name}.blind.json" try: - _conceal_one_row(_read_blind(staging)) - except _custody.CustodyError: - shutil.rmtree(staging.parent) - raise - os.replace(staging, record) - staging.parent.rmdir() - print(f"[blinding] drew blind {name} for {', '.join(bases)}") - print(f"[blinding] commit {record.relative_to(registry.parent.parent)}/") - return record - - -def share(name, base, *, cat_config): - """Add ``base`` to blind ``name``'s ``bases``.""" - registry = _custody.registry_of(cat_config) - records = _custody.records(registry) - if name not in records: - raise _custody.CustodyError(f"no blind {name} in {registry}") - if records[name].revealed is not None: - raise _custody.CustodyError(f"blind {name} is revealed; draw a new one") - _may_cover(_catalogues(cat_config), records, name, [base]) - with open(registry / name / "bases", "a") as f: - f.write(f"{base}\n") - print(f"[blinding] {base} shares blind {name}; commit {registry / name / 'bases'}") - - -def _parts_under(root, commitment, skip): - """SACC files under ``root`` concealed under ``commitment``, found by content.""" - for path in sorted(Path(root).rglob("*.sacc")): - if skip in path.parents: - continue - try: - stamp = _custody.read_stamp(sacc_io.load(path).metadata) - except _custody.CustodyError: - continue # not born through the door, so not under any blind - if stamp.commitment == commitment: - yield path - - -def reveal(name, *, root, cat_config): - """Publish blind ``name``'s seed and move its concealed files aside. - - @sc reveal-is-reproduction - Revealing never subtracts a shift: the seed is published, every file - concealed under the blind moves to ``/revealed//``, the - declaration is flipped by a reviewed change, and the pipeline re-measures - true products that :func:`audit` compares against the archive. No file ever - mixes shifted and true signal. - """ - registry = _custody.registry_of(cat_config) - blind = _open(registry, name) - record = _custody.records(registry)[name] - revealed = registry / name / "revealed.json" - if record.revealed is not None and record.revealed != blind.seed: - raise _custody.CustodyError(f"{revealed} records another seed") - archive = Path(root) / "revealed" / name - concealed = list(_parts_under(root, record.commitment["seed_commitment"], archive)) - # Publishing cannot be undone, and a reveal that archived nothing leaves - # the concealed products unauditable: an empty (mistyped, or unbound in - # the container) root is refused first. A re-run finds its archive. - if not concealed and not any(archive.rglob("*.sacc")): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o440) + except FileExistsError: raise _custody.CustodyError( - f"nothing concealed under blind {name} in {root}; check --root (and " - "that the container binds it). The seed stays unpublished." - ) - if record.revealed is None: - fd = os.open(revealed, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444) - with os.fdopen(fd, "w") as f: - json.dump({"seed": blind.seed}, f) - for path in concealed: - target = archive / path.relative_to(root) - target.parent.mkdir(parents=True, exist_ok=True) - shutil.move(path, target) - print( - f"[blinding] published the seed of {name}; moved {len(concealed)} files " - f"to {archive}" - ) - print( - f"[blinding] commit {revealed} and declare `blinding: unblinded` on " - f"{', '.join(record.bases)}, then re-run" - ) - return archive - - -# How closely blinded − true must equal the seed's shift, relative to the -# block's largest shift: under the theory stack the blind was drawn with, and -# under another one. -SHIFT_TOLERANCE = 1e-6 -SHIFT_TOLERANCE_OTHER_STACK = 1e-3 -# A re-measurement repeats a measurement to float noise (TreeCorr's threaded -# sums reorder); the audit compares its numbers to this relative tolerance. -_REMEASURED = 1e-10 -# How far, in σ, a derived B-mode may move under a blind: 10× what a shift at -# the envelope's corners induces on the production grids, with shape-noise σ at -# UNIONS depth. There COSEBIs B (20 modes, [12, 83]′) move by ≤ 6e-3σ, and -# pure-mode ξ_B by ≤ 1e-2σ (ξ−_B at θ ≈ 2′: the estimator's own E→B leakage of -# the shift). A derivation from mixed inputs moves B by ~1σ. -B_SIGMA = { - sacc_io.COSEBI_BB: 5e-2, - sacc_io.PURE_TYPES["xip_B"]: 1e-1, - sacc_io.PURE_TYPES["xim_B"]: 1e-1, -} - - -def _same(x, y): - if isinstance(x, str) or isinstance(y, str): - return x == y - return bool(np.isclose(x, y, rtol=_REMEASURED, atol=0.0) or x == y) - - -def _rows_match(a, b): - if len(a.data) != len(b.data): - return False - for x, y in zip(a.data, b.data): - tx, ty = ({k: v for k, v in d.tags.items() if k != "window"} for d in (x, y)) - if (x.data_type, tuple(x.tracers), set(tx)) != ( - y.data_type, - tuple(y.tracers), - set(ty), - ) or not all(_same(tx[k], ty[k]) for k in tx): - return False - return True - - -def _covariance(s): - return None if s.covariance is None else np.asarray(s.covariance.dense) - - -def _same_covariance(a, b): - if a is None or b is None: - return a is None and b is None - scale = np.max(np.abs(b)) - return a.shape == b.shape and np.allclose(a, b, rtol=0.0, atol=_REMEASURED * scale) - - -def _signal(s): - """``s``'s signal rows and their covariance, or None if it has none.""" - keep = np.array([dp.data_type in sacc_io.SIGNAL for dp in s.data], bool) - if not keep.any(): - return None - out = s.copy() - out.keep_indices(keep) - return out - - -def _audit_part(blinded, true, fiducial, hidden, tolerance): - """Problems with one archived part against its re-measured twin. - - Only signal rows are compared: ρ/τ carries none, and a re-run does not - reproduce it (TreeCorr draws its jackknife patches afresh), so a part - without signal is judged by its stamp alone. - """ - stamp, true_stamp = (_custody.read_stamp(x.metadata) for x in (blinded, true)) - if true_stamp != _custody.Custody("unblinded", stamp.catalogue): - return [f"the live file is stamped {true_stamp.token}"], None - blinded, true = _signal(blinded), _signal(true) - if blinded is None and true is None: - return [], {"signal_rows": 0} - if blinded is None or true is None or not _rows_match(blinded, true): - return ["rows, tags or tracers differ"], None - problems = [] - cov = _covariance(true) - if not _same_covariance(_covariance(blinded), cov): - problems.append("covariances differ") - - values = np.asarray(true.mean) - delta = np.asarray(blinded.mean) - values - residual, rest = 0.0, np.ones(len(delta), bool) - for block, factor in factors(true, fiducial, hidden): - scale = np.max(np.abs(factor)) - residual = max(residual, np.max(np.abs(delta[block.rows] - factor)) / scale) - rest[block.rows] = False - if residual > tolerance: - problems.append(f"blinded − true ≠ shift(seed): residual {residual:.2e}") - - kinds = np.array([dp.data_type for dp in true.data]) - moved = {} - for kind in dict.fromkeys(kinds[rest]): - rows = rest & (kinds == kind) - if kind not in sacc_io.DERIVED: - if np.max(np.abs(delta[rows])) > _REMEASURED * np.max(np.abs(values[rows])): - problems.append(f"{kind} moved, but the blind leaves it unshifted") - elif cov is None: - problems.append(f"{kind} has no σ to judge its shift by") - else: - ratio = np.abs(delta[rows]) / np.sqrt(np.diag(cov)[rows]) - moved[kind] = float(np.max(ratio[np.isfinite(ratio)], initial=0.0)) - for kind, bound in B_SIGMA.items(): - if moved.get(kind, 0.0) > bound: - problems.append(f"{kind} moved by {moved[kind]:.2e}σ under the blind") - return problems, {"residual": residual, "shift_over_sigma": moved} - - -def audit(name, *, archive, true_root, cat_config, out=None): - """Check every archived file of blind ``name`` against its re-measured twin. - - The published seed must be the committed one. For each archived file, the - live file at the same relative path must be stamped unblinded for the same - catalogue. Its signal rows must match the archived ones in tags, tracers, - covariance (numbers to a re-measurement's float noise), - and blinded − true must equal the seed's shift on every ξ± and Cℓ_EE block - to :data:`SHIFT_TOLERANCE` of the block's largest shift - (:data:`SHIFT_TOLERANCE_OTHER_STACK` under another theory stack); Cℓ_BB and Cℓ_EB may not move, and - a derived statistic's B rows by at most :data:`B_SIGMA`. A derived - statistic's E rows are reported, in σ, not proven: the archive does not - record which ξ± parts, through which kernel, they came from. - """ - registry = _custody.registry_of(cat_config) - report = {"blind": name, "ok": False, "problems": [], "parts": {}} - try: - blind = _open(registry, name) - except _custody.CustodyError as err: - report["problems"].append(str(err)) - return _write_report(report, out) - record = _custody.records(registry)[name] - if record.revealed != blind.seed: - report["problems"].append("the published seed is not the committed one") - return _write_report(report, out) - stack = _theory_stack() - tolerance = SHIFT_TOLERANCE - if stack != record.commitment.get("theory_stack"): - tolerance = SHIFT_TOLERANCE_OTHER_STACK - report["theory_stack"] = { - "record": record.commitment.get("theory_stack"), - "now": stack, - } - fiducial, hidden = blind.config.theory, blind.hidden - report["shift"] = { - key: getattr(hidden, key) - getattr(fiducial, key) - for key in blind.config.envelope - } - archive, true_root = Path(archive), Path(true_root) - for path in sorted(archive.rglob("*.sacc")): - relative = str(path.relative_to(archive)) - live = true_root / relative - if not live.exists(): - report["parts"][relative] = {"problems": ["no re-measured file"]} - continue - blinded = sacc_io.load(path) - stamp = _custody.read_stamp(blinded.metadata) - if stamp.commitment != record.commitment["seed_commitment"]: - report["parts"][relative] = {"problems": ["not concealed under this blind"]} - continue - problems, numbers = _audit_part( - blinded, sacc_io.load(live), fiducial, hidden, tolerance - ) - report["parts"][relative] = {"problems": problems, **(numbers or {})} - if not report["parts"]: - report["problems"].append("the archive holds no parts") - report["ok"] = not report["problems"] and not any( - part["problems"] for part in report["parts"].values() - ) - return _write_report(report, out) - - -def _write_report(report, out): - if out is not None: - Path(out).write_text(json.dumps(report, indent=2, default=float) + "\n") - return report + f"blind {name} exists at {path}; a blind is drawn once" + ) from None + with os.fdopen(fd, "w") as f: + json.dump(record, f, indent=1) + print(f"[blinding] drew blind {name}, commitment {_custody.commitment(record)}") + return Blind(name, path) -def verify(path, *, cat_config): - """Problems with a file's stamp against its catalogue's custody (seedless).""" - stamp = _custody.read_stamp(sacc_io.load(path).metadata) - try: - declared = declared_custody(cat_config, stamp.catalogue) - except _custody.CustodyError as err: - return [str(err)] - problems = [] - if stamp != declared: - problems.append( - f"the stamp, {stamp.token}, is not {stamp.catalogue}'s custody, " - f"{declared.token}" - ) - if stamp.status == "blinded" and stamp.draw_scheme != draw_scheme(): - problems.append("this install draws under another scheme") - return problems +def show(name, catalogues): + """Print blind ``name``'s public record: everything but the seed.""" + path = _custody.registry(catalogues) / f"{name}.blind.json" + record = json.loads(path.read_text()) + print(f"blind {name}, commitment {_custody.commitment(record)}") + for key in ("envelope", "fiducial", "draw_scheme"): + print(f" {key}: {json.dumps(record[key])}") def main(argv=None): - parser = argparse.ArgumentParser( - prog="python -m sp_validation.blinding", description=__doc__.split("\n")[0] - ) - commands = parser.add_subparsers(dest="command", required=True) - for command, arguments in { - "init": ("blind", "bases+"), - "share": ("blind", "base"), - "reveal": ("blind",), - "audit": ("blind",), - "verify": ("file",), - }.items(): - sub = commands.add_parser(command) - for argument in arguments: - name, plus = argument.rstrip("+"), argument.endswith("+") - sub.add_argument(name, nargs="+" if plus else None) - sub.add_argument("--cat-config", required=True) - commands.choices["init"].add_argument( - "--config", - help="JSON of BlindingConfig fields over the defaults: an `envelope` " - "replaces the default one; `theory` fields replace theirs", - ) - commands.choices["reveal"].add_argument("--root", required=True) - commands.choices["audit"].add_argument("--archive", required=True) - commands.choices["audit"].add_argument("--true-root", required=True) - commands.choices["audit"].add_argument("--out") - a = parser.parse_args(argv) + import yaml - if a.command == "init": - config = None - if a.config: - given = json.loads(Path(a.config).read_text()) - default = BlindingConfig().record() - theory = {**default["theory"], **given.get("theory", {})} - config = _recorded_config(a.blind, {**default, **given, "theory": theory}) - init(a.blind, a.bases, cat_config=a.cat_config, config=config) - elif a.command == "share": - share(a.blind, a.base, cat_config=a.cat_config) - elif a.command == "reveal": - reveal(a.blind, root=a.root, cat_config=a.cat_config) - elif a.command == "audit": - report = audit( - a.blind, - archive=a.archive, - true_root=a.true_root, - cat_config=a.cat_config, - out=a.out, - ) - print(json.dumps(report, indent=2, default=float)) - return 0 if report["ok"] else 1 - elif a.command == "verify": - problems = verify(a.file, cat_config=a.cat_config) - for problem in problems: - print(f"[verify] {problem}") - print(f"[verify] {a.file}: {'ok' if not problems else 'FAILED'}") - return 1 if problems else 0 + parser = argparse.ArgumentParser(prog="python -m sp_validation.blinding") + parser.add_argument("command", choices=("init", "show")) + parser.add_argument("blind") + parser.add_argument("--cat-config", default=str(_custody.REPO_CAT_CONFIG)) + a = parser.parse_args(argv) + catalogues = yaml.safe_load(Path(a.cat_config).read_text()) + (init if a.command == "init" else show)(a.blind, catalogues) return 0 if __name__ == "__main__": - sys.exit(main()) + raise SystemExit(main()) diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 65650b84..fd5ba794 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -9,11 +9,11 @@ from cs_util.cosmo import get_cosmo from shear_psf_leakage import run_object, run_scale +from .. import custody as _custody from ..b_modes import ( _get_pte_from_scale_cut, find_conservative_scale_cut_key, ) -from ..custody import custody_of, registry_of, seed_path from ..statistics import chi2_and_pte from ..version import __version__ from .catalog_characterization import CatalogCharacterizationMixin @@ -131,6 +131,9 @@ class CosmologyValidation( noise debiasing, making those realizations reproducible run-to-run. cosmo_params : dict, optional Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. + custody : dict, optional + ``{version: custody token}`` (a workflow job's ``params.custody``), + replacing the catalogue config's declaration for those versions. Attributes ---------- @@ -198,6 +201,7 @@ def __init__( cell_seed=8192, path_onecovariance=None, cosmo_params=None, + custody=None, ): self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method @@ -264,6 +268,7 @@ def __init__( self.cc = cc = yaml.load(file, Loader=yaml.FullLoader) # The catalogues as declared, before virtual versions are materialised. self._declared = copy.deepcopy(cc) + self._tokens = dict(custody or {}) def resolve_paths_for_version(ver): """Resolve relative paths for a version using its subdir.""" @@ -315,7 +320,7 @@ def ensure_version_exists(ver): ensure_version_exists(seed_base) if ver not in cc: cc[ver] = copy.deepcopy(cc[seed_base]) - cc[ver]["shear"]["path"] = seed_path( + cc[ver]["shear"]["path"] = _custody.seed_path( cc[seed_base]["shear"], seed_label ) resolve_paths_for_version(ver) @@ -331,6 +336,14 @@ def ensure_version_exists(ver): final_versions.append(ver) self.versions = final_versions + _custody.check_mix( + { + v: _custody.parse(self._tokens[v]).blind + if v in self._tokens + else _custody.declared(self._declared, v) + for v in self.versions + } + ) if output_dir is not None: cc["paths"]["output"] = output_dir @@ -447,15 +460,12 @@ def results_objectwise(self): return self._results_objectwise def custody(self, version): - """The custody ``version``'s catalogue is declared under. - - Read from the catalogue config this object was built from and the blind - registry beside it (:func:`sp_validation.custody.custody_of`); every - SACC this object writes for ``version`` is sealed under it. - """ - return custody_of( - self._declared, version, registry=registry_of(self.catalog_config_path) - ) + """The custody every SACC this object writes for ``version`` is sealed + under: the token it was given for ``version`` (a workflow job's + ``params.custody``), else the catalogue config's declaration.""" + if version in self._tokens: + return _custody.parse(self._tokens[version], self._declared) + return _custody.custody_of(self._declared, version) def basename(self, version, treecorr_config=None, npatch=None): cfg = treecorr_config or self.treecorr_config diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 62518b75..5e6ef277 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -58,7 +58,7 @@ def calculate_2pcf( self.print_magenta(f"Computing {ver} ξ±") npatch = int(npatch or self.npatch) custody = self.custody(ver) - if custody.status == "blinded": + if custody.blinded: from .. import blinding blinding.open_blind(custody) diff --git a/src/sp_validation/custody.py b/src/sp_validation/custody.py index 987a43d6..0f58f3c4 100644 --- a/src/sp_validation/custody.py +++ b/src/sp_validation/custody.py @@ -1,14 +1,23 @@ -"""Custody: whether a catalogue's signal is blinded, unblinded or a mock. - -The custody of every catalogue version (:func:`custody_of`), the stamp a SACC -born under it carries, and the reader of the blind registry beside the -catalogue config (``cosmo_val/blinds/``; its writer is -:mod:`sp_validation.blinding`). - -The host Snakemake loads this module by path, with no sp_validation installed, -so it imports nothing but the standard library. +"""Custody: whether a catalogue's signal is blinded, and under which blind. + +@sc custody-is-declared +Every catalogue entry of the catalogue config declares ``blind: none``, +``blind: mock`` or ``blind: ``, and an entry without one is refused. +``_leak_corr`` and ``_seed`` versions take their entry's. Entries reading +one shear file declare one blind, the repository's ``cosmo_val/cat_config.yaml`` +included whatever config is passed; and a set of versions shown together may +hold a blinded catalogue only beside mocks and catalogues under the same blind. + +A blind is the record ``/.blind.json``, drawn by +:mod:`sp_validation.blinding`. Its commitment, the fork's +``seed_commitment`` of the whole canonical record, is public: it names the +blind in custody tokens and SACC stamps. + +The host Snakemake loads this module by path, so it imports only the standard +library (and PyYAML, which Snakemake carries, for the repository config). """ +import functools import hashlib import json import os @@ -16,26 +25,13 @@ from dataclasses import dataclass, field from pathlib import Path -STATUSES = ("blinded", "unblinded", "mock") - -# Top-level keys of the catalogue config that are not catalogues. +NONE, MOCK = "none", "mock" NOT_CATALOGUES = ("nz", "paths") - -# smokescreen.COMMITMENT_DOMAIN, spelt here so this module needs only the -# standard library. +REPO_CAT_CONFIG = Path(__file__).parents[2] / "cosmo_val" / "cat_config.yaml" +# smokescreen.COMMITMENT_DOMAIN, spelt here for the host; test_custody pins it. COMMITMENT_DOMAIN = b"smokescreen-seed-commitment-v1|" - -# The stamp every SACC carries: the custody it was born under. -STAMP_KEYS = ( - "blinding", - "blinding_catalogue", - "blinding_blind", - "blinding_commitment", - "blinding_config", - "blinding_scheme", -) -_BLIND_KEYS = STAMP_KEYS[2:] - +STAMP_KEYS = ("blind", "blind_commitment") +BLIND_NAME = re.compile(r"[a-z0-9][a-z0-9_.-]*") _SEED_SUFFIX = re.compile(r"_seed\d+$") _LEAK_SUFFIX = "_leak_corr" @@ -46,182 +42,84 @@ class CustodyError(ValueError): @dataclass(frozen=True) class Custody: - """The custody of one base catalogue, and of every variant of it.""" + """``none``, ``mock`` or a blind's name, and that blind's commitment.""" - status: str - catalogue: str - blind: str | None = None + blind: str commitment: str | None = None - config_digest: str | None = None - draw_scheme: int | None = None - # Where the blind is opened from; two custodies are equal whatever it is. registry: Path | None = field(default=None, compare=False) + @property + def blinded(self): + return self.blind not in (NONE, MOCK) + @property def token(self): """One string that changes whenever the custody does.""" - if self.status == "blinded": - return f"blinded:{self.catalogue}:{self.blind}:{self.commitment}" - return f"{self.status}:{self.catalogue}" + return f"{self.blind}:{self.commitment}" if self.blinded else self.blind @property def stamp(self): - """The ``blinding*`` metadata a SACC born under this custody carries.""" - stamp = {"blinding": self.status, "blinding_catalogue": self.catalogue} - if self.status == "blinded": - stamp.update( - { - "blinding_blind": self.blind, - "blinding_commitment": self.commitment, - "blinding_config": self.config_digest, - "blinding_scheme": int(self.draw_scheme), - } - ) - return stamp + """The metadata a SACC born under this custody carries.""" + if self.blinded: + return {"blind": self.blind, "blind_commitment": self.commitment} + return {"blind": self.blind} + + +def parse(token, catalogues=None): + """The custody a token (:attr:`Custody.token`) names, with the registry of + ``catalogues`` (a catalogue config) when it is blinded.""" + blind, _, commitment = token.partition(":") + custody = Custody(blind, commitment or None) + if custody.blinded and catalogues is not None: + return Custody(blind, commitment, registry(catalogues)) + return custody + +def commitment(record): + """sha256 of the domain-prefixed canonical JSON of a blind's record.""" + text = json.dumps(record, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(COMMITMENT_DOMAIN + text.encode("utf-8")).hexdigest() -def seed_commitment(seed): - """The public commitment to a seed: sha256 of the domain-prefixed seed.""" - return hashlib.sha256(COMMITMENT_DOMAIN + str(seed).encode("utf-8")).hexdigest() +@functools.cache +def _guard(path): + for parent in (path, *path.parents): + if (parent / ".git").exists(): + if parent != path: + raise CustodyError( + f"the blind registry {path} lies inside the git worktree " + f"{parent}; point paths.blinds outside it, or at the root of " + "a dedicated private blinds repository" + ) + break + return path -def registry_of(cat_config): - """The blind registry beside a catalogue config, spelled as the config is.""" - return Path(cat_config).absolute().parent / "blinds" + +def registry(catalogues): + """The blind registry the catalogue config names (``paths.blinds``).""" + path = (catalogues.get("paths") or {}).get("blinds") + if not path or not os.path.isabs(os.path.expanduser(path)): + raise CustodyError(f"paths.blinds, {path!r}, must be an absolute path") + return _guard(Path(os.path.realpath(os.path.expanduser(path)))) def _variant_chain(version): - """``version``, then each name stripping a trailing ``_leak_corr`` or - ``_seed`` leaves: the variants CosmologyValidation materialises from an - entry, in any order.""" + """``version``, then each name that stripping ``_leak_corr``/``_seed`` leaves.""" chain = [version] while True: name = chain[-1] - if name.endswith(_LEAK_SUFFIX): - stripped = name[: -len(_LEAK_SUFFIX)] - else: - stripped = _SEED_SUFFIX.sub("", name) + stripped = ( + name[: -len(_LEAK_SUFFIX)] + if name.endswith(_LEAK_SUFFIX) + else _SEED_SUFFIX.sub("", name) + ) if stripped in ("", name): return chain chain.append(stripped) -def base_catalogue(catalogues, version): - """The base catalogue of ``version``: the entry it is a variant of, or its - own, then that entry's ``base:`` links. - - A config entry named as a variant of another (``_leak_corr``, - ``_seed``) shares that entry's custody, as the variant it is - named for would; it may repeat that entry's ``blinding`` or ``base``, never - declare another. - """ - chain = _variant_chain(version) - base = _follow_base(catalogues, chain[-1], version) - for alias in chain[:-1]: - entry = catalogues.get(alias) - if not isinstance(entry, dict): - continue - status = declaration(catalogues, base) or "blinded" - if "base" in entry and _follow_base(catalogues, entry["base"], alias) != base: - claim = f"base: {entry['base']}" - elif entry.get("blinding", status) != status: - claim = f"blinding: {entry['blinding']}" - else: - continue - raise CustodyError( - f"{alias} is a variant of {chain[-1]}, so it shares the custody of " - f"{base} ({status}), but declares {claim}; declare custody on {base}, " - f"or rename {alias} to make it a catalogue of its own" - ) - return base - - -def _follow_base(catalogues, name, version): - """The entry the ``base:`` links from ``name`` end at.""" - seen = [] - while True: - if name in NOT_CATALOGUES: - raise CustodyError(f"{name} is not a catalogue") - if name not in catalogues: - via = f" (reached from {version} through base:)" if seen else "" - raise CustodyError(f"no catalogue {name} in the catalogue config{via}") - if name in seen: - raise CustodyError( - f"base: links form a cycle: {' -> '.join(seen + [name])}" - ) - seen.append(name) - parent = catalogues[name].get("base") - if parent is None: - return name - if "blinding" in catalogues[name]: - raise CustodyError( - f"{name} names its base {parent}; declare custody on {parent}, " - f"not on {name}" - ) - name = parent - - -def declaration(catalogues, base): - """What base catalogue ``base`` declares: a status, or ``None`` (blinded).""" - declared = catalogues[base].get("blinding") - if declared is not None and declared not in STATUSES: - raise CustodyError( - f"{base} declares blinding: {declared!r}; declare blinded, unblinded " - "or mock" - ) - return declared - - -@dataclass(frozen=True) -class Record: - """A blind's public record: its commitment, the bases it covers, and the - seed once it is revealed.""" - - name: str - commitment: dict - bases: tuple - revealed: str | None - - -def read_record(path): - """The public record of the blind stored at directory ``path``. - - The one reader of a blind's public files; its sealed seed is - ``blinding``'s to open. - """ - path = Path(path) - try: - commitment = json.loads((path / "commitment.json").read_text()) - bases = tuple((path / "bases").read_text().split()) - except (OSError, ValueError) as err: - raise CustodyError(f"blind record {path} is incomplete: {err}") from None - revealed = path / "revealed.json" - seed = json.loads(revealed.read_text())["seed"] if revealed.exists() else None - return Record(path.name, commitment, bases, seed) - - -def records(registry): - """Every blind in the registry, by name.""" - if registry is None or not Path(registry).is_dir(): - return {} - return { - path.name: read_record(path) - for path in sorted(Path(registry).iterdir()) - if path.is_dir() and not path.name.startswith(".") - } - - -def _reads(entry): - """The shear catalogue file a catalogue entry reads, if it names one.""" - shear = entry.get("shear") if isinstance(entry, dict) else None - if not isinstance(shear, dict) or "path" not in shear: - return None - return os.path.normpath(os.path.join(str(entry.get("subdir", "")), shear["path"])) - - def entry_name(catalogues, version): - """The catalogue-config entry describing ``version``: its own, or that of - the first name its variant chain reaches.""" + """The config entry describing ``version``: its own, or its variant chain's.""" for name in _variant_chain(version): if name in catalogues and name not in NOT_CATALOGUES: return name @@ -229,250 +127,155 @@ def entry_name(catalogues, version): def seed_path(shear, seed): - """The shear path of seed label ``seed``, from an entry's ``shear`` block. - - Its ``path_template`` formatted with ``seed`` (an int) and ``seed_label``; - without one, its ``path`` with the digits after its last ``seed`` replaced. - """ + """The shear path of seed label ``seed``: ``path_template`` formatted, or the + digits after ``path``'s last ``seed`` replaced.""" if shear.get("path_template"): return shear["path_template"].format(seed=int(seed), seed_label=seed) path = shear.get("path", "") match = re.match(r"(.*seed\D?)\d+", path) if match is None: - raise CustodyError( - f"shear path {path!r} has no path_template and no seed to " - "put a seed in" - ) + raise CustodyError(f"shear path {path!r} has no seed to put a seed in") return match.group(1) + seed + path[match.end() :] +def _reads(entry): + shear = entry.get("shear") if isinstance(entry, dict) else None + if not isinstance(shear, dict) or "path" not in shear: + return None + return os.path.normpath(os.path.join(str(entry.get("subdir", "")), shear["path"])) + + def shear_file(catalogues, version): - """The shear catalogue file ``version`` reads, as CosmologyValidation reads - it: its entry's, with the seed of a ``_seed`` variant rendered in.""" - chain = _variant_chain(version) + """The shear catalogue file ``version`` reads, a ``_seed``'s seed rendered.""" name = entry_name(catalogues, version) - entry = catalogues[name] - seeds = [_SEED_SUFFIX.search(n) for n in chain[: chain.index(name)]] - seeds = [m.group()[len("_seed") :] for m in seeds if m] + entry, chain = catalogues[name], _variant_chain(version) + seeds = [ + m.group()[5:] + for n in chain[: chain.index(name)] + if (m := _SEED_SUFFIX.search(n)) + ] if seeds: - shear = {**entry["shear"], "path": seed_path(entry["shear"], seeds[0])} - entry = {**entry, "shear": shear} + entry = { + **entry, + "shear": {**entry["shear"], "path": seed_path(entry["shear"], seeds[0])}, + } return _reads(entry) -def _concealer(catalogues, recs, base): - """What conceals ``base``'s data: the unrevealed blind covering it; ``""`` - when it is declared blinded and none does; ``None`` when it is public.""" - covering = sorted( - r.name for r in recs.values() if base in r.bases and not r.revealed - ) - if covering: - return ", ".join(covering) - return None if declaration(catalogues, base) in ("unblinded", "mock") else "" +def _blinds_by_file(catalogues): + by_file = {} + for name, entry in catalogues.items(): + if name not in NOT_CATALOGUES and (path := _reads(entry)): + by_file.setdefault(os.path.realpath(path), set()).add(entry.get("blind")) + return by_file + + +@functools.cache +def _repo_by_file(path, mtime): + import yaml + return _blinds_by_file(yaml.safe_load(Path(path).read_text())) -def refuse_twins(catalogues, recs, version): - """Refuse when a catalogue reading ``version``'s shear file is concealed - otherwise than ``version`` is, under the blind records ``recs``. - A blind conceals data, not a name: every catalogue reading one shear file - is concealed under one blind, or none is. - """ - base = base_catalogue(catalogues, version) +def of_file(path): + """The blinds the repository config declares for shear file ``path``.""" + if not REPO_CAT_CONFIG.exists(): + return set() + by_file = _repo_by_file(str(REPO_CAT_CONFIG), REPO_CAT_CONFIG.stat().st_mtime_ns) + return by_file.get(os.path.realpath(path), set()) + + +def declared(catalogues, version): + """The blind ``version`` is declared under: ``none``, ``mock`` or a name.""" + name = entry_name(catalogues, version) + blind = catalogues[name].get("blind") + if blind is None: + raise CustodyError( + f"{name} declares no `blind:`; declare `blind: none` (public), " + "`blind: mock` or `blind: ` in its catalogue config entry" + ) + if not isinstance(blind, str) or not BLIND_NAME.fullmatch(blind): + raise CustodyError(f"{name} declares blind: {blind!r}, not a blind name") try: - here = shear_file(catalogues, version) + path = shear_file(catalogues, version) except CustodyError: - # A seed file its entry cannot name, so no entry reads it: whatever - # would read it (rule xi, CosmologyValidation) refuses it there. - here = None - readers = { - name: base_catalogue(catalogues, name) - for name, entry in catalogues.items() - if here is not None and name not in NOT_CATALOGUES and _reads(entry) == here - } - hiding = { - b: _concealer(catalogues, recs, b) - for b in dict.fromkeys([base, *readers.values()]) - } - twins = [name for name, b in readers.items() if hiding[b] != hiding[base]] - if twins: - states = "; ".join( - f"{b}: {'public' if c is None else f'blind {c}' if c else 'no blind yet'}" - for b, c in hiding.items() - ) + return blind # a seed file no entry names: nothing else reads it + if path is None: + return blind + blinds = {blind} | _blinds_by_file(catalogues).get(os.path.realpath(path), set()) + blinds |= of_file(path) + if len(blinds) > 1: raise CustodyError( - f"{version} and {', '.join(twins)} read one shear file, {here}, but are " - f"not concealed alike ({states}). Catalogues reading one file share " - "one blind: declare them blinded and draw it for them together " - "(blinding init) or share it (blinding share), or make them variants " - "of one base with `base:`" + f"{version} reads {path}, which catalogue entries declare under " + f"{sorted(map(str, blinds))}; entries reading one file declare one blind " + f"(and {REPO_CAT_CONFIG} is authoritative for the files it names)" ) + return blind -def _no_blind(version, base, declared, registry): - why = ( - "declared blinded" - if declared == "blinded" - else "cosmo_val/cat_config.yaml declares no custody for it, so it is blinded" - ) - repo = registry.parent.parent - cat_config = registry.parent / "cat_config.yaml" - return ( - f"{version} is blinded ({why}) and no blind covers it in this checkout.\n" - "git pull first: its blind may already be drawn and committed. Never " - "draw a second one.\n" - "Only the custodian draws a blind, once:\n" - f" APPTAINERENV_PYTHONPATH={repo}/src spv-container exec " - f"python -m sp_validation.blinding init {base} " - f"--cat-config {cat_config}\n" - "then commits cosmo_val/blinds//.\n" - "Re-processing a catalogue whose blind is still concealed? Share it " - f"instead: python -m sp_validation.blinding share {base} " - f"--cat-config {cat_config}\n" - "Predates blinding: declare `blinding: unblinded`. A mock: " - "`blinding: mock`. A variant: `base: `." - ) - - -def custody_of(catalogues, version, *, registry): - """The custody of ``version``, from its base's declaration and the registry. - - @sc custody-is-declared - Custody is declared once per *base* catalogue, as ``blinding:`` on its - entry in ``cosmo_val/cat_config.yaml``, and read only here, with the blind - registry; no workflow config, environment variable or call argument can - change it. An entry declaring nothing is blinded, so a new catalogue fails - closed. Variants share their base's custody and blind: ``_leak_corr`` and - ``_seed`` versions, and entries naming their parent with ``base:``. A - blinded base is covered by exactly one blind, which also conceals every - catalogue reading its shear file (:func:`refuse_twins`). - - Raises - ------ - CustodyError - With the one thing to do, when the declaration and the registry - disagree: a blinded catalogue with no blind, a revealed blind still - declared blinded, an unblinded catalogue under a concealed blind, a - mock under a blind, two blinds over one catalogue, or catalogues - reading one shear file concealed otherwise. - """ - registry = Path(registry) - base = base_catalogue(catalogues, version) - declared = declaration(catalogues, base) - recs = records(registry) - refuse_twins(catalogues, recs, version) - covering = [r for r in recs.values() if base in r.bases] - if len(covering) > 1: - names = ", ".join(r.name for r in covering) - raise CustodyError(f"{base} is covered by several blinds: {names}") - record = covering[0] if covering else None - - if declared == "mock": - if record is not None: - raise CustodyError( - f"{base} is declared mock but blind {record.name} covers it; a " - "mock is never blinded" - ) - return Custody("mock", base) - - if declared == "unblinded": - if record is None: - return Custody("unblinded", base) - if record.revealed is None: - raise CustodyError( - f"{base} is declared unblinded, but blind {record.name} still " - "conceals it. Unblinding is the reveal: python -m " - f"sp_validation.blinding reveal {record.name} --root " - f"--cat-config {registry.parent / 'cat_config.yaml'}" - ) - if seed_commitment(record.revealed) != record.commitment["seed_commitment"]: - raise CustodyError( - f"the seed published for blind {record.name} does not match its " - "commitment" - ) - return Custody("unblinded", base) - - if record is None: - raise CustodyError(_no_blind(version, base, declared, registry)) - if record.revealed is not None: - raise CustodyError( - f"blind {record.name} is public; declare `blinding: unblinded` on " - f"{base} (the reveal PR does this)" +def custody_of(catalogues, version): + """The custody of ``version``: a blinded one reads its blind's commitment.""" + blind = declared(catalogues, version) + if blind in (NONE, MOCK): + return Custody(blind) + where = registry(catalogues) + try: + record = json.loads((where / f"{blind}.blind.json").read_text()) + except (OSError, ValueError): + fix = ( + f"no blind {blind} in {where}; `python -m sp_validation.blinding init " + f"{blind}` draws one, only if you mean to create it" + if where.is_dir() and os.access(where, os.R_OK | os.X_OK) + else f"get read access to {where}" ) - c = record.commitment - return Custody( - "blinded", - base, - record.name, - c["seed_commitment"], - c["config_digest"], - int(c["draw_scheme"]), - registry, - ) - - -def summary(catalogues, versions, *, registry): - """One ``[custody]`` line per base catalogue among ``versions``.""" - by_base = {} + raise CustodyError( + f"{version} is blinded under {blind}: {fix}. Setting `blind: none` " + "would unblind it." + ) from None + return Custody(blind, commitment(record), where) + + +def check_mix(blinds): + """Refuse ``{version: blind}`` showing a blinded catalogue beside another + real-sky catalogue not under its blind.""" + real = {v: b for v, b in blinds.items() if b != MOCK} + for version, blind in real.items(): + for other, theirs in real.items(): + if blind not in (NONE, theirs): + raise CustodyError( + f"{version} (blinded under {blind}) and {other} " + f"({'public' if theirs == NONE else f'blinded under {theirs}'}) " + "cannot be shown together: overlaying them shows the blind's shift" + ) + + +def summary(catalogues, versions): + """One ``[custody]`` line per catalogue entry among ``versions``.""" + by_entry = {} for version in dict.fromkeys(versions): - by_base.setdefault(base_catalogue(catalogues, version), []).append(version) + by_entry.setdefault(entry_name(catalogues, version), []).append(version) lines = [] - for base, members in by_base.items(): - custody = custody_of(catalogues, base, registry=registry) - variants = [v for v in members if v != base] - also = f" (+ {', '.join(variants)})" if variants else "" - state = ( - f"blinded under {custody.blind}" - if custody.status == "blinded" - else custody.status - ) - lines.append(f"[custody] {base}{also}: {state}") + for name, members in by_entry.items(): + custody = custody_of(catalogues, name) + also = [v for v in members if v != name] + also = f" (+ {', '.join(also)})" if also else "" + state = f"blinded under {custody.blind}" if custody.blinded else custody.blind + lines.append(f"[custody] {name}{also}: {state}") return lines def read_stamp(metadata): - """The custody a SACC's stamp records; a missing or malformed stamp raises. - - The result carries no registry: it is what the file says, to compare with - what a catalogue is declared under (``==``). - """ - status = metadata.get("blinding") - if status not in STATUSES: + """The custody a SACC's stamp records; a missing or malformed stamp raises.""" + blind = metadata.get("blind") + if not isinstance(blind, str) or not BLIND_NAME.fullmatch(blind): raise CustodyError( - f"no valid custody stamp (blinding={status!r}): every SACC is born " + f"no valid custody stamp (blind={blind!r}): every SACC is born " "through sacc_io.save" ) - if "blinding_catalogue" not in metadata: - raise CustodyError("custody stamp names no catalogue") - present = [k for k in _BLIND_KEYS if k in metadata] - if status == "blinded" and len(present) != len(_BLIND_KEYS): - missing = sorted(set(_BLIND_KEYS) - set(present)) - raise CustodyError(f"blinded custody stamp lacks {missing}") - if status != "blinded" and present: - raise CustodyError(f"a {status} custody stamp carries blind keys {present}") - if status != "blinded": - return Custody(status, metadata["blinding_catalogue"]) - return Custody( - status, - metadata["blinding_catalogue"], - metadata["blinding_blind"], - metadata["blinding_commitment"], - metadata["blinding_config"], - int(metadata["blinding_scheme"]), - ) - - -def confirm(custody, token): - """``custody``, if its token is the job's ``params.custody``; else raise. - - ``token`` is resolved by the Snakemake process that runs the job: the - launch under a local executor, the job step at job start under slurm. The - job's own resolution, in the container, must agree with it. - """ - if custody.token != token: + custody = Custody(blind, metadata.get("blind_commitment")) + if custody.blinded != (custody.commitment is not None): raise CustodyError( - f"custody of {custody.catalogue} differs between the job and its " - f"Snakemake: the job resolves {custody.token}, Snakemake resolved " - f"{token}; launch again" + f"malformed custody stamp: blind={blind}, " + f"blind_commitment={custody.commitment}" ) return custody diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index fe69d53f..cb3f0f72 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -1032,16 +1032,16 @@ def seal(s, custody): "this SACC is already stamped; a loaded or sealed SACC is re-written " "only as a derivation (save(..., derived_from=[...]))" ) - if custody.status == "blinded": + if custody.blinded: _refuse_unruled(s) types = {dp.data_type for dp in s.data} derived = types & set(DERIVED) - if custody.status == "blinded" and derived: + if custody.blinded and derived: raise ValueError( f"a blinded catalogue's {sorted(derived)} rows are derived " "statistics: save them with derived_from=[their input parts]" ) - if custody.status == "blinded" and types & set(SHIFTABLE): + if custody.blinded and types & set(SHIFTABLE): from . import blinding out = blinding.conceal(s, blinding.open_blind(custody)) @@ -1069,10 +1069,10 @@ def _derive(s, parts, custody): ) if custody is not None and custody != stamp: raise ValueError( - f"parts are stamped {stamp.token}, but {custody.catalogue} is " - f"declared {custody.token}" + f"parts are stamped {stamp.token}, but the catalogue is declared " + f"{custody.token}" ) - if stamp.status == "blinded": + if stamp.blinded: _refuse_unruled(s) inputs = {_row_key(dp) for p in parts for dp in p.data if dp.data_type in SHIFTABLE} stray = [ @@ -1127,7 +1127,7 @@ def load(path): """Load the SACC at ``path``, refusing a file without a valid custody stamp. @sc stamped-or-refused - Every file ``save`` wrote carries one of the three stamps; anything else was + Every file ``save`` wrote carries a custody stamp; anything else was not born through the door and is refused, with no escape hatch. """ s = sacc.Sacc.load_fits(str(path)) diff --git a/src/sp_validation/tests/_synthetic.py b/src/sp_validation/tests/_synthetic.py index 18e36f25..5aa8c388 100644 --- a/src/sp_validation/tests/_synthetic.py +++ b/src/sp_validation/tests/_synthetic.py @@ -2,10 +2,9 @@ The glue tests run the real compute seams on it: a shear catalogue (RA/Dec/e1/e2/w), a PSF star catalogue with the columns the leakage and ρ/τ -seams read, a cs_util-readable dndz, and a ``cat_config.yaml`` beside a -``blinds/`` registry directory, as in the repository layout. Every catalogue -entry in the config reads its own copy of the same galaxies and declares its -own custody. +seams read, a cs_util-readable dndz, and a ``cat_config.yaml`` whose blind +registry is ``blinds/`` beside it. Every catalogue entry in the config reads its +own copy of the same galaxies and declares its own custody. """ import copy @@ -36,10 +35,10 @@ def write_synthetic_catalogs( Add a ``psf`` block (ρ/τ and pseudo-Cℓ read it via ``get_params_rho_tau``). catalogues : dict, optional - ``{version: declaration}``, one catalogue entry each, reading its own - copy of the shear catalogue (catalogues reading one file share one - custody). A declaration is a ``blinding`` value, or ``None`` for an - entry that declares none. Defaults to one mock, ``TestCatalog``. + ``{version: blind}``, one catalogue entry each, reading its own copy of + the shear catalogue. ``blind`` is the entry's ``blind:`` value, or + ``None`` for an entry that declares none. Defaults to one mock, + ``TestCatalog``. Returns ------- @@ -132,14 +131,14 @@ def write_synthetic_catalogs( config = { "nz": {"subdir": str(nz_dir), "dndz": {"path": "dndz_{pipeline}_A.txt"}}, - "paths": {"output": str(output_dir)}, + "paths": {"output": str(output_dir), "blinds": str(tmp_path / "blinds")}, } - for version, declaration in catalogues.items(): + for version, blind in catalogues.items(): config[version] = copy.deepcopy(entry) config[version]["shear"]["path"] = f"shear_{version}.fits" shear.write(cat_dir / f"shear_{version}.fits", overwrite=True) - if declaration is not None: - config[version]["blinding"] = declaration + if blind is not None: + config[version]["blind"] = blind config_path = tmp_path / "cat_config.yaml" config_path.write_text(yaml.safe_dump(config, sort_keys=False)) diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py index 3c82c44f..5f974ca4 100644 --- a/src/sp_validation/tests/test_assemble_sacc.py +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -52,7 +52,7 @@ def _theta(n=6): META = {"catalogue_version": "vSYNTH", "npatch": 1} -MOCK = Custody("mock", "vSYNTH") +MOCK = Custody("mock") def _xi_cov_txt(tmp_path, n=12, seed=21): diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 4cfd3cb9..32b20810 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -1,66 +1,52 @@ """The blind and the file door. -A blind is drawn once (``blinding init``) into a registry beside a catalogue -config; ``sacc_io.save`` is the only writer, and conceals a blinded -catalogue's ξ± and Cℓ_EE rows in memory before the file exists. Blinds here -use the fast Eisenstein–Hu theory. +A blind is drawn once (``blinding init``) into a registry outside git; +``sacc_io.save`` is the only writer, and conceals a blinded catalogue's ξ± and +Cℓ_EE rows in memory before the file exists. Blinds here use the fast +Eisenstein–Hu theory. """ +import dataclasses import json +import stat +import traceback import numpy as np import pytest -import yaml from hypothesis import example, given, settings from hypothesis import strategies as st from sp_validation import blinding as bd from sp_validation import custody as cu from sp_validation import sacc_io as sio +from sp_validation.blinding_theory import TheoryConfig VERSIONS = ("TOY", "OTHER", "TOY_OPEN", "OTHER_OPEN", "TOY_MOCK") +FAST = dataclasses.asdict(TheoryConfig(transfer_function="eisenstein_hu")) -def _registry(root, *blinds): - """A catalogue config declaring one catalogue per custody state, and - ``blinds`` ((name, base) pairs) drawn for it, each from a fixed seed.""" - entries = { - "TOY": {}, - "OTHER": {}, - "TOY_OPEN": {"blinding": "unblinded"}, - "OTHER_OPEN": {"blinding": "unblinded"}, - "TOY_MOCK": {"blinding": "mock"}, +def _catalogues(root): + """A catalogue config declaring one catalogue per custody.""" + blinds = { + "TOY": "toy", + "OTHER": "other", + "TOY_OPEN": "none", + "OTHER_OPEN": "none", + "TOY_MOCK": "mock", } - (root / "cat_config.yaml").write_text(yaml.safe_dump(entries)) - (root / "fast.json").write_text( - json.dumps({"theory": {"transfer_function": "eisenstein_hu"}}) - ) - with pytest.MonkeyPatch.context() as m: - for blind, base in blinds: - m.setattr(bd.secrets, "token_hex", lambda n, b=blind: f"{b}-seed") - _init(root, blind, base) - return root - - -def _init(root, blind, *bases): - cat_config, config = root / "cat_config.yaml", root / "fast.json" - bd.main( - ["init", blind, *bases, "--cat-config", str(cat_config)] - + ["--config", str(config)] - ) - - -def _custody(root, version): - return bd.declared_custody(root / "cat_config.yaml", version) + config = {"paths": {"blinds": str(root / "blinds")}} + for name, blind in blinds.items(): + config[name] = {"shear": {"path": str(root / f"{name}.fits")}, "blind": blind} + return config @pytest.fixture(scope="module") def blinds(tmp_path_factory): """The five custodies: TOY under blind `toy`, OTHER under `other`.""" - root = _registry( - tmp_path_factory.mktemp("registry"), ("toy", "TOY"), ("other", "OTHER") - ) - return {v: _custody(root, v) for v in VERSIONS} + cats = _catalogues(tmp_path_factory.mktemp("registry")) + for name in ("toy", "other"): + bd.init(name, cats, fiducial=FAST) + return {v: cu.custody_of(cats, v) for v in VERSIONS} def _nz(z0): @@ -149,7 +135,7 @@ def test_seal_shifts_only_the_signal_it_has_a_rule_for(blinds, content, version) derived rows or a type with no blinding rule is refused. Unblinded and mock births keep their values. Each is stamped with its custody.""" s, custody = part(**content), blinds[version] - blinded = custody.status == "blinded" + blinded = custody.blinded if blinded and (content["derived"] or content["unruled"]): with pytest.raises(ValueError, match="derived_from|no blinding rule"): sio.seal(s, custody) @@ -209,7 +195,7 @@ def test_a_derivation_carries_its_inputs_one_stamp( allowed = ( all(blinds[v].stamp == stamp for v in inputs) and (declared is None or blinds[declared].stamp == stamp) - and not (stamp["blinding"] == "blinded" and content == "plaintext") + and not (blinds[inputs[0]].blinded and content == "plaintext") ) path = tmp_path_factory.mktemp("derived") / "d.sacc" kwargs = dict( @@ -228,105 +214,69 @@ def test_a_derivation_carries_its_inputs_one_stamp( # --------------------------------------------------------------------------- # -# The blind: drawn once, its seed never on disk +# The blind: drawn once, opened only under its commitment, never shown # --------------------------------------------------------------------------- # -SEED = "5eed" * 8 - - -@pytest.mark.parametrize("fault", [None, "after_key", "encrypt"]) -def test_init_never_writes_the_seed(tmp_path, monkeypatch, fault): - """A normal init, or one interrupted anywhere, leaves no file holding the - seed.""" - from cryptography import fernet - - _registry(tmp_path) - monkeypatch.setattr(bd.secrets, "token_hex", lambda n: SEED) - if fault == "after_key": - monkeypatch.setattr(bd.os, "replace", lambda src, dst: 1 / 0) - elif fault == "encrypt": - monkeypatch.setattr(fernet.Fernet, "encrypt", lambda self, data: 1 / 0) - if fault is None: - _init(tmp_path, "toy", "TOY") - else: - with pytest.raises(ZeroDivisionError): - _init(tmp_path, "toy", "TOY") - files = [p for p in tmp_path.rglob("*") if p.is_file()] - assert not [p for p in files if SEED.encode() in p.read_bytes()] - - -def test_a_blind_is_drawn_once(tmp_path): - _registry(tmp_path, ("toy", "TOY")) - with pytest.raises(cu.CustodyError, match="exists"): - _init(tmp_path, "toy", "OTHER") - with pytest.raises(cu.CustodyError, match="already covered"): - _init(tmp_path, "again", "TOY") - - -def test_the_commitment_hides_the_rng_seed(): - """The fork seeds its RNG from the undomained sha256 of the seed; the - domain prefix keeps the public commitment from publishing it.""" - from smokescreen.param_shifts import _normalize_seed - - seed = "the-secret" - assert int(cu.seed_commitment(seed)[:16], 16) != _normalize_seed(seed) - - -def test_the_digest_binds_the_config(): - config = bd.BlindingConfig() - moved = bd.BlindingConfig(envelope={**config.envelope, "S8": 0.08}) - assert moved.digest() != config.digest() - +def test_a_blind_is_drawn_once_and_kept_private(tmp_path): + cats = _catalogues(tmp_path) + blind = bd.init("toy", cats, fiducial=FAST) + assert stat.S_IMODE(blind.path.stat().st_mode) == 0o440 + assert stat.S_IMODE(blind.path.parent.stat().st_mode) == 0o700 + with pytest.raises(cu.CustodyError, match="drawn once"): + bd.init("toy", cats, fiducial=FAST) + + +def test_a_blind_opens_only_under_its_commitment(blinds, monkeypatch): + custody = blinds["TOY"] + assert bd.open_blind(custody) == bd.Blind( + "toy", custody.registry / "toy.blind.json" + ) + with pytest.raises(cu.CustodyError, match="launch again"): + bd.open_blind(dataclasses.replace(custody, commitment="0" * 64)) + monkeypatch.setattr(bd, "draw_scheme", lambda: 99) + with pytest.raises(cu.CustodyError, match="draw scheme 2; .* under 99"): + bd.open_blind(custody) -def test_a_blind_opens_only_with_its_key(tmp_path): - from cryptography.fernet import Fernet - root = _registry(tmp_path, ("toy", "TOY")) - key = root / "blinds" / "toy" / "key" - key.chmod(0o644) - key.write_bytes(Fernet.generate_key()) - with pytest.raises(cu.CustodyError, match="does not decrypt"): - bd.open_blind(_custody(root, "TOY")) +def _hidden_failure(blind, fiducial): + """The traceback, locals included, of a theory failing at the hidden point + with its parameters in its message.""" + def failing(params, *args): + raise RuntimeError(f"cannot evaluate {params}") -# --------------------------------------------------------------------------- # -# The reveal: publish, then prove blinded − true = shift(seed) -# --------------------------------------------------------------------------- # -@pytest.mark.parametrize("seed", ["committed", "wrong"]) -def test_the_audit_proves_the_shift(tmp_path, seed): - """A concealed part passes the audit against its re-measured twin under - the published committed seed, and fails under any other.""" - root = _registry(tmp_path, ("toy", "TOY")) - blinded = _custody(root, "TOY") - true = part() - archived = sio.seal(true, blinded) - published = bd.open_blind(blinded).seed if seed == "committed" else "not-it" - (root / "blinds" / "toy" / "revealed.json").write_text( - json.dumps({"seed": published}) - ) - for tree, s, stamp in ( - ("archive", archived, blinded.stamp), - ("live", true, cu.Custody("unblinded", "TOY").stamp), - ): - s = s.copy() - s.metadata.update(stamp) - (root / tree).mkdir() - s.save_fits(str(root / tree / "part.sacc")) - report = bd.audit( - "toy", - archive=root / "archive", - true_root=root / "live", - cat_config=root / "cat_config.yaml", + with pytest.MonkeyPatch.context() as m: + m.setattr(bd, "xi_ccl", failing) + with pytest.raises(bd.BlindingError) as failure: + bd._at_hidden(bd._blocks(part(cl=False)), fiducial, blind) + trace = traceback.TracebackException.from_exception( + failure.value, capture_locals=True ) - assert report["ok"] == (seed == "committed"), report - - -@pytest.mark.parametrize("S8, Omega_m", [(0.80, 0.30), (0.725, 0.20), (0.875, 0.40)]) -def test_ccl_total_matter_is_the_blind_axis(S8, Omega_m): - """CCL's Ωm and S8 at a point's parameters are the point's own.""" - ccl = pytest.importorskip("pyccl") - from sp_validation.blinding_theory import TheoryConfig - - point = TheoryConfig(S8=S8, Omega_m=Omega_m) - cosmo = ccl.Cosmology(**point.ccl_params()) - assert cosmo["Omega_m"] == pytest.approx(Omega_m, rel=1e-12) - assert cosmo["sigma8"] * np.sqrt(cosmo["Omega_m"] / 0.3) == pytest.approx(S8) + return "".join(trace.format()) + + +def test_the_hidden_cosmology_never_shows(tmp_path, capfd): + """Neither the seed nor the hidden S8, Ωm or σ8 reaches a repr, the + terminal or a traceback's locals.""" + cats = _catalogues(tmp_path) + blind = bd.init("toy", cats, fiducial=FAST) + record = json.loads(blind.path.read_text()) + hidden = bd._hidden(blind) + sigma8 = hidden["S8"] / np.sqrt(hidden["Omega_m"] / 0.3) + needles = [record["seed"], record["seed"][:16]] + [ + form(x) + for x in (hidden["S8"], hidden["Omega_m"], sigma8) + for form in (repr, "{:.4g}".format, "{:.6g}".format) + ] + bd.show("toy", cats) + sio.seal(part(cl=False), cu.custody_of(cats, "TOY")) + haystacks = [ + repr(blind), + repr(hidden), + str(hidden), + f"{hidden}", + "".join(capfd.readouterr()), + _hidden_failure(blind, TheoryConfig(**record["fiducial"])), + ] + for i, haystack in enumerate(haystacks): + # The message names the haystack only: a failure must not print a needle. + assert not any(n in haystack for n in needles), f"haystack {i} shows it" diff --git a/src/sp_validation/tests/test_config_paths_exist.py b/src/sp_validation/tests/test_config_paths_exist.py index a3de6873..3ce1797b 100644 --- a/src/sp_validation/tests/test_config_paths_exist.py +++ b/src/sp_validation/tests/test_config_paths_exist.py @@ -177,12 +177,13 @@ def _located_elsewhere(source: Path, key: str, value: str) -> bool: """Whether a path-shaped value names nothing the config itself locates. ``paths.output`` in the catalogue config is where cosmo_val writes, created - by the run. A calibration ``params.input_path`` with no directory is opened + by the run, and ``paths.blinds`` the blind registry ``blinding init`` + creates. A calibration ``params.input_path`` with no directory is opened relative to its consumer's run directory (the image-sim run, or the catalogue directory ``scripts/masking.py`` prefixes). """ if source.name == "cat_config.yaml": - return key == "paths.output" + return key in ("paths.output", "paths.blinds") return ( source.parent.name == "calibration" and key == "params.input_path" diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 8d18b890..057c22eb 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -79,6 +79,7 @@ def _make_seed_config(tmp_path, shear_filename): }, "paths": {"output": str(output_dir)}, base_version: { + "blind": "none", "subdir": str(base_dir), "pipeline": "SP", "shear": { @@ -498,47 +499,40 @@ def block_variances(c): # --------------------------------------------------------------------------- # @pytest.fixture def blinded_and_twin(tmp_path): - """TOY, blinded under `toy`, and TOY_OPEN: the same galaxies, unblinded.""" - import json + """TOY, blinded under `toy`, and TOY_OPEN: the same galaxies, public.""" + import dataclasses + + import yaml from sp_validation import blinding + from sp_validation.blinding_theory import TheoryConfig params, _ = write_synthetic_catalogs( tmp_path, n_gal=4000, coherent_shear=True, - catalogues={"TOY": None, "TOY_OPEN": "unblinded"}, - ) - fast = tmp_path / "fast.json" - fast.write_text(json.dumps({"theory": {"transfer_function": "eisenstein_hu"}})) - blinding.main( - [ - "init", - "toy", - "TOY", - "--cat-config", - params["catalog_config"], - "--config", - str(fast), - ] + catalogues={"TOY": "toy", "TOY_OPEN": "none"}, ) - return CosmologyValidation( - versions=["TOY", "TOY_OPEN"], - npatch=1, - theta_min=5.0, - theta_max=60.0, - nbins=6, - **params, + fast = TheoryConfig(transfer_function="eisenstein_hu") + catalogues = yaml.safe_load(open(params["catalog_config"])) + blinding.init("toy", catalogues, fiducial=dataclasses.asdict(fast)) + grid = dict(npatch=1, theta_min=5.0, theta_max=60.0, nbins=6, **params) + return ( + CosmologyValidation(versions=["TOY"], **grid), + CosmologyValidation(versions=["TOY_OPEN"], **grid), + grid, ) def test_a_blinded_catalogues_xi_leaves_concealed(blinded_and_twin): """[signal-leaves-sealed] calculate_2pcf returns, and caches, a blinded - catalogue's ξ± shifted from its unblinded twin's by exactly the blind's shift.""" + catalogue's ξ± shifted from its public twin's by exactly the blind's shift; + and the two are never held together.""" from sp_validation import blinding + from sp_validation.custody import CustodyError - cv = blinded_and_twin - blinded, twin = (cv.calculate_2pcf(v) for v in ("TOY", "TOY_OPEN")) + cv, cv_open, grid = blinded_and_twin + blinded, twin = cv.calculate_2pcf("TOY"), cv_open.calculate_2pcf("TOY_OPEN") shift = blinding.conceal(twin, blinding.open_blind(cv.custody("TOY"))).mean shift = shift - twin.mean @@ -546,5 +540,7 @@ def test_a_blinded_catalogues_xi_leaves_concealed(blinded_and_twin): np.testing.assert_allclose( blinded.mean - twin.mean, shift, rtol=1e-8, atol=1e-12 * np.abs(shift).max() ) - assert blinded.metadata["blinding"] == "blinded" + assert blinded.metadata["blind"] == "toy" assert cv.xi_parts["TOY", "reporting"] is blinded + with pytest.raises(CustodyError, match="shows the blind's shift"): + CosmologyValidation(versions=["TOY", "TOY_OPEN"], **grid) diff --git a/src/sp_validation/tests/test_custody.py b/src/sp_validation/tests/test_custody.py index 76165d9a..f95a64e8 100644 --- a/src/sp_validation/tests/test_custody.py +++ b/src/sp_validation/tests/test_custody.py @@ -1,9 +1,7 @@ -"""Custody is declared per base catalogue and resolved in one place. +"""Custody is declared on every catalogue entry and resolved in one place. -``sp_validation.custody.custody_of`` reads a catalogue's declaration and the -blind registry beside the catalogue config, here hand-written (the host never -decrypts, so a record needs no real seed). The host Snakemake and a container -job must resolve the same custody for every version. +Blind records are hand-written here: the host never draws, so a record needs +no real seed. """ import importlib.util @@ -12,190 +10,166 @@ import pytest import yaml -from hypothesis import example, given, settings -from hypothesis import strategies as st from sp_validation import custody as cu REPO = Path(__file__).resolve().parents[3] -def _write_blind(registry, name, bases, *, revealed=None): - """A registry record as ``blinding init`` writes it, with a stand-in seed.""" - record = registry / name - record.mkdir(parents=True) - seed = f"seed-of-{name}" - (record / "commitment.json").write_text( - json.dumps( - { - "blind": name, - "seed_commitment": cu.seed_commitment(seed), - "config_digest": "d" * 64, - "draw_scheme": 2, - } - ) - ) - (record / "bases").write_text("".join(f"{b}\n" for b in bases)) - if revealed is not None: - (record / "revealed.json").write_text(json.dumps({"seed": revealed})) - return seed - - -def _catalogues(**declarations): - """A parsed catalogue config: one entry per ``name=declaration``.""" - entries = {"nz": {"subdir": "/nz"}, "paths": {"output": "./output"}} - for name, declaration in declarations.items(): - entries[name] = { - "shear": { - "path": f"{name}.fits", - "e1_col_corrected": "e1_corrected", - "e2_col_corrected": "e2_corrected", - } +def _catalogues(tmp_path, **blinds): + """A parsed catalogue config: one entry per ``name=blind``, each reading + its own file; ``None`` declares nothing.""" + entries = { + "nz": {"subdir": "/nz"}, + "paths": {"output": "./output", "blinds": str(tmp_path / "blinds")}, + } + for name, blind in blinds.items(): + shear = { + "path": f"{name}.fits", + "e1_col_corrected": "e1c", + "e2_col_corrected": "e2c", } - if isinstance(declaration, dict): - entries[name].update(declaration) - elif declaration is not None: - entries[name]["blinding"] = declaration + entries[name] = {"subdir": str(tmp_path), "shear": shear} + if blind is not None: + entries[name]["blind"] = blind return entries -# --------------------------------------------------------------------------- # -# The custody table -# --------------------------------------------------------------------------- # -NO_BLIND = ["declares no custody for it, so it is blinded", "blinding init"] -Y3 = {"y3": (["SP_v9"], None)} # a blind over SP_v9, concealed -PUBLIC = {"y3": (["SP_v9"], "seed-of-y3")} # the same, its seed published -# case: (declarations, blinds {name: (bases, published seed)}, expected). -# expected is the custody's token, or the fragments of the refusal. -# fmt: off -TABLE = { - "undeclared": ({"SP_v9": None}, {}, NO_BLIND), - "undeclared_covered": ({"SP_v9": None}, Y3, "blinded:SP_v9:y3:{y3}"), - "blinded_revealed": ({"SP_v9": None}, PUBLIC, ["declare `blinding: unblinded`"]), - "unblinded": ({"SP_v9": "unblinded"}, {}, "unblinded:SP_v9"), - "unblinded_concealed": ({"SP_v9": "unblinded"}, Y3, ["blinding reveal y3"]), - "unblinded_revealed": ({"SP_v9": "unblinded"}, PUBLIC, "unblinded:SP_v9"), - "mock": ({"SP_v9": "mock"}, {}, "mock:SP_v9"), - "mock_covered": ({"SP_v9": "mock"}, Y3, ["a mock is never blinded"]), -} -# fmt: on - - -@pytest.mark.parametrize("case", TABLE) -def test_the_custody_table(tmp_path, case): - """custody_of: the declaration on the base catalogue against the blinds - covering it; absent means blinded.""" - declarations, blinds, expected = TABLE[case] - registry = tmp_path / "blinds" - commitments = { - name: cu.seed_commitment(_write_blind(registry, name, bases, revealed=seed)) - for name, (bases, seed) in blinds.items() - } - cats = _catalogues(**declarations) - if isinstance(expected, str): - custody = cu.custody_of(cats, "SP_v9", registry=registry) - assert custody.token == expected.format(**commitments) - return - with pytest.raises(cu.CustodyError) as refused: - cu.custody_of(cats, "SP_v9", registry=registry) - for fragment in expected: - assert fragment in str(refused.value) - - -SUFFIXES = st.lists( - st.one_of(st.just("_leak_corr"), st.integers(0, 99999).map("_seed{:05d}".format)), - max_size=3, +def _record(tmp_path, name): + """A blind's record in the registry; returns its commitment.""" + record = {"seed": f"seed-of-{name}", "envelope": {"S8": 0.075}, "draw_scheme": 2} + (tmp_path / "blinds").mkdir(exist_ok=True) + (tmp_path / "blinds" / f"{name}.blind.json").write_text(json.dumps(record)) + return cu.commitment(record) + + +def test_every_entry_declares_its_blind(tmp_path): + cats = _catalogues(tmp_path, OPEN="none", MOCK="mock", Y3="y3", NEW=None) + commitment = _record(tmp_path, "y3") + assert cu.custody_of(cats, "OPEN").token == "none" + assert cu.custody_of(cats, "MOCK").token == "mock" + assert cu.custody_of(cats, "Y3").token == f"y3:{commitment}" + with pytest.raises(cu.CustodyError, match="declares no `blind:`"): + cu.custody_of(cats, "NEW") + # _leak_corr and _seed versions take their entry's custody + for version in ("Y3_leak_corr", "Y3_seed00042_leak_corr"): + assert cu.custody_of(cats, version) == cu.custody_of(cats, "Y3") + + +def test_a_missing_blind_fails_closed_naming_the_remedy(tmp_path): + cats = _catalogues(tmp_path, SP_v9="y3") + with pytest.raises(cu.CustodyError, match="get read access") as absent: + cu.custody_of(cats, "SP_v9") + _record(tmp_path, "other") + with pytest.raises(cu.CustodyError, match="blinding init y3") as missing: + cu.custody_of(cats, "SP_v9") + for refusal in (absent, missing): + assert "SP_v9 is blinded under y3" in str(refusal.value) + assert "`blind: none` would unblind it" in str(refusal.value) + + +def test_entries_reading_one_file_declare_one_blind(tmp_path, monkeypatch): + """Within the passed config, and against the repository config, which is + authoritative for the files it names.""" + cats = _catalogues(tmp_path, TOY="y3", TWIN="none") + cats["TWIN"]["shear"]["path"] = "TOY.fits" + with pytest.raises(cu.CustodyError, match="declare one blind"): + cu.declared(cats, "TWIN") + cats["TWIN"]["blind"] = "y3" + assert cu.declared(cats, "TWIN") == "y3" + + repo = tmp_path / "repo_cat_config.yaml" + repo.write_text(yaml.safe_dump(_catalogues(tmp_path, TOY="y3"))) + monkeypatch.setattr(cu, "REPO_CAT_CONFIG", repo) + personal = _catalogues(tmp_path, MINE="none") + personal["MINE"]["shear"]["path"] = "TOY.fits" + with pytest.raises(cu.CustodyError, match="authoritative"): + cu.declared(personal, "MINE") + assert cu.of_file(tmp_path / "TOY.fits") == {"y3"} + assert cu.of_file(tmp_path / "elsewhere.fits") == set() + + +@pytest.mark.parametrize( + "blinds, allowed", + [ + ({"A": "y3", "B": "none"}, False), + ({"A": "y3", "B": "y4"}, False), + ({"A": "none", "B": "y3"}, False), + ({"A": "y3", "B": "mock"}, True), + ({"A": "y3", "B": "y3"}, True), + ({"A": "none", "B": "mock"}, True), + ], ) +def test_the_mixing_rule(blinds, allowed): + """A blinded catalogue is shown only beside mocks and its own blind.""" + if allowed: + cu.check_mix(blinds) + else: + with pytest.raises(cu.CustodyError, match="shows the blind's shift"): + cu.check_mix(blinds) + +def test_the_registry_stays_out_of_git_worktrees(tmp_path): + (tmp_path / "repo" / ".git").mkdir(parents=True) + inside = {"paths": {"blinds": str(tmp_path / "repo" / "blinds")}} + with pytest.raises(cu.CustodyError, match="inside the git worktree"): + cu.registry(inside) + root = {"paths": {"blinds": str(tmp_path / "repo")}} + assert cu.registry(root) == (tmp_path / "repo").resolve() + with pytest.raises(cu.CustodyError, match="absolute"): + cu.registry({"paths": {"blinds": "blinds"}}) -@settings(max_examples=30, deadline=None) -@given(entry=st.sampled_from(["SP_v9", "SP_v9_ecut07"]), suffixes=SUFFIXES) -@example(entry="SP_v9", suffixes=["_leak_corr"]) -@example(entry="SP_v9_ecut07", suffixes=["_seed00042", "_leak_corr"]) -def test_every_variant_shares_its_base(tmp_path_factory, entry, suffixes): - """``_leak_corr`` and ``_seed`` versions, in any order, of a catalogue - or of an entry naming its parent with ``base:``, resolve to the base's - custody and blind.""" - registry = tmp_path_factory.mktemp("variants") / "blinds" - _write_blind(registry, "y3", ["SP_v9"]) - cats = _catalogues(SP_v9=None, SP_v9_ecut07={"base": "SP_v9"}) - version = entry + "".join(suffixes) - base = cu.custody_of(cats, "SP_v9", registry=registry) - assert cu.custody_of(cats, version, registry=registry) == base - - -def test_catalogues_reading_one_file_share_one_blind(tmp_path): - """A blind conceals a shear file, whatever entry reads it: an entry - reading a concealed catalogue's file under another custody is refused - until the blind covers it too.""" - registry = tmp_path / "blinds" - _write_blind(registry, "y3", ["TOY"]) - cats = _catalogues(TOY=None, TWIN="unblinded") - cats["TWIN"]["shear"]["path"] = cats["TOY"]["shear"]["path"] - for version in ("TOY", "TWIN"): - with pytest.raises(cu.CustodyError, match="not concealed alike"): - cu.custody_of(cats, version, registry=registry) - del cats["TWIN"]["blinding"] - (registry / "y3" / "bases").write_text("TOY\nTWIN\n") - assert cu.custody_of(cats, "TWIN", registry=registry).blind == "y3" + +def test_the_commitment_is_the_forks_of_the_whole_record(): + smokescreen = pytest.importorskip("smokescreen") + record = {"seed": "s", "envelope": {"S8": 0.075}, "draw_scheme": 2} + canonical = json.dumps(record, sort_keys=True, separators=(",", ":")) + assert cu.commitment(record) == smokescreen.seed_commitment(canonical) def test_every_catalogue_in_the_repository_resolves(): - """Each entry of the committed cat_config has a custody, from the repo registry.""" - path = REPO / "cosmo_val" / "cat_config.yaml" - cats = yaml.safe_load(path.read_text()) - registry = cu.registry_of(path) - statuses = { - name: cu.custody_of(cats, name, registry=registry).status + cats = yaml.safe_load((REPO / "cosmo_val" / "cat_config.yaml").read_text()) + blinds = { + name: cu.custody_of(cats, name).blind for name in cats if name not in cu.NOT_CATALOGUES } - assert statuses and set(statuses.values()) <= set(cu.STATUSES), statuses + assert blinds and set(blinds.values()) <= {cu.NONE, cu.MOCK}, blinds + +def test_host_and_job_resolve_the_same_custody(tmp_path): + """`common.custody_token(v)` is what a job's CosmologyValidation seals under.""" + from sp_validation.cosmo_val import CosmologyValidation -# --------------------------------------------------------------------------- # -# The host and a job resolve the same custody -# --------------------------------------------------------------------------- # -def _toy_checkout(tmp_path): - """A checkout-shaped tree: workflow/common.py, src/, cosmo_val/{cat_config,blinds}.""" (tmp_path / "workflow").mkdir() (tmp_path / "workflow" / "common.py").write_text( (REPO / "workflow" / "common.py").read_text() ) (tmp_path / "src").symlink_to(REPO / "src") - cats = _catalogues( - TOY=None, - TOY_OPEN="unblinded", - TOY_MOCK="mock", - TOY_ecut07={"base": "TOY"}, - ) - for entry in (cats["TOY"], cats["TOY_OPEN"], cats["TOY_MOCK"], cats["TOY_ecut07"]): - entry["subdir"] = str(tmp_path) (tmp_path / "cosmo_val").mkdir() + cats = _catalogues(tmp_path, TOY="toy", TOY_OPEN="none", TOY_MOCK="mock") + _record(tmp_path, "toy") (tmp_path / "cosmo_val" / "cat_config.yaml").write_text(yaml.safe_dump(cats)) - _write_blind(tmp_path / "cosmo_val" / "blinds", "toy", ["TOY"]) - return tmp_path - - -def test_host_and_job_resolve_the_same_custody(tmp_path): - """The host and a job agree on each version's custody: - `common.custody_token(v)` equals `CosmologyValidation.custody(v).token`.""" - from sp_validation.cosmo_val import CosmologyValidation - - root = _toy_checkout(tmp_path) spec = importlib.util.spec_from_file_location( - "toy_common", root / "workflow" / "common.py" + "toy_common", tmp_path / "workflow" / "common.py" ) common = importlib.util.module_from_spec(spec) spec.loader.exec_module(common) common.CATALOG_CONFIG = yaml.safe_load(Path(common.CAT_CONFIG).read_text()) - versions = ["TOY", "TOY_leak_corr", "TOY_OPEN", "TOY_MOCK", "TOY_ecut07"] - cv = CosmologyValidation( - versions=versions, - catalog_config=common.CAT_CONFIG, - output_dir=str(common.COSMO_VAL), - ) - for version in versions: - assert common.custody_token(version) == cv.custody(version).token, version - assert common.custody_token("TOY").startswith("blinded:TOY:toy:") + for version in ("TOY", "TOY_leak_corr", "TOY_OPEN", "TOY_MOCK"): + token = common.custody_token(version) + job = CosmologyValidation( + versions=[version], + catalog_config=common.CAT_CONFIG, + output_dir=str(tmp_path / "out"), + custody={version: token}, + ) + interactive = CosmologyValidation( + versions=[version], + catalog_config=common.CAT_CONFIG, + output_dir=str(tmp_path / "out"), + ) + assert job.custody(version) == interactive.custody(version), version + assert job.custody(version).token == token + assert common.custody_token("TOY").startswith("toy:") diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py index 42454963..7ef2ecfa 100644 --- a/src/sp_validation/tests/test_custody_e2e.py +++ b/src/sp_validation/tests/test_custody_e2e.py @@ -1,14 +1,14 @@ -"""End to end: one blinded catalogue through its rule scripts, then its reveal. - -A synthetic catalogue ``TOY`` is declared blinded and its blind drawn with -``blinding init``. The ξ± and assembly rule scripts run as Snakemake runs them -(``runpy`` with a ``snakemake`` object); the assembled file verifies against -the declaration, and no file of the blinded run holds a true ξ± value. The -blind is then revealed, the declaration flipped, the chain re-run, and the -audit proves blinded − true = shift(seed). +"""End to end: one blinded catalogue through its rule scripts, then unblinded. + +A synthetic catalogue ``TOY`` is declared under a blind drawn with ``blinding +init``. The ξ± and assembly rule scripts run as Snakemake runs them (``runpy`` +with a ``snakemake`` object), and no file of the blinded run holds a true ξ± +value. The declaration is then flipped to ``blind: none`` and the chain re-run +into a fresh tree: blinded − true is the blind's shift, and nothing but the +mean moved. """ -import json +import dataclasses import re import runpy import types @@ -21,6 +21,7 @@ from sp_validation import blinding as bd from sp_validation import custody as cu from sp_validation import sacc_io as sio +from sp_validation.blinding_theory import TheoryConfig SCRIPTS = Path(__file__).resolve().parents[3] / "workflow" / "scripts" GRID = {"min_sep": 5.0, "max_sep": 60.0, "nbins": 6, "npatch": 1} @@ -56,7 +57,7 @@ def run_rule(script, **fields): def run_chain(cat_config, out, cov): """Rules xi and assemble_sacc for TOY into ``out``.""" out.mkdir(exist_ok=True) - token = bd.declared_custody(cat_config, "TOY").token + token = cu.custody_of(yaml.safe_load(cat_config.read_text()), "TOY").token common = {"cat_config": str(cat_config), "custody": token} xi = out / "TOY_xi_reporting.sacc" run_rule( @@ -70,7 +71,7 @@ def run_chain(cat_config, out, cov): "assemble_sacc.py", input={"xi_reporting": str(xi), "xi_cov": str(cov)}, output={"sacc": str(out / "TOY.sacc")}, - params={"version": "TOY", "expected": ["xi_reporting"]} | common, + params={"version": "TOY", "expected": ["xi_reporting"], "custody": token}, ) @@ -102,48 +103,51 @@ def near(x, rtol): return found -def test_a_blinded_catalogue_from_birth_to_audit(tmp_path, monkeypatch): +def test_a_blinded_catalogue_from_birth_to_unblinding(tmp_path, monkeypatch): monkeypatch.syspath_prepend(str(SCRIPTS)) import cv_runner # Rule jobs line-buffer their own streams; under pytest they are captured. monkeypatch.setattr(cv_runner, "_unbuffer_streams", lambda: None) params, _ = write_synthetic_catalogs( - tmp_path, n_gal=20000, coherent_shear=True, catalogues={"TOY": None} + tmp_path, n_gal=20000, coherent_shear=True, catalogues={"TOY": "toy"} ) cat_config = Path(params["catalog_config"]) - fast = tmp_path / "fast.json" - fast.write_text(json.dumps({"theory": {"transfer_function": "eisenstein_hu"}})) - monkeypatch.setattr(bd.secrets, "token_hex", lambda n: "e2e-seed") - bd.main( - ["init", "toy", "TOY", "--cat-config", str(cat_config), "--config", str(fast)] - ) + config = yaml.safe_load(cat_config.read_text()) + fast = TheoryConfig(transfer_function="eisenstein_hu") + blind = bd.init("toy", config, fiducial=dataclasses.asdict(fast)) cov = tmp_path / "cov.txt" np.savetxt(cov, np.diag(np.full(2 * GRID["nbins"], 1e-10))) - out = tmp_path / "cosmo_val" # --- the blinded run ----------------------------------------------------- - run_chain(cat_config, out, cov) - blinded = bd.declared_custody(cat_config, "TOY") - for part in out.glob("*.sacc"): - assert cu.read_stamp(sio.load(part).metadata).stamp == blinded.stamp - verify = ["verify", str(out / "TOY.sacc"), "--cat-config", str(cat_config)] - assert bd.main(verify) == 0 - blinded_run = {str(p): p.read_bytes() for p in out.rglob("*") if p.is_file()} - - # --- the reveal: archive, flip the declaration, re-measure, audit -------- - reveal = ["reveal", "toy", "--root", str(out), "--cat-config", str(cat_config)] - assert bd.main(reveal) == 0 - config = yaml.safe_load(cat_config.read_text()) - config["TOY"]["blinding"] = "unblinded" + blinded_out = tmp_path / "blinded" + run_chain(cat_config, blinded_out, cov) + custody = cu.custody_of(config, "TOY") + for part in blinded_out.glob("*.sacc"): + assert cu.read_stamp(sio.load(part).metadata) == custody + blinded_run = { + str(p): p.read_bytes() for p in blinded_out.rglob("*") if p.is_file() + } + + # --- flip to public, re-measure into a fresh tree ------------------------ + config["TOY"]["blind"] = "none" cat_config.write_text(yaml.safe_dump(config, sort_keys=False)) - run_chain(cat_config, out, cov) - archive = out / "revealed" / "toy" - report = bd.audit("toy", archive=archive, true_root=out, cat_config=cat_config) - assert report["ok"], json.dumps(report, indent=1, default=str) - assert len(report["parts"]) == 2 + true_out = tmp_path / "true" + run_chain(cat_config, true_out, cov) + + for name in ("TOY_xi_reporting.sacc", "TOY.sacc"): + blinded, true = (sio.load(out / name) for out in (blinded_out, true_out)) + assert cu.read_stamp(true.metadata) == cu.Custody("none") + shift = bd.conceal(true, blind).mean - true.mean + assert np.all(shift != 0) + np.testing.assert_allclose( + blinded.mean - true.mean, shift, rtol=1e-8, atol=1e-12 * np.abs(shift).max() + ) + np.testing.assert_allclose( + blinded.covariance.dense, true.covariance.dense, rtol=1e-10 + ) # --- no true ξ± value was written by the blinded run --------------------- - gg = sio.xi_correlation(sio.load(out / "TOY_xi_reporting.sacc")) + gg = sio.xi_correlation(sio.load(true_out / "TOY_xi_reporting.sacc")) assert not plaintext(blinded_run, np.concatenate([gg.xip, gg.xim])) assert not list(tmp_path.rglob("*_xi_*.txt")) diff --git a/src/sp_validation/tests/test_cv_init_params.py b/src/sp_validation/tests/test_cv_init_params.py index ffc87d8b..03c691ea 100644 --- a/src/sp_validation/tests/test_cv_init_params.py +++ b/src/sp_validation/tests/test_cv_init_params.py @@ -16,7 +16,9 @@ REPO = Path(__file__).resolve().parents[3] -EXEMPT = {} +EXEMPT = { + "custody": "per-version custody tokens, passed from the rule's params.custody", +} def _load_common(): diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 2efe24e8..5a9128f0 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -150,7 +150,7 @@ def _write_synthetic_config(tmp_path): "star": {**psf_cfg}, "psf": psf_cfg, "patch_number": 150, - "blinding": "mock", + "blind": "mock", }, } config_path = tmp_path / "config.yaml" diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 54f2ed40..59857c38 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -63,7 +63,7 @@ def _cosebi_block(s, tr): # --------------------------------------------------------------------------- # # 1. Per-writer round-trip (arrays / tags / windows / NZ bitwise) # --------------------------------------------------------------------------- # -MOCK = Custody("mock", "vTEST") +MOCK = Custody("mock") def _roundtrip(s, tmp_path, name="rt"): diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index 159a3e92..602e874a 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -33,7 +33,7 @@ def _theta(n=6): return np.geomspace(1.0, 100.0, n) -MOCK = Custody("mock", "vSYNTH") +MOCK = Custody("mock") def _roundtrip(s, tmp_path, name): diff --git a/uv.lock b/uv.lock index 8c905aa5..88df41c5 100644 --- a/uv.lock +++ b/uv.lock @@ -3612,7 +3612,6 @@ dependencies = [ { name = "clmm" }, { name = "colorama" }, { name = "cosmo-numba" }, - { name = "cryptography" }, { name = "cs-util" }, { name = "emcee" }, { name = "getdist" }, @@ -3696,7 +3695,6 @@ requires-dist = [ { name = "cosmo-numba", git = "https://github.com/aguinot/cosmo-numba.git?rev=main" }, { name = "cosmology", marker = "extra == 'glass'", specifier = "==2022.10.9" }, { name = "cosmosis", marker = "extra == 'workflow'", specifier = ">=3.25" }, - { name = "cryptography" }, { name = "cs-util", git = "https://github.com/CosmoStat/cs_util.git?rev=develop" }, { name = "emcee" }, { name = "fast-pt", marker = "extra == 'workflow'", specifier = ">=3.2,<4" }, diff --git a/workflow/README.md b/workflow/README.md index 2153f64f..b774e8ef 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -27,58 +27,32 @@ each namespaces cleanly under `results//`. ## Custody: which catalogues are blinded -Every catalogue in `cosmo_val/cat_config.yaml` declares its custody on its base -entry, `blinding: blinded | unblinded | mock`; an entry that declares none is -blinded. Variants (`_leak_corr`, `_seed`, and entries naming their parent -with `base:`) share their base's custody and blind; an entry named like a -variant is one, and may repeat its parent's `blinding` or `base`, never declare -another. A blind conceals data, not a name: catalogues reading one shear file -are concealed under one blind or none, so a blind is drawn (or shared) for all -of them. A launch prints one -`[custody]` line per catalogue it runs; no `--config` or config file changes a -declaration. - -For a blinded catalogue, every ξ± and pseudo-Cℓ_EE value is shifted by a hidden -cosmology before it is first written. COSEBIs and pure-E/B computed from the -shifted ξ± carry the blind with them; B-modes stay usable. Every SACC file -records the custody it was born under, and assembly refuses parts under any -other. Under a blind a SACC holds only data types with a blinding rule -(`sacc_io.SIGNAL` and ρ/τ), so a new statistic is refused until it has one. ξ± -leaves `CosmologyValidation.calculate_2pcf` only as its sealed part -(no text dump), so the ξ± figures and notebooks draw what the part holds; the -jackknife `calculate_pure_eb` and the aperture mass, which need TreeCorr's own -measurement, refuse a blinded catalogue. - -A blinded catalogue needs a blind, drawn once, by a person. The launch stops -with this command when none covers it: +Every entry of `cosmo_val/cat_config.yaml` declares `blind: none` (public), +`blind: mock`, or the name of the blind its signal is concealed under; an entry +without one is refused. `_leak_corr` and `_seed` versions take their entry's. +Entries reading one shear file declare one blind, and the repository config is +authoritative for the files it names. A run may show a blinded catalogue only +beside mocks and catalogues under the same blind, since overlaying it with any +other version shows the shift. A launch prints one `[custody]` line per +catalogue. + +Under a blind, every ξ± and pseudo-Cℓ_EE value is shifted by t(hidden) − +t(fiducial) before it is first written; COSEBIs and pure-E/B computed from the +shifted ξ± carry the blind, and B-modes stay usable. Every SACC records the +custody it was born under, and rule params carry the custody token, so a flip +reruns what it touches. + +A blind is one record, `/.blind.json`, outside any git +worktree; read access to it is access to the blind. Draw one, once: ```bash -APPTAINERENV_PYTHONPATH=$PWD/src spv-container exec python -m sp_validation.blinding \ - init --cat-config cosmo_val/cat_config.yaml +spv-container exec python -m sp_validation.blinding init ``` -then commit `cosmo_val/blinds//`. No rule draws or rewrites a blind, so -`-F`, `--forcerun` or a fresh output tree reuse it, and every collaborator's run -of the catalogue uses the same one. A re-processing of a catalogue whose blind -is still concealed shares it: `… blinding share ---cat-config …`. - -Revealing re-measures; nothing subtracts a shift: - -1. `… blinding reveal --root --cat-config …` publishes the - seed (`revealed.json`) and moves every file concealed under the blind to - `/revealed//`; -2. one reviewed commit adds `revealed.json` and declares `blinding: unblinded` - on the blind's catalogues; -3. the pipeline re-measures the true products; -4. `… blinding audit --archive /revealed/ --true-root - --cat-config …` checks blinded − true = shift(seed) on every ξ± - and Cℓ_EE row of every archived file, and bounds the derived B-modes (their - E-mode shifts are reported). ρ/τ is judged by its stamp: it carries no - signal, and a re-run does not reproduce it. - -`… blinding verify --cat-config …` checks a file's stamp against the -registry, without the seed. +`… blinding show ` prints its public record. For an entry under a blind, +measure signal only through `CosmologyValidation` or the workflow; never set +`blind: none` to get a run through; never print, paste or commit a +`.blind.json`. ## Running on the cluster — the candide profile diff --git a/workflow/common.py b/workflow/common.py index 55c214e0..80560270 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -5,10 +5,10 @@ stdlib-only project modules it needs by file path. """ -import functools import importlib.util import json import os +import re import sys from pathlib import Path @@ -65,10 +65,8 @@ def _load_checkout_module(name): ) ) # The catalogue config of the launched checkout: the one file both the host -# (CATALOG_CONFIG, loaded in configure) and every job read catalogues from, and -# the blind registry beside it. +# (CATALOG_CONFIG, loaded in configure) and every job read catalogues from. CAT_CONFIG = str(REPO_ROOT / "cosmo_val" / "cat_config.yaml") -REGISTRY = _custody.registry_of(CAT_CONFIG) BLOCK_PAIRS = [("++", "1"), ("--", "2"), ("+-", "3")] # Fiducial cosmology: Planck 2018 (astropy Planck18, Table 2 + BAO) @@ -194,43 +192,27 @@ def configure(workflow_config): def announce_custody(workflow_config): - """Print each run catalogue's custody; stop the launch if one cannot run. + """Print each run catalogue's custody, or stop the launch with why it cannot run. - Resolves every version the config names, so a blinded catalogue without a - blind fails here, with the command to run, before any job is scheduled. + Every version the config names must resolve (a blinded one, to its blind), + and the overlaid ``versions`` must pass the mixing rule. """ from snakemake.exceptions import WorkflowError - if "type" in workflow_config.get("cosmo_val", {}): - raise WorkflowError( - "cosmo_val.type is not a setting: custody is declared per catalogue " - "in cosmo_val/cat_config.yaml. Remove it from the config." - ) - versions = [ - *workflow_config.get("versions", []), - *(FIDUCIAL.get(k) for k in ("version", "mock_version") if FIDUCIAL.get(k)), - ] + versions = workflow_config.get("versions", []) + fiducial = [FIDUCIAL.get(k) for k in ("version", "mock_version")] try: - lines = _custody.summary(CATALOG_CONFIG, versions, registry=REGISTRY) + _custody.check_mix({v: _custody.declared(CATALOG_CONFIG, v) for v in versions}) + lines = _custody.summary(CATALOG_CONFIG, [*versions, *filter(None, fiducial)]) except _custody.CustodyError as err: raise WorkflowError(str(err)) from None - _print_once(tuple(lines)) - - -@functools.cache -def _print_once(lines): - """Print ``lines`` once per launch, however many Snakefiles configure.""" - for line in lines: - print(line, file=sys.stderr) + print("\n".join(lines), file=sys.stderr) def custody_token(version): - """The custody token of ``version``: a producer's ``params`` trigger. - - A producer carrying it re-runs when its catalogue's custody changes, and its - job refuses to run under any other custody. - """ - return _custody.custody_of(CATALOG_CONFIG, version, registry=REGISTRY).token + """``version``'s custody token: a producer's ``params.custody``, so a + custody change reruns it, and the custody its job seals under.""" + return _custody.custody_of(CATALOG_CONFIG, version).token def fiducial_binning_suffix(fiducial=None): @@ -293,11 +275,9 @@ def covariance_path( def base_version(version): - """The base catalogue of ``version``: its entry, then its ``base:`` links. - - Variants share their base's footprint and plotting style. - """ - return _custody.base_catalogue(CATALOG_CONFIG, version) + """Strip the `_leak_corr` / `_ecut{N}` suffixes to the base catalogue + version, whose footprint and plotting style its variants share.""" + return re.sub(r"_ecut\d+", "", re.sub(r"_leak_corr$", "", version)) def catalogue_entry(version): diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 1dd844c6..1ccb2bab 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -508,7 +508,6 @@ rule assemble_sacc: sacc=cv_analysis_sacc("{version}"), params: version="{version}", - cat_config=CAT_CONFIG, custody=lambda w: custody_token(w.version), # The statistics this rule wired, so a typo'd input keyword cannot # silently drop one. diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 947ea27b..48f1f012 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -16,11 +16,11 @@ import argparse import numpy as np +import yaml +from sp_validation import custody as _custody from sp_validation import sacc_io -from sp_validation.blinding import declared_custody from sp_validation.cosmo_val.sacc_writers import assemble_analysis_sacc -from sp_validation.custody import confirm # NaMaster iNKA covariance FITS: per-spectrum HDU names, in SACC insertion order. _CL_HDUS = ("COVAR_EE_EE", "COVAR_BB_BB", "COVAR_EB_EB") @@ -101,10 +101,9 @@ def assemble_sacc( """Assemble ``{version}.sacc`` from the per-statistic ``part_paths`` mapping. @sc one-custody-per-assembly - Every part, signal-bearing or not, must carry one stamp, and it must be the - custody ``version`` is declared under: a stale concealed part after a - reveal, a part under another blind or catalogue, a mock in data and - unblinded parts in a blinded file are all refused. + Every part, signal-bearing or not, must carry one stamp, and it must be + ``custody``: a stale part after a custody flip, a part under another blind, + a mock in data and public parts in a blinded file are all refused. Parameters ---------- @@ -117,7 +116,7 @@ def assemble_sacc( Statistics that must be present, from the caller's config toggles. A typo'd input keyword would otherwise silently drop a statistic. custody : sp_validation.custody.Custody - The custody ``version`` is declared under. + The custody ``version`` runs under. xi_cov, pseudo_cl_cov Covariance sourcing — see the module docstring. """ @@ -162,7 +161,7 @@ def _from_snakemake(smk): version=p["version"], part_paths=part_paths, out_path=str(smk.output[0]), - custody=confirm(declared_custody(p["cat_config"], p["version"]), p["custody"]), + custody=_custody.parse(p["custody"]), expected=list(p["expected"]), xi_cov=getattr(inp, "xi_cov", None), pseudo_cl_cov=getattr(inp, "pseudo_cl_cov", None), @@ -194,7 +193,7 @@ def _from_cli(argv=None): version=a.version, part_paths=part_paths, out_path=a.out, - custody=declared_custody(a.cat_config, a.version), + custody=_custody.custody_of(yaml.safe_load(open(a.cat_config)), a.version), xi_cov=a.xi_cov, pseudo_cl_cov=a.pseudo_cl_cov, ) diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index a8ef827c..a548f488 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -25,7 +25,6 @@ import os from sp_validation.cosmo_val import CosmologyValidation -from sp_validation.custody import confirm def generate_pseudo_cl( @@ -66,7 +65,7 @@ def generate_pseudo_cl( Power for powspace binning (0.5 = sqrt spacing) custody : str, optional The custody token Snakemake resolved for ``version`` (the rule's - ``params.custody``); the part is not written under any other. + ``params.custody``), which the part is sealed under. Returns ------- @@ -120,6 +119,7 @@ def generate_pseudo_cl( theta_min=1.0, theta_max=250.0, nbins=20, + custody=None if custody is None else {version: custody}, ) if binning == "linear": cv_kwargs["ell_step"] = ell_step @@ -128,8 +128,6 @@ def generate_pseudo_cl( cv_kwargs["power"] = power cv = CosmologyValidation(**cv_kwargs) - if custody is not None: - confirm(cv.custody(version), custody) # Pseudo-Cls only (no covariance), born directly at the final out_path. cv.calculate_pseudo_cl(out_path=out_path) diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index c79ee191..2f1f4fc4 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -25,7 +25,6 @@ import os from sp_validation.cosmo_val import CosmologyValidation -from sp_validation.custody import confirm def run_2pcf( @@ -50,7 +49,7 @@ def run_2pcf( the part (the Snakemake-declared output); it defaults to a binning-derived name under the resolved output directory for the CLI path. ``custody`` is the custody token Snakemake resolved for ``ver`` (the rule's - ``params.custody``); the part is not written under any other. + ``params.custody``), which the part is sealed under. Returns ------- @@ -58,10 +57,11 @@ def run_2pcf( The part as written. """ cv = CosmologyValidation( - versions=[ver], catalog_config=cat_config, output_dir=output_dir + versions=[ver], + catalog_config=cat_config, + output_dir=output_dir, + custody=None if custody is None else {ver: custody}, ) - if custody is not None: - confirm(cv.custody(ver), custody) out_path = sacc_out or os.path.join( output_dir or cv.cc["paths"]["output"], f"{ver}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc", diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index 4b16fd59..26f0b349 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -7,7 +7,6 @@ from cv_runner import _unbuffer_streams, verify_outputs from sp_validation.cosmo_val import CosmologyValidation -from sp_validation.custody import confirm _unbuffer_streams() params = snakemake.params # noqa: F821 — injected by Snakemake's script: directive @@ -20,7 +19,7 @@ npatch=int(params["npatch"]), catalog_config=params["cat_config"], output_dir=params["output_dir"], + custody={params["ver"]: params["custody"]}, ) -confirm(cv.custody(params["ver"]), params["custody"]) cv.calculate_rho_tau_stats() verify_outputs(snakemake) # noqa: F821 diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py index 6d00ae4f..8c7dab63 100644 --- a/workflow/tests/conftest.py +++ b/workflow/tests/conftest.py @@ -11,13 +11,12 @@ The ``toy`` fixture is a disposable checkout: copies of ``workflow/`` and ``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, one catalogue per -custody state, hand-written blind records (the host never decrypts), stand-ins +custody state, a hand-written blind record (the host never draws), stand-ins for the processed CosmoCov covariances (their inputs live on candide), and both output roots in tmp. """ import dataclasses -import hashlib import importlib.util import json import os @@ -34,10 +33,11 @@ # The toy catalogue and its leakage-corrected variant: blinded under `toy`. VERSIONS = ("SP_v0.1", "SP_v0.1_leak_corr") -# Declared unblinded, and covered by the blind `stale`, which is not revealed. -STALE = "SP_v0.5" -# Declares no custody, and no blind covers it. +PUBLIC = "SP_v0.2" +# Declared under the blind `gone`, which the registry does not hold. UNCOVERED = "SP_v0.4" +# Declares no blind. +UNDECLARED = "SP_v0.5" @dataclasses.dataclass @@ -130,7 +130,7 @@ def snakemake(self, *args, config=(), cwd=None, env=None, timeout=300): ) -def _cat_config(data): +def _cat_config(data, registry): """One catalogue per custody state, each reading its own touched file.""" def entry(name, **declaration): @@ -155,31 +155,20 @@ def entry(name, **declaration): } return { - VERSIONS[0]: entry(VERSIONS[0]), - "SP_v0.2": entry("SP_v0.2", blinding="unblinded"), - "SP_v0.3": entry("SP_v0.3", blinding="mock"), - UNCOVERED: entry(UNCOVERED), - STALE: entry(STALE, blinding="unblinded"), - "paths": {"output": "./output"}, + VERSIONS[0]: entry(VERSIONS[0], blind="toy"), + PUBLIC: entry(PUBLIC, blind="none"), + "SP_v0.3": entry("SP_v0.3", blind="mock"), + UNCOVERED: entry(UNCOVERED, blind="gone"), + UNDECLARED: entry(UNDECLARED), + "paths": {"output": "./output", "blinds": str(registry)}, } -def _registry(root): - """Blind records as ``blinding init`` commits them, minus the ciphertext.""" - for name, bases in (("toy", [VERSIONS[0]]), ("stale", [STALE])): - record = root / "cosmo_val" / "blinds" / name - record.mkdir(parents=True) - (record / "commitment.json").write_text( - json.dumps( - { - "blind": name, - "seed_commitment": hashlib.sha256(name.encode()).hexdigest(), - "config_digest": hashlib.sha256(b"config").hexdigest(), - "draw_scheme": 2, - } - ) - ) - (record / "bases").write_text("".join(f"{b}\n" for b in bases)) +def _registry(registry): + """The blind `toy`, as ``blinding init`` writes it but for a stand-in seed.""" + registry.mkdir() + record = {"seed": "toy-seed", "envelope": {"S8": 0.075}, "draw_scheme": 2} + (registry / "toy.blind.json").write_text(json.dumps(record)) @pytest.fixture(scope="session") @@ -195,9 +184,9 @@ def toy(tmp_path_factory): (root / "data").mkdir() (root / "cosmo_val").mkdir() (root / "cosmo_val" / "cat_config.yaml").write_text( - yaml.safe_dump(_cat_config(root / "data")) + yaml.safe_dump(_cat_config(root / "data", root / "blinds")) ) - _registry(root) + _registry(root / "blinds") rundir = root / "papers" / "cosmo_val" config_path = rundir / "config" / "config.yaml" diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 494c9d90..c7862d12 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -9,9 +9,10 @@ import pytest import yaml from conftest import ( + PUBLIC, REPO, - STALE, UNCOVERED, + UNDECLARED, VERSIONS, on_candide, parse_jobs, @@ -63,11 +64,11 @@ def test_one_integration_grid(toy, forced, grids): def test_custody_is_the_checkouts_and_no_rule_touches_a_blind(toy, forced, tmp_path): """[P9, P10, P8] A catalogue and its variant share one custody line; even a forced run schedules nothing that reads or writes the registry; and a - config file declaring the catalogue unblinded changes nothing.""" + config file declaring the catalogue public changes nothing.""" output, jobs = forced line = f"[custody] {VERSIONS[0]} (+ {VERSIONS[1]}): blinded under toy" - assert [x for x in output.splitlines() if x.startswith("[custody]")] == [line] - registry = (toy.root / "cosmo_val" / "blinds").resolve() + assert {x for x in output.splitlines() if x.startswith("[custody]")} == {line} + registry = (toy.root / "blinds").resolve() assert not [j.rule for j in jobs if "blind" in j.rule] touched = [ f @@ -78,42 +79,26 @@ def test_custody_is_the_checkouts_and_no_rule_touches_a_blind(toy, forced, tmp_p assert not touched, touched override = tmp_path / "override.yaml" - override.write_text(yaml.safe_dump({VERSIONS[0]: {"blinding": "unblinded"}})) + override.write_text(yaml.safe_dump({VERSIONS[0]: {"blind": "none"}})) result = toy.snakemake("-n", "assemble_sacc_all", "--configfile", str(override)) assert result.returncode == 0, result.stdout assert line in result.stdout -def _launch_refusals(toy, tmp_path): - """Launches the DAG cannot honour: ``(args, Toy.snakemake kwargs, message)``.""" - campaign = tmp_path / "campaign.yaml" - config = dict(toy.config, cosmo_val=dict(toy.config["cosmo_val"], type="mock")) - campaign.write_text(yaml.safe_dump(config)) - return { - # [P6] a blinded catalogue no blind covers: pull, then draw or share - "no_blind": ( - [], - dict(config=[f'versions=["{UNCOVERED}"]']), - ["git pull first", "python -m sp_validation.blinding init", "share"], - ), - # [P11] unblinded under a concealed blind, and the removed campaign switch - "unrevealed": ( - [], - dict(config=[f'versions=["{STALE}"]']), - ["blinding reveal stale"], - ), - "campaign_type": ( - ["--configfile", str(campaign)], - {}, - ["custody is declared per catalogue in cosmo_val/cat_config.yaml"], - ), - } - - -@pytest.mark.parametrize("case", ["no_blind", "unrevealed", "campaign_type"]) -def test_a_launch_the_dag_cannot_honour_stops_with_the_fix(toy, tmp_path, case): - args, launch, message = _launch_refusals(toy, tmp_path)[case] - result = toy.snakemake("-n", "assemble_sacc_all", *args, **launch) +LAUNCH_REFUSALS = { + # [P6] a blind the registry does not hold, an entry declaring none, and a + # blinded catalogue overlaid with a public one + "no_blind": ([UNCOVERED], [f"{UNCOVERED} is blinded under gone", "init gone"]), + "undeclared": ([UNDECLARED], ["declares no `blind:`"]), + "mixed": ([VERSIONS[0], PUBLIC], ["shows the blind's shift"]), +} + + +@pytest.mark.parametrize("case", LAUNCH_REFUSALS) +def test_a_launch_the_dag_cannot_honour_stops_with_the_fix(toy, case): + versions, message = LAUNCH_REFUSALS[case] + listed = ", ".join(f'"{v}"' for v in versions) + result = toy.snakemake("-n", "assemble_sacc_all", config=[f"versions=[{listed}]"]) assert result.returncode != 0, result.stdout for fragment in message: assert fragment in result.stdout, result.stdout diff --git a/workflow/tests/test_toy_run.py b/workflow/tests/test_toy_run.py index b81050a4..b22998d7 100644 --- a/workflow/tests/test_toy_run.py +++ b/workflow/tests/test_toy_run.py @@ -4,9 +4,9 @@ host Snakemake, the image `spv-container` manages, jobs on this node. A toy checkout under your home directory (which the profile binds) carries copies of workflow/, papers/cosmo_val/ and src/, and one synthetic catalogue declared -unblinded. The first launch makes its reporting parts and their figure; then -the catalogue is declared blinded, a blind is drawn for it, and the same launch -re-measures the parts concealed. +public. The first launch makes its reporting parts and their figure; then a +blind is drawn and declared on the catalogue, and the same launch re-measures +the parts concealed. """ import json @@ -19,7 +19,9 @@ import pytest import yaml -from conftest import REPO, container, on_candide +from conftest import REPO, _load_module, container, on_candide + +custody = _load_module(REPO / "src" / "sp_validation" / "custody.py", "custody", {}) VERSIONS = ("SP_v0.1", "SP_v0.1_leak_corr") REPORTING = {"theta_min": 5.0, "theta_max": 60.0, "nbins": 6, "npatch": 4} @@ -56,7 +58,7 @@ def _stamps(image, root, parts): def _toy_checkout(root, image): - """A checkout with one synthetic catalogue, SP_v0.1, declared unblinded.""" + """A checkout with one synthetic catalogue, SP_v0.1, declared public.""" skip = shutil.ignore_patterns(".snakemake", "__pycache__", "tests") shutil.copytree(REPO / "workflow", root / "workflow", ignore=skip) shutil.copytree( @@ -76,7 +78,7 @@ def _toy_checkout(root, image): "from pathlib import Path\n" "from _synthetic import write_synthetic_catalogs\n" f"write_synthetic_catalogs(Path({str(root / 'cosmo_val')!r})," - f" catalogues={{{VERSIONS[0]!r}: 'unblinded'}})", + f" catalogues={{{VERSIONS[0]!r}: 'none'}})", ) config_path = root / "papers" / "cosmo_val" / "config" / "config.yaml" @@ -86,9 +88,6 @@ def _toy_checkout(root, image): config["fiducial"]["mock_version"] = VERSIONS[0] config["cosmo_val"].update(REPORTING) config_path.write_text(yaml.safe_dump(config, sort_keys=False)) - (root / "fast.json").write_text( - json.dumps({"theory": {"transfer_function": "eisenstein_hu"}}) - ) @pytest.mark.candide @@ -141,40 +140,41 @@ def launch(*args): check=False, ) - # Declared unblinded: parts stamped unblinded, and their figure drawn. + # Declared public: parts stamped public, and their figure drawn. result = launch() assert result.returncode == 0, result.stdout - assert [s["blinding"] for s in _stamps(image, root, parts)] == ["unblinded"] * 2 + assert [s["blind"] for s in _stamps(image, root, parts)] == ["none"] * 2 assert (out / "xi_p.png").is_file() - # Declared blinded, and its blind drawn: the parts' params changed. - catalogues = yaml.safe_load(cat_config.read_text()) - catalogues[VERSIONS[0]]["blinding"] = "blinded" - cat_config.write_text(yaml.safe_dump(catalogues, sort_keys=False)) + # A blind drawn and declared: the parts' params changed. _in_image( image, root, "python", - "-m", - "sp_validation.blinding", - "init", - "toy", - VERSIONS[0], - "--cat-config", + "-c", + "import dataclasses, sys, yaml\n" + "from sp_validation import blinding\n" + "from sp_validation.blinding_theory import TheoryConfig\n" + "fast = TheoryConfig(transfer_function='eisenstein_hu')\n" + "blinding.init('toy', yaml.safe_load(open(sys.argv[1]))," + " fiducial=dataclasses.asdict(fast))", str(cat_config), - "--config", - str(root / "fast.json"), ) + catalogues = yaml.safe_load(cat_config.read_text()) + catalogues[VERSIONS[0]]["blind"] = "toy" + cat_config.write_text(yaml.safe_dump(catalogues, sort_keys=False)) dry = launch("-n") assert dry.returncode == 0, dry.stdout assert "Params have changed" in dry.stdout, dry.stdout result = launch() assert result.returncode == 0, result.stdout - record = json.loads((root / "cosmo_val/blinds/toy/commitment.json").read_text()) + record = json.loads((root / "cosmo_val/blinds/toy.blind.json").read_text()) for stamp in _stamps(image, root, parts): - assert stamp["blinding"] == "blinded", stamp - assert stamp["blinding_commitment"] == record["seed_commitment"], stamp + assert stamp == { + "blind": "toy", + "blind_commitment": custody.commitment(record), + } assert not list(out.rglob("*_xi_*.txt")) assert not list((root / "cosmo_val" / "output").iterdir()) From d4b52e3eed91165c2f95381d12fd2e9ee57cbfc4 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 03:00:13 +0200 Subject: [PATCH 139/160] =?UTF-8?q?Theory=20is=20one=20pluggable=20functio?= =?UTF-8?q?n=20per=20data=20type;=20=CE=B3t=20is=20blindable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `theory.py` maps each SACC data type to f(params, s, rows): ξ±, Cℓ_EE and γt default to CCL through cs_util.cosmo.get_cosmo on the CAMB HMCode2020-feedback route the CosmoSIS inference runs, and `predict` groups rows by function, tracer pair and window. `conceal` shifts `sacc_io.shiftable` rows by predict(hidden) − predict(fiducial), refuses a shiftable type without a theory and a (type, pair) left unmoved, and silences the theory's output at the hidden point. γt joins SHIFTABLE and γ× UNSHIFTED; γt around stars or randoms is a null and stays unshifted. Source tracers carry quantity galaxy_shear, and `add_lens`/`add_gamma_t` write lens samples and γt. The fiducial is cs_util's Planck18 with logT_AGN 7.5, no IA and unit lens bias. `blinding_theory.py` and its bespoke cosmology are gone; tests shift with an analytic toy theory, and one slow test runs the CCL defaults. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 1 + src/sp_validation/blinding.py | 113 ++++----- src/sp_validation/blinding_theory.py | 260 -------------------- src/sp_validation/sacc_io.py | 78 ++++-- src/sp_validation/tests/_synthetic.py | 15 ++ src/sp_validation/tests/conftest.py | 10 + src/sp_validation/tests/test_blinding.py | 124 ++++++++-- src/sp_validation/tests/test_cosmo_val.py | 8 +- src/sp_validation/tests/test_custody_e2e.py | 9 +- src/sp_validation/theory.py | 187 ++++++++++++++ workflow/README.md | 9 +- workflow/tests/test_toy_run.py | 7 +- 12 files changed, 428 insertions(+), 393 deletions(-) delete mode 100644 src/sp_validation/blinding_theory.py create mode 100644 src/sp_validation/theory.py diff --git a/CLAUDE.md b/CLAUDE.md index 48db4a4e..57dad8db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,6 +42,7 @@ is the container (full scientific stack pre-built). For a local dev environment: - `cat.py`: Catalogue handling and manipulation - `cosmo_val.py`: Cosmology validation routines - `cosmology.py`: Cosmological calculations and theory +- `theory.py`: Theory data vectors for SACC rows, one pluggable function per data type - `galaxy.py`: Galaxy-specific processing - `io.py`: Input/output utilities - `plots.py`: Plotting functions diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 42af6520..860baf20 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -10,8 +10,9 @@ by exception type alone. @sc hidden-draw-uniform-s8-om -The hidden point is the fiducial moved by the fork's per-key draw from the -seed, uniform within the envelope in S8 and Ωm. +The hidden point is the fiducial (:func:`sp_validation.theory.fiducial`) moved +by the fork's per-key draw from the seed, uniform within the envelope in S8 and +Ωm. ``python -m sp_validation.blinding init `` draws a blind; ``show `` prints its public record (everything but the seed). @@ -19,18 +20,20 @@ import argparse import dataclasses +import io import json import os import secrets import warnings from collections.abc import Mapping +from contextlib import redirect_stderr, redirect_stdout from pathlib import Path import numpy as np from . import custody as _custody from . import sacc_io -from .blinding_theory import TheoryConfig, cl_ee, xi_ccl +from . import theory as _theory ENVELOPE = {"S8": 0.075, "Omega_m": 0.1} SECRET = "secret: never print, paste or commit" @@ -106,53 +109,18 @@ def _hidden(blind): return _Hidden({k: v + shift[k] if k in shift else v for k, v in fiducial.items()}) -def _nz(s, name): - tracer = s.tracers[name] - if not hasattr(tracer, "nz"): - raise ValueError(f"shiftable rows name tracer {name}, which has no n(z)") - return np.asarray(tracer.z, float), np.asarray(tracer.nz, float) +def _at_hidden(s, rows, theory, blind): + """The theory of ``rows`` at the hidden point; a failure names only its type. - -def _blocks(s): - """Each ξ± tracer pair and Cℓ_EE (pair, window): rows, and theory(params, config).""" - xi, cl = {}, {} - for i, dp in enumerate(s.data): - if dp.data_type in (sacc_io.XI_PLUS, sacc_io.XI_MINUS): - xi.setdefault(tuple(dp.tracers), []).append(i) - elif dp.data_type == sacc_io.CL_EE: - key = (tuple(dp.tracers), id(dp.tags.get("window"))) - cl.setdefault(key, []).append(i) - blocks = [] - for pair, rows in xi.items(): - theta = np.array([s.data[i].tags["theta"] for i in rows], float) - grid, at = np.unique(theta, return_inverse=True) - plus = np.array([s.data[i].data_type == sacc_io.XI_PLUS for i in rows]) - nzs = [_nz(s, t) for t in pair] - - def theory(p, c, nzs=nzs, grid=grid, at=at, plus=plus): - xip, xim = xi_ccl(p, c, *nzs, grid) - return np.where(plus, np.asarray(xip)[at], np.asarray(xim)[at]) - - blocks.append((np.array(rows), theory)) - for (pair, _), rows in cl.items(): - window = s.get_bandpower_windows(rows) - nzs = [_nz(s, t) for t in pair] - - def theory(p, c, nzs=nzs, w=window): - return np.asarray(w.weight).T @ cl_ee(p, c, *nzs, np.asarray(w.values)) - - blocks.append((np.array(rows), theory)) - return blocks - - -def _at_hidden(blocks, fiducial, blind): - """Each block's theory at the hidden point; a failure names only its type.""" + Warnings and the theory's own prints are silenced, and the error is raised + outside the ``except``, so no context or frame carries the hidden point. + """ def evaluate(): - with warnings.catch_warnings(): + quiet = io.StringIO() + with warnings.catch_warnings(), redirect_stdout(quiet), redirect_stderr(quiet): warnings.simplefilter("ignore") - point = dataclasses.replace(fiducial, **_hidden(blind)) - return [theory(point.ccl_params(), fiducial) for _, theory in blocks] + return _theory.predict(s, _hidden(blind), rows, theory) try: return evaluate() @@ -163,36 +131,41 @@ def evaluate(): ) -def conceal(s, blind): - """A copy of ``s`` with every ξ± and Cℓ_EE row shifted by the blind. +def conceal(s, blind, theory=None): + """A copy of ``s`` with its shiftable rows moved by the blind's shift. - The shift is t(hidden) − t(fiducial), evaluated at the fiducial first; a - block the blind leaves unmoved, or moves to a non-finite value, is refused. + The shift is t(hidden) − t(fiducial), from ``theory`` (default + :data:`sp_validation.theory.THEORY`), evaluated at the fiducial first. A + shiftable type without a theory, and a data type and tracer pair the blind + leaves unmoved or moves to a non-finite value, are refused. """ - record = json.loads(blind.path.read_text()) - fiducial = TheoryConfig(**record["fiducial"]) - blocks = _blocks(s) - at_fiducial = [theory(fiducial.ccl_params(), fiducial) for _, theory in blocks] - out = s.copy() - for (rows, _), t_fid, t_hid in zip( - blocks, at_fiducial, _at_hidden(blocks, fiducial, blind) - ): - delta = np.asarray(t_hid, float) - np.asarray(t_fid, float) - if ( - delta.shape != rows.shape - or not np.all(np.isfinite(delta)) - or not delta.any() - ): + theory = _theory.THEORY if theory is None else theory + rows = sacc_io.shiftable(s) + types = {s.data[i].data_type for i in rows} + if types - set(theory): + raise BlindingError( + f"no theory for the shiftable {sorted(types - set(theory))}; " + "give it a function in sp_validation.theory.THEORY" + ) + fiducial = json.loads(blind.path.read_text())["fiducial"] + at_fiducial = _theory.predict(s, fiducial, rows, theory) + delta = _at_hidden(s, rows, theory, blind) - at_fiducial + groups = {} + for n, i in enumerate(rows): + groups.setdefault((s.data[i].data_type, s.data[i].tracers), []).append(n) + for (data_type, pair), at in groups.items(): + if not np.all(np.isfinite(delta[at])) or not delta[at].any(): raise BlindingError( - f"blind {blind.name} leaves {len(rows)} shiftable rows unmoved or " - "non-finite; the theory must depend on the cosmology" + f"blind {blind.name} leaves {data_type} of {pair} unmoved or " + "non-finite; its theory must depend on the cosmology" ) - for row, d in zip(rows, delta): - out.data[int(row)].value += float(d) + out = s.copy() + for row, d in zip(rows, delta): + out.data[int(row)].value += float(d) return out -def init(name, catalogues, *, fiducial=None): +def init(name, catalogues): """Draw blind ``name`` into the catalogue config's registry, once.""" if not _custody.BLIND_NAME.fullmatch(name) or name in ( _custody.NONE, @@ -205,7 +178,7 @@ def init(name, catalogues, *, fiducial=None): "_": SECRET, "seed": secrets.token_hex(32), "envelope": ENVELOPE, - "fiducial": fiducial or dataclasses.asdict(TheoryConfig()), + "fiducial": _theory.fiducial(), "draw_scheme": draw_scheme(), } path = where / f"{name}.blind.json" diff --git a/src/sp_validation/blinding_theory.py b/src/sp_validation/blinding_theory.py deleted file mode 100644 index 629bd01b..00000000 --- a/src/sp_validation/blinding_theory.py +++ /dev/null @@ -1,260 +0,0 @@ -"""Blinding theory: the fiducial configuration and CCL's shear two-point prediction. - -:Name: blinding_theory.py - -:Description: :class:`TheoryConfig` is the fiducial cosmology and model - recipe; :func:`xi_ccl` and :func:`cl_ee` are the tomographic shear ξ± and - Cℓ_EE between two bins' n(z). CCL builds the nonlinear P(k) through its - Boltzmann-CAMB HMCode2020 route and projects with its own Limber - (``angular_cl``) and FFTLog (``correlation``). - - The generic cosmology machinery here is destined for ``cs_util.cosmo`` - (cs_util#80). - - Only ``numpy`` is imported at module level; CCL is imported inside the - functions that need it, so importing :class:`TheoryConfig` never drags in a - theory backend. -""" - -import dataclasses - -import numpy as np - -# Fixed constants of the fiducial, passed explicitly to CCL rather than left to -# its default. -NEFF = 3.046 -T_CMB = 2.7255 - - -# --------------------------------------------------------------------------- # -# Configuration surface — the ONE place fiducial cosmology + model choices live -# --------------------------------------------------------------------------- # -@dataclasses.dataclass(frozen=True) -class TheoryConfig: - """Fiducial cosmology and model configuration for the theory paths. - - Every field is a deliberate, configurable choice. The defaults mirror the - ``cosmo_inference`` CosmoSIS fiducial (the ``SP_v1.4.6.3_A_cell`` pipeline - + ``values_ia.ini`` central values), so the CCL theory computed here and - the CAMB theory CosmoSIS computes agree. Adopting a different named group - fiducial is a change to these *values*, not to any code. - - Cosmology is parametrised by the blind axes ``S8`` and ``Omega_m`` and - converted to CCL's native ``sigma8``/``Omega_c`` by :meth:`sigma8` / - :meth:`omega_c`. - - ``halofit_version`` names the nonlinear recipe CAMB runs, whether CCL - calls it or CosmoSIS does. - """ - - # Cosmological parameters (blind axes S8, Omega_m + the rest). - S8: float = 0.80 # values_ia.ini S_8_input central - Omega_m: float = 0.30 - Omega_b: float = 0.0469 # ombh2=0.023 at h=0.7 -> 0.023/0.7^2 - h: float = 0.70 - n_s: float = 0.96 - m_nu: float = 0.06 # Σm_ν in eV, distributed under `mass_split` - w0: float = -1.0 - wa: float = 0.0 - - # Neutrino mass split: normal hierarchy (CosmoSIS `neutrino_hierarchy=normal`). - mass_split: str = "normal" - - # Boltzmann backend for the CCL path (#280). `boltzmann_camb` shares one - # power-spectrum path with the CosmoSIS+CAMB inference stack; any other - # backend falls back to CCL's own halofit (see `ccl_cosmology`), a - # deliberate cross-check tool rather than a production setting. - transfer_function: str = "boltzmann_camb" - - # CAMB HMCode2020 + baryonic feedback. - halofit_version: str = "mead2020_feedback" - hmcode_logT_AGN: float = 7.5 # values_ia.ini logT_AGN central - - # Intrinsic alignments: NLA. The fiducial defaults IA OFF (ia_bias=0) — - # the blinding shift is a difference of two theory vectors at the same IA, - # so IA nearly cancels there, and IA-off keeps the CAMB↔CCL cross-check a - # clean test of the shear calculation. Set `ia_bias` nonzero (CosmoSIS - # central A=1.0) to include NLA. - ia_bias: float = 0.0 - ia_z_piv: float = 0.62 - ia_alphaz: float = 0.0 - - def sigma8(self): - """CCL ``sigma8`` implied by ``S8`` and ``Omega_m``. - - ``S8 ≡ σ8 √(Ωm / 0.3)`` — the standard weak-lensing definition — so - ``σ8 = S8 / √(Ωm / 0.3)``. At the fiducial (S8=0.80, Ωm=0.30), - σ8 = 0.80. - """ - return self.S8 / np.sqrt(self.Omega_m / 0.3) - - def omega_c(self): - """CCL cold-dark-matter density ``Omega_c = Omega_m − Omega_b − Ω_ν``. - - ``Ω_ν`` is CCL's own massive-neutrino density for this ``m_nu``, - ``mass_split``, ``h``, ``Neff`` and ``T_CMB`` (independent of - ``Omega_c``), so CCL's total ``Omega_m`` is exactly ``Omega_m``. - """ - import pyccl as ccl - - trial = ccl.Cosmology(**self._ccl_params(self.Omega_m - self.Omega_b)) - return self.Omega_m - self.Omega_b - trial["Omega_nu_mass"] - - def ccl_params(self): - """This point as a plain CCL-native parameter mapping. - - Exactly the keys ``Omega_c, Omega_b, h, n_s, sigma8, m_nu, - mass_split, w0, wa, Neff, T_CMB`` and no others, so no CCL default - rides along; ``Neff``/``T_CMB`` are the fixed module constants. - """ - return self._ccl_params(self.omega_c()) - - def _ccl_params(self, omega_c): - return { - "Omega_c": omega_c, - "Omega_b": self.Omega_b, - "h": self.h, - "n_s": self.n_s, - "sigma8": self.sigma8(), - "m_nu": self.m_nu, - "mass_split": self.mass_split, - "w0": self.w0, - "wa": self.wa, - "Neff": NEFF, - "T_CMB": T_CMB, - } - - -# --------------------------------------------------------------------------- # -# CCL: cosmology construction, Cℓ_EE, ξ± -# --------------------------------------------------------------------------- # -# The two cosmologies of a blind (fiducial and hidden) are evaluated for every -# block of a SACC; caching the ccl.Cosmology per parameter point avoids -# re-running the CAMB P(k) computation for each block. -_COSMO_CACHE = {} - - -def ccl_cosmology(params, config): - """A ``pyccl.Cosmology`` at ``params`` with ``config``'s nonlinear recipe. - - ``params`` is a plain CCL-native mapping (:meth:`TheoryConfig.ccl_params`, - possibly with keys overlaid by the hidden draw); ``config`` supplies the - non-sampled recipe tokens. Under ``boltzmann_camb`` the nonlinear P(k) - runs through CAMB's HMCode2020; any other backend has no CAMB run to hand - tokens to and takes CCL's own halofit. Cached per parameter point: CCL - memoises P(k) on the object, so the cache saves repeated Boltzmann runs. - """ - import pyccl as ccl - - key = ( - tuple(sorted(params.items())), - config.transfer_function, - config.halofit_version, - config.hmcode_logT_AGN, - ) - if key not in _COSMO_CACHE: - nonlinear = ( - { - "matter_power_spectrum": "camb", - "extra_parameters": { - "camb": { - "halofit_version": config.halofit_version, - "HMCode_logT_AGN": config.hmcode_logT_AGN, - } - }, - } - if config.transfer_function == "boltzmann_camb" - else {"matter_power_spectrum": "halofit"} - ) - _COSMO_CACHE[key] = ccl.Cosmology( - **params, - transfer_function=config.transfer_function, - **nonlinear, - ) - return _COSMO_CACHE[key] - - -# The ℓ grid the ξ± Hankel projection integrates over: integers 2…49, then 200 -# log-spaced multipoles up to 6·10⁴ — dense enough at low ℓ (where ξ± at large θ -# lives) and wide enough for the small-θ tail. ``ccl.correlation`` interpolates -# C(ℓ) internally, so this fixes the resolution of every ξ± this module produces. -XI_ELL = np.unique( - np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)]).astype(float) -) - - -# Tracers are rebuilt for every pair of every block at both cosmologies of a -# blind, and each build runs CCL's lensing-kernel integral over the bin's n(z). -# Cached for the same reason as `_COSMO_CACHE`, and keyed on `id(cosmo)` -# because that cache pins every cosmology for the process lifetime, so an id -# can never be recycled onto a different object. -_TRACER_CACHE = {} - - -def _tracer(cosmo, z, nz, config): - """A ``WeakLensingTracer`` for one bin's n(z), NLA from ``config``. - - With the fiducial ``ia_bias = 0`` the tracer is built bare — no IA term. - A nonzero ``ia_bias`` enters as the NLA amplitude - ``A(z) = ia_bias · ((1+z)/(1+z_piv))^alphaz``. - """ - z = np.asarray(z) - nz = np.asarray(nz) - key = ( - id(cosmo), - z.tobytes(), - nz.tobytes(), - config.ia_bias, - config.ia_z_piv, - config.ia_alphaz, - ) - if key not in _TRACER_CACHE: - _TRACER_CACHE[key] = _build_tracer(cosmo, z, nz, config) - return _TRACER_CACHE[key] - - -def _build_tracer(cosmo, z, nz, config): - import pyccl as ccl - - if config.ia_bias == 0.0: - return ccl.WeakLensingTracer(cosmo, dndz=(z, nz)) - a_ia = config.ia_bias * ((1 + z) / (1 + config.ia_z_piv)) ** config.ia_alphaz - return ccl.WeakLensingTracer(cosmo, dndz=(z, nz), ia_bias=(z, a_ia), use_A_ia=True) - - -def cl_ee(params, config, nz_i, nz_j, ell): - """Cross Cℓ_EE at ``ell`` for the bin pair with n(z) ``nz_i``, ``nz_j``. - - Two-tracer: one :class:`~pyccl.WeakLensingTracer` per bin from that bin's - own ``(z, nz)``, then ``angular_cl(cosmo, tracer_i, tracer_j, ell)`` — - the cross-spectrum for i ≠ j, the auto-spectrum when the two n(z) are the - same bin. The shear ``angular_cl`` is the E-mode spectrum; B and EB are - zero in theory, which is why only Cℓ_EE ever receives a blinding shift. - """ - import pyccl as ccl - - cosmo = ccl_cosmology(params, config) - tracer_i = _tracer(cosmo, *nz_i, config) - tracer_j = _tracer(cosmo, *nz_j, config) - return ccl.angular_cl(cosmo, tracer_i, tracer_j, np.asarray(ell, dtype=float)) - - -def xi_ccl(params, config, nz_i, nz_j, theta_arcmin): - """ξ± at ``theta_arcmin`` for one bin pair. - - Cross Cℓ_EE on :data:`XI_ELL`, then ``ccl.correlation`` (FFTLog Hankel - transform) at θ in degrees, ``type="GG+"`` / ``"GG-"``. - - Returns - ------- - (np.ndarray, np.ndarray) - ``(xip, xim)`` aligned to ``theta_arcmin``. - """ - import pyccl as ccl - - cosmo = ccl_cosmology(params, config) - cl = cl_ee(params, config, nz_i, nz_j, XI_ELL) - theta_deg = np.asarray(theta_arcmin) / 60.0 - xip = ccl.correlation(cosmo, ell=XI_ELL, C_ell=cl, theta=theta_deg, type="GG+") - xim = ccl.correlation(cosmo, ell=XI_ELL, C_ell=cl, theta=theta_deg, type="GG-") - return xip, xim diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index cb3f0f72..9783a4fc 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -107,6 +107,8 @@ CL_EB = "galaxy_shear_cl_eb" COSEBI_EE = "galaxy_shear_cosebi_ee" COSEBI_BB = "galaxy_shear_cosebi_bb" +GAMMA_T = "galaxy_shearDensity_xi_t" +GAMMA_X = "galaxy_shearDensity_xi_x" # Custom data-type strings (all parse under sacc.parse_data_type_name). PURE_TYPES = { @@ -160,7 +162,13 @@ def new_sacc(nz, metadata=None): items = nz.items() if isinstance(nz, dict) else enumerate(nz) s = sacc.Sacc() for i, (z, nz_i) in items: - s.add_tracer("NZ", source_name(i), np.asarray(z), np.asarray(nz_i)) + s.add_tracer( + "NZ", + source_name(i), + np.asarray(z), + np.asarray(nz_i), + quantity="galaxy_shear", + ) # The PSF star sample sits alongside the source bins because a Sacc has a # single tracer namespace — every data point references tracers from one # flat list. This is bookkeeping, not physics: psf_stars is a Misc tracer @@ -417,6 +425,24 @@ def add_pure_eb( _add_theta_series(s, PURE_TYPES[key], tracers, theta, arr, grid=grid) +def add_lens(s, name, nz=None, *, quantity="galaxy_density"): + """Add a γt lens sample: galaxies with their ``(z, n(z))``, or, as a + tracer without one, ``quantity="stars"`` or ``"randoms"``.""" + if nz is None: + s.add_tracer("Misc", name, quantity=quantity) + else: + s.add_tracer("NZ", name, *map(np.asarray, nz), quantity=quantity) + + +def add_gamma_t(s, source_bin, lens, theta, gamma_t, gamma_x=None, **tags): + """Add γt (and optionally γ×) of source bin ``source_bin`` around ``lens``.""" + _check_ascending("theta", theta) + tracers = (source_name(source_bin), lens) + _add_theta_series(s, GAMMA_T, tracers, theta, gamma_t, **tags) + if gamma_x is not None: + _add_theta_series(s, GAMMA_X, tracers, theta, gamma_x, **tags) + + def add_rho(s, k, theta, rho_p, rho_m, *, grid="reporting"): """Add a ρ_k PSF statistic (ρ+ then ρ−) on the ``psf_stars`` tracer. @@ -978,10 +1004,10 @@ def _check_grid_consistency(s, angle): # --------------------------------------------------------------------------- # # The blinding rule of each data type a blinded catalogue's SACC may hold. A # data type with none is refused under a blind, so a new statistic fails closed. -# Signal the blind shifts: -SHIFTABLE = (XI_PLUS, XI_MINUS, CL_EE) +# Signal the blind shifts, each by its theory in sp_validation.theory.THEORY: +SHIFTABLE = (XI_PLUS, XI_MINUS, CL_EE, GAMMA_T) # signal a pure E-mode shift leaves unchanged: -UNSHIFTED = (CL_BB, CL_EB) +UNSHIFTED = (CL_BB, CL_EB, GAMMA_X) # signal derived from ξ±, which moves only through the rows it is computed from: DERIVED = (COSEBI_EE, COSEBI_BB, *PURE_TYPES.values()) SIGNAL = SHIFTABLE + UNSHIFTED + DERIVED @@ -989,6 +1015,21 @@ def _check_grid_consistency(s, angle): _SIGNAL_FREE = re.compile( "|".join(t.format(k=r"\d+") for t in (RHO_PLUS, RHO_MINUS, TAU_PLUS, TAU_MINUS)) ) +# γt around a lens tracer of one of these quantities is a null test, not signal. +NULL_LENSES = ("stars", "randoms") + + +def shiftable(s): + """Indices of the rows of ``s`` a blind shifts.""" + return np.array( + [ + i + for i, dp in enumerate(s.data) + if dp.data_type in SHIFTABLE + and not any(s.tracers[t].quantity in NULL_LENSES for t in dp.tracers) + ], + dtype=int, + ) def _refuse_unruled(s): @@ -1001,8 +1042,9 @@ def _refuse_unruled(s): if unruled: raise ValueError( f"{unruled}: no blinding rule for these data types, so a blinded " - "catalogue's SACC cannot hold them; give a new statistic its rule in " - "sacc_io (and its shift in blinding) first" + "catalogue's SACC cannot hold them. Signal is SHIFTABLE in sacc_io, " + "with a function in sp_validation.theory.THEORY; UNSHIFTED is only for " + "signal that a pure E-mode shift leaves unchanged" ) @@ -1021,11 +1063,11 @@ def seal(s, custody): @sc born-sealed A catalogue-born SACC leaves memory only through here. Under a blinded - custody every ξ± and Cℓ_EE row is shifted on a copy before the stamp is - minted; a SACC with no signal (ρ/τ) is stamped without opening the blind; a - derived statistic (COSEBIs, pure-E/B) is refused here and saved with - ``derived_from``; a data type with no blinding rule is refused. An - already-stamped SACC is re-written only as a derivation. + custody every :func:`shiftable` row (ξ±, Cℓ_EE, γt) is shifted on a copy + before the stamp is minted; a SACC with none is stamped without opening + the blind; a derived statistic (COSEBIs, pure-E/B) is refused here and + saved with ``derived_from``; a data type with no blinding rule is refused. + An already-stamped SACC is re-written only as a derivation. """ if _stamped(s): raise ValueError( @@ -1041,7 +1083,7 @@ def seal(s, custody): f"a blinded catalogue's {sorted(derived)} rows are derived " "statistics: save them with derived_from=[their input parts]" ) - if custody.blinded and types & set(SHIFTABLE): + if custody.blinded and len(shiftable(s)): from . import blinding out = blinding.conceal(s, blinding.open_blind(custody)) @@ -1074,15 +1116,11 @@ def _derive(s, parts, custody): ) if stamp.blinded: _refuse_unruled(s) - inputs = {_row_key(dp) for p in parts for dp in p.data if dp.data_type in SHIFTABLE} - stray = [ - i - for i, dp in enumerate(s.data) - if dp.data_type in SHIFTABLE and _row_key(dp) not in inputs - ] + inputs = {_row_key(p.data[i]) for p in parts for i in shiftable(p)} + stray = [i for i in shiftable(s) if _row_key(s.data[i]) not in inputs] if stray: raise ValueError( - f"{len(stray)} ξ±/Cℓ_EE rows of a derivation are not copies of its " + f"{len(stray)} shiftable rows of a derivation are not copies of its " "inputs' rows; a catalogue's shiftable signal is born only through " "save(custody=)" ) @@ -1105,7 +1143,7 @@ def save(s, path, *, custody=None, derived_from=None): derivation whose stamp must also be ``c``'s. @sc derived-inherit - A derivation's inputs must share one stamp, and each of its ξ±/Cℓ_EE rows + A derivation's inputs must share one stamp, and each of its shiftable rows must be a copy of an input row (type, tracers, value, tags; windows by index), so plaintext cannot be saved under a concealed stamp; under a blinded stamp every data type needs a blinding rule, as at birth. diff --git a/src/sp_validation/tests/_synthetic.py b/src/sp_validation/tests/_synthetic.py index 5aa8c388..e1d1786e 100644 --- a/src/sp_validation/tests/_synthetic.py +++ b/src/sp_validation/tests/_synthetic.py @@ -5,13 +5,28 @@ seams read, a cs_util-readable dndz, and a ``cat_config.yaml`` whose blind registry is ``blinds/`` beside it. Every catalogue entry in the config reads its own copy of the same galaxies and declares its own custody. + +``TOY_THEORY`` stands in for :data:`sp_validation.theory.THEORY`, so a blind +shifts without running CAMB. """ import copy +from types import MappingProxyType import numpy as np import yaml +from sp_validation import sacc_io + + +def toy_theory(params, s, rows): + """A power law in θ or ℓ whose amplitude grows with S8 and Ωm.""" + x = np.array([s.data[i].tags.get("theta", s.data[i].tags.get("ell")) for i in rows]) + return 1e-4 * params["S8"] ** 2 * params["Omega_m"] ** 0.3 * (x / 10.0) ** -0.8 + + +TOY_THEORY = MappingProxyType({t: toy_theory for t in sacc_io.SHIFTABLE}) + def write_synthetic_catalogs( tmp_path, diff --git a/src/sp_validation/tests/conftest.py b/src/sp_validation/tests/conftest.py index 40424120..781ebb93 100644 --- a/src/sp_validation/tests/conftest.py +++ b/src/sp_validation/tests/conftest.py @@ -17,3 +17,13 @@ def pure_eb_xi(): """ with np.load(PURE_EB_XI) as npz: return {k: (v.item() if v.ndim == 0 else v) for k, v in npz.items()} + + +@pytest.fixture +def toy_theory(monkeypatch): + """Blinds shift by the analytic ``_synthetic.TOY_THEORY``.""" + from _synthetic import TOY_THEORY + + from sp_validation import theory + + monkeypatch.setattr(theory, "THEORY", TOY_THEORY) diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 32b20810..27f1f8bf 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -1,9 +1,9 @@ """The blind and the file door. A blind is drawn once (``blinding init``) into a registry outside git; -``sacc_io.save`` is the only writer, and conceals a blinded catalogue's ξ± and -Cℓ_EE rows in memory before the file exists. Blinds here use the fast -Eisenstein–Hu theory. +``sacc_io.save`` is the only writer, and conceals a blinded catalogue's +shiftable rows in memory before the file exists. Blinds here shift by the +analytic ``TOY_THEORY``; one slow test runs the CCL defaults. """ import dataclasses @@ -13,16 +13,24 @@ import numpy as np import pytest +from _synthetic import TOY_THEORY from hypothesis import example, given, settings from hypothesis import strategies as st from sp_validation import blinding as bd from sp_validation import custody as cu from sp_validation import sacc_io as sio -from sp_validation.blinding_theory import TheoryConfig +from sp_validation import theory VERSIONS = ("TOY", "OTHER", "TOY_OPEN", "OTHER_OPEN", "TOY_MOCK") -FAST = dataclasses.asdict(TheoryConfig(transfer_function="eisenstein_hu")) +DEFAULTS = theory.THEORY + + +@pytest.fixture(scope="module", autouse=True) +def _toy_theory(): + with pytest.MonkeyPatch.context() as m: + m.setattr(theory, "THEORY", TOY_THEORY) + yield def _catalogues(root): @@ -45,7 +53,7 @@ def blinds(tmp_path_factory): """The five custodies: TOY under blind `toy`, OTHER under `other`.""" cats = _catalogues(tmp_path_factory.mktemp("registry")) for name in ("toy", "other"): - bd.init(name, cats, fiducial=FAST) + bd.init(name, cats) return {v: cu.custody_of(cats, v) for v in VERSIONS} @@ -54,10 +62,19 @@ def _nz(z0): return z, np.exp(-0.5 * ((z - z0) / 0.2) ** 2) -def part(*, xi_tags=("reporting",), cl=True, rho=False, derived=None, unruled=False): +def part( + *, + xi_tags=("reporting",), + cl=True, + rho=False, + gt=(), + derived=None, + unruled=False, +): """A two-bin part: ξ± under each of ``xi_tags`` (None: no tag) and pseudo-Cℓ (EE, BB, EB) on pairs (0,0), (0,1), (1,1), and optionally ρ/τ, - a derived statistic's rows and rows of a type with no blinding rule.""" + γt and γ× of bin 1 around each lens quantity in ``gt``, a derived + statistic's rows and rows of a type with no blinding rule.""" s = sio.new_sacc({0: _nz(0.5), 1: _nz(0.9)}) theta = np.geomspace(2.0, 200.0, 6) ell = np.array([30.0, 80.0, 150.0, 280.0, 450.0]) @@ -86,6 +103,10 @@ def part(*, xi_tags=("reporting",), cl=True, rho=False, derived=None, unruled=Fa ) if rho: sio.add_rho(s, 0, theta, np.arange(1, 7) * 1e-7, np.arange(1, 7) * 2e-7) + for quantity in gt: + nz = _nz(0.3) if quantity == "galaxy_density" else None + sio.add_lens(s, quantity, nz, quantity=quantity) + sio.add_gamma_t(s, 1, quantity, theta, 1e-4 * (theta / 10) ** -0.7, 0 * theta) if derived == "cosebis": sio.add_cosebis(s, (0, 0), np.arange(1, 6) * 1e-10, (12.0, 83.0), Bn=np.ones(5)) elif derived == "pure_eb": @@ -93,7 +114,7 @@ def part(*, xi_tags=("reporting",), cl=True, rho=False, derived=None, unruled=Fa if unruled: for x in (5.0, 20.0, 80.0): s.add_data_point( - "galaxy_shearDensity_xi_t", ("source_0", "source_0"), 1e-5, theta=x + "galaxy_density_xi", ("source_0", "source_0"), 1.0, theta=x ) s.add_covariance(np.abs(np.asarray(s.mean)) ** 2 + 1e-20) return s @@ -115,6 +136,9 @@ def _values(s): ), "cl": st.booleans(), "rho": st.booleans(), + "gt": st.lists( + st.sampled_from(["galaxy_density", "stars", "randoms"]), unique=True + ), "derived": st.sampled_from([None, "cosebis", "pure_eb"]), "unruled": st.booleans(), } @@ -125,13 +149,19 @@ def _values(s): @given(content=PARTS, version=st.sampled_from(["TOY", "TOY_OPEN", "TOY_MOCK"])) @example( content=dict( - xi_tags=["mystery", None], cl=True, rho=True, derived=None, unruled=False + xi_tags=["mystery", None], + cl=True, + rho=True, + gt=["galaxy_density", "stars"], + derived=None, + unruled=False, ), version="TOY", ) def test_seal_shifts_only_the_signal_it_has_a_rule_for(blinds, content, version): - """Under a blind every ξ± and Cℓ_EE row moves, whatever its grid tag, and - every other value and the covariance stay bitwise; a birth carrying + """Under a blind every ξ±, Cℓ_EE and galaxy-lens γt row moves, whatever + its grid tag, and every other value (γ×, γt around stars or randoms) and + the covariance stay bitwise; a birth carrying derived rows or a type with no blinding rule is refused. Unblinded and mock births keep their values. Each is stamped with its custody.""" s, custody = part(**content), blinds[version] @@ -142,7 +172,12 @@ def test_seal_shifts_only_the_signal_it_has_a_rule_for(blinds, content, version) return sealed = sio.seal(s, custody) assert cu.read_stamp(sealed.metadata).stamp == custody.stamp - shiftable = np.array([dp.data_type in sio.SHIFTABLE for dp in s.data]) + shiftable = np.array( + [ + dp.data_type in sio.SHIFTABLE and not {"stars", "randoms"} & {*dp.tracers} + for dp in s.data + ] + ) moved = _values(sealed) != _values(s) assert np.array_equal(moved, shiftable & blinded) assert np.array_equal(sealed.covariance.dense, s.covariance.dense) @@ -218,11 +253,11 @@ def test_a_derivation_carries_its_inputs_one_stamp( # --------------------------------------------------------------------------- # def test_a_blind_is_drawn_once_and_kept_private(tmp_path): cats = _catalogues(tmp_path) - blind = bd.init("toy", cats, fiducial=FAST) + blind = bd.init("toy", cats) assert stat.S_IMODE(blind.path.stat().st_mode) == 0o440 assert stat.S_IMODE(blind.path.parent.stat().st_mode) == 0o700 with pytest.raises(cu.CustodyError, match="drawn once"): - bd.init("toy", cats, fiducial=FAST) + bd.init("toy", cats) def test_a_blind_opens_only_under_its_commitment(blinds, monkeypatch): @@ -237,17 +272,17 @@ def test_a_blind_opens_only_under_its_commitment(blinds, monkeypatch): bd.open_blind(custody) -def _hidden_failure(blind, fiducial): +def _hidden_failure(blind): """The traceback, locals included, of a theory failing at the hidden point - with its parameters in its message.""" + with its parameters in its message and on stdout.""" def failing(params, *args): - raise RuntimeError(f"cannot evaluate {params}") + print(dict(params)) + raise RuntimeError(f"cannot evaluate {dict(params)}") - with pytest.MonkeyPatch.context() as m: - m.setattr(bd, "xi_ccl", failing) - with pytest.raises(bd.BlindingError) as failure: - bd._at_hidden(bd._blocks(part(cl=False)), fiducial, blind) + s = part(cl=False) + with pytest.raises(bd.BlindingError) as failure: + bd._at_hidden(s, sio.shiftable(s), dict.fromkeys(sio.SHIFTABLE, failing), blind) trace = traceback.TracebackException.from_exception( failure.value, capture_locals=True ) @@ -258,7 +293,7 @@ def test_the_hidden_cosmology_never_shows(tmp_path, capfd): """Neither the seed nor the hidden S8, Ωm or σ8 reaches a repr, the terminal or a traceback's locals.""" cats = _catalogues(tmp_path) - blind = bd.init("toy", cats, fiducial=FAST) + blind = bd.init("toy", cats) record = json.loads(blind.path.read_text()) hidden = bd._hidden(blind) sigma8 = hidden["S8"] / np.sqrt(hidden["Omega_m"] / 0.3) @@ -269,14 +304,55 @@ def test_the_hidden_cosmology_never_shows(tmp_path, capfd): ] bd.show("toy", cats) sio.seal(part(cl=False), cu.custody_of(cats, "TOY")) + failure = _hidden_failure(blind) haystacks = [ repr(blind), repr(hidden), str(hidden), f"{hidden}", "".join(capfd.readouterr()), - _hidden_failure(blind, TheoryConfig(**record["fiducial"])), + failure, ] for i, haystack in enumerate(haystacks): # The message names the haystack only: a failure must not print a needle. assert not any(n in haystack for n in needles), f"haystack {i} shows it" + + +# --------------------------------------------------------------------------- # +# The shift: from each data type's theory, refused where it cannot be made +# --------------------------------------------------------------------------- # +def test_conceal_refuses_a_shift_it_cannot_make(blinds): + """A shiftable type without a theory, and a theory that ignores the + cosmology, are refused rather than stamped as blinded.""" + blind = bd.open_blind(blinds["TOY"]) + s = part(cl=False, gt=["galaxy_density"]) + no_gt = {t: f for t, f in TOY_THEORY.items() if t != sio.GAMMA_T} + with pytest.raises(bd.BlindingError, match="no theory"): + bd.conceal(s, blind, theory=no_gt) + flat = {**TOY_THEORY, sio.GAMMA_T: lambda params, s, rows: np.ones(len(rows))} + with pytest.raises(bd.BlindingError, match="unmoved"): + bd.conceal(s, blind, theory=flat) + + +@pytest.mark.slow +def test_the_ccl_defaults_shift_with_s8(blinds, monkeypatch): + """The default theories, at a hidden point with S8 above the fiducial, + raise ξ+, Cℓ_EE and γt, and γt's shift does not depend on tracer order.""" + fiducial = theory.fiducial() + monkeypatch.setattr(bd, "_hidden", lambda blind: {**fiducial, "S8": 0.9}) + blind = bd.open_blind(blinds["TOY"]) + s = part(gt=["galaxy_density"]) + delta = bd.conceal(s, blind, theory=DEFAULTS).mean - s.mean + for data_type in (sio.XI_PLUS, sio.CL_EE, sio.GAMMA_T): + assert np.all(delta[s.indices(data_type)] > 0), data_type + + swapped = s.copy() + for dp in swapped.data: + if dp.data_type == sio.GAMMA_T: + dp.tracers = dp.tracers[::-1] + rows = s.indices(sio.GAMMA_T) + np.testing.assert_allclose( + (bd.conceal(swapped, blind, theory=DEFAULTS).mean - s.mean)[rows], + delta[rows], + rtol=1e-10, + ) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 057c22eb..c5467fd4 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -498,14 +498,11 @@ def block_variances(c): # I14: a blinded catalogue's ξ± leaves CosmologyValidation only concealed # --------------------------------------------------------------------------- # @pytest.fixture -def blinded_and_twin(tmp_path): +def blinded_and_twin(tmp_path, toy_theory): """TOY, blinded under `toy`, and TOY_OPEN: the same galaxies, public.""" - import dataclasses - import yaml from sp_validation import blinding - from sp_validation.blinding_theory import TheoryConfig params, _ = write_synthetic_catalogs( tmp_path, @@ -513,9 +510,8 @@ def blinded_and_twin(tmp_path): coherent_shear=True, catalogues={"TOY": "toy", "TOY_OPEN": "none"}, ) - fast = TheoryConfig(transfer_function="eisenstein_hu") catalogues = yaml.safe_load(open(params["catalog_config"])) - blinding.init("toy", catalogues, fiducial=dataclasses.asdict(fast)) + blinding.init("toy", catalogues) grid = dict(npatch=1, theta_min=5.0, theta_max=60.0, nbins=6, **params) return ( CosmologyValidation(versions=["TOY"], **grid), diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py index 7ef2ecfa..04ae3458 100644 --- a/src/sp_validation/tests/test_custody_e2e.py +++ b/src/sp_validation/tests/test_custody_e2e.py @@ -8,7 +8,6 @@ mean moved. """ -import dataclasses import re import runpy import types @@ -21,7 +20,6 @@ from sp_validation import blinding as bd from sp_validation import custody as cu from sp_validation import sacc_io as sio -from sp_validation.blinding_theory import TheoryConfig SCRIPTS = Path(__file__).resolve().parents[3] / "workflow" / "scripts" GRID = {"min_sep": 5.0, "max_sep": 60.0, "nbins": 6, "npatch": 1} @@ -103,7 +101,9 @@ def near(x, rtol): return found -def test_a_blinded_catalogue_from_birth_to_unblinding(tmp_path, monkeypatch): +def test_a_blinded_catalogue_from_birth_to_unblinding( + tmp_path, monkeypatch, toy_theory +): monkeypatch.syspath_prepend(str(SCRIPTS)) import cv_runner @@ -114,8 +114,7 @@ def test_a_blinded_catalogue_from_birth_to_unblinding(tmp_path, monkeypatch): ) cat_config = Path(params["catalog_config"]) config = yaml.safe_load(cat_config.read_text()) - fast = TheoryConfig(transfer_function="eisenstein_hu") - blind = bd.init("toy", config, fiducial=dataclasses.asdict(fast)) + blind = bd.init("toy", config) cov = tmp_path / "cov.txt" np.savetxt(cov, np.diag(np.full(2 * GRID["nbins"], 1e-10))) diff --git a/src/sp_validation/theory.py b/src/sp_validation/theory.py new file mode 100644 index 00000000..38f5c15e --- /dev/null +++ b/src/sp_validation/theory.py @@ -0,0 +1,187 @@ +"""Theory data vectors: a prediction for the rows of a SACC, per data type. + +@sc theory-plugin +A theory is a function ``f(params, s, rows) -> values``: the prediction, at the +parameter point ``params``, for rows ``rows`` of SACC ``s``, which share a data +type's theory, a tracer pair and a bandpower window; values are aligned to +``rows``. ``params`` is a plain mapping with the keys of :func:`fiducial`, +never a CCL object, so an emulator can stand in for CCL. :data:`THEORY` maps +each data type to its default, and :func:`predict` takes any other mapping as +``theory=``. + +@sc theory-ccl-default +The defaults build one ``pyccl.Cosmology`` per point through +``cs_util.cosmo.get_cosmo``, on the CAMB HMCode2020-feedback route the CosmoSIS +inference runs, with σ8 = S8/√(Ωm/0.3). ``Omega_m`` is ``get_cosmo``'s +argument, from which it subtracts CAMB's Ω_ν for Ω_c. + +pyccl and cs_util are imported only when a default theory runs. +""" + +import functools +from types import MappingProxyType + +import numpy as np + +from .sacc_io import CL_EE, GAMMA_T, XI_MINUS, XI_PLUS + +# Multipoles the ξ± and γt Hankel transforms integrate over: every ℓ below 50, +# then 200 log-spaced up to 6·10⁴. +ELL = np.unique(np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)])) +_COSMOLOGY = ("S8", "Omega_m", "Omega_b", "h", "n_s", "m_nu", "w0", "wa", "logT_AGN") + + +def fiducial(): + """cs_util's Planck 2018 point, with feedback, no IA and unit lens bias. + + ``b_lens`` is the linear bias of every γt lens sample: the γt shift scales + as ``b_lens`` over the sample's true bias. + """ + from cs_util.cosmo import PLANCK18 as p + + return { + "S8": float(p["sigma_8"] * np.sqrt(p["Omega_m"] / 0.3)), + **{ + k: float(p[k]) + for k in ("Omega_m", "Omega_b", "h", "n_s", "m_nu", "w0", "wa") + }, + "logT_AGN": 7.5, + "A_IA": 0.0, + "b_lens": 1.0, + } + + +def cosmology(params): + """The ``pyccl.Cosmology`` at ``params``, built once per point.""" + return _cosmology(tuple(float(params[k]) for k in _COSMOLOGY)) + + +@functools.lru_cache(maxsize=4) +def _cosmology(point): + from cs_util.cosmo import get_cosmo + + p = dict(zip(_COSMOLOGY, point)) + return get_cosmo( + Omega_m=p["Omega_m"], + Omega_b=p["Omega_b"], + h=p["h"], + sig8=p["S8"] / np.sqrt(p["Omega_m"] / 0.3), + ns=p["n_s"], + w0=p["w0"], + wa=p["wa"], + mnu=p["m_nu"], + # get_cosmo's default "halofit" ignores extra_params. + matter_power_spectrum="camb", + extra_params={ + "camb": { + "halofit_version": "mead2020_feedback", + "HMCode_logT_AGN": p["logT_AGN"], + } + }, + ) + + +def tag(s, rows, name): + """Tag ``name`` (or ``"data_type"``) of each of ``rows``.""" + return np.array( + [ + s.data[i].data_type if name == "data_type" else s.data[i].tags[name] + for i in rows + ] + ) + + +def nz(s, name): + """The ``(z, n(z))`` of tracer ``name``.""" + tracer = s.tracers[name] + if getattr(tracer, "nz", None) is None: + raise ValueError(f"tracer {name} has no n(z)") + return np.asarray(tracer.z, float), np.asarray(tracer.nz, float) + + +def source_lens(s, rows): + """The source and lens tracers of γt-like rows, told apart by quantity.""" + names = s.data[rows[0]].tracers + by = {s.tracers[n].quantity: n for n in names} + if len(names) != 2 or set(by) != {"galaxy_shear", "galaxy_density"}: + raise ValueError( + f"tracers {names} are not one galaxy_shear and one galaxy_density tracer" + ) + return by["galaxy_shear"], by["galaxy_density"] + + +def _lensing(cosmo, params, s, name): + import pyccl as ccl + + z, n = nz(s, name) + ia = None if params["A_IA"] == 0 else (z, np.full_like(z, params["A_IA"])) + return ccl.WeakLensingTracer(cosmo, dndz=(z, n), ia_bias=ia) + + +def _shear_cl(params, s, rows, ell): + import pyccl as ccl + + cosmo = cosmology(params) + a, b = (_lensing(cosmo, params, s, t) for t in s.data[rows[0]].tracers) + return ccl.angular_cl(cosmo, a, b, ell) + + +def shear_xi(params, s, rows): + """ξ± of one shear pair: Limber C_ℓ on :data:`ELL`, then CCL's Hankel transform.""" + import pyccl as ccl + + cosmo, cl = cosmology(params), _shear_cl(params, s, rows, ELL) + theta = tag(s, rows, "theta") / 60.0 + xip, xim = ( + ccl.correlation(cosmo, ell=ELL, C_ell=cl, theta=theta, type=t) + for t in ("GG+", "GG-") + ) + return np.where(tag(s, rows, "data_type") == XI_PLUS, xip, xim) + + +def shear_cl(params, s, rows): + """Cℓ_EE of one shear pair through its bandpower window.""" + window = s.get_bandpower_windows(rows) + cl = _shear_cl(params, s, rows, np.asarray(window.values, float)) + return np.asarray(window.weight).T @ cl + + +def gamma_t(params, s, rows): + """γt of a source bin around a lens sample of linear bias ``b_lens``.""" + import pyccl as ccl + + cosmo = cosmology(params) + source, lens = source_lens(s, rows) + z, n = nz(s, lens) + bias = (z, np.full_like(z, params["b_lens"])) + counts = ccl.NumberCountsTracer(cosmo, has_rsd=False, dndz=(z, n), bias=bias) + cl = ccl.angular_cl(cosmo, counts, _lensing(cosmo, params, s, source), ELL) + theta = tag(s, rows, "theta") / 60.0 + return ccl.correlation(cosmo, ell=ELL, C_ell=cl, theta=theta, type="NG") + + +THEORY = MappingProxyType( + {XI_PLUS: shear_xi, XI_MINUS: shear_xi, CL_EE: shear_cl, GAMMA_T: gamma_t} +) + + +def predict(s, params, rows=None, theory=None): + """The theory of ``rows`` (default: all) of ``s`` at ``params``. + + Rows are grouped by theory function, tracer pair and bandpower window, one + call per group, so ξ+ and ξ− of a pair share their C_ℓ. ``theory`` maps + data type to function (default :data:`THEORY`); a row without one raises. + """ + theory = THEORY if theory is None else theory + rows = np.arange(len(s.data)) if rows is None else np.asarray(rows, int) + groups = {} + for n, i in enumerate(rows): + dp = s.data[i] + if dp.data_type not in theory: + raise ValueError(f"no theory for {dp.data_type}") + window = id(dp.tags.get("window")) + groups.setdefault((theory[dp.data_type], dp.tracers, window), []).append(n) + out = np.empty(len(rows)) + for (function, *_), at in groups.items(): + out[at] = function(params, s, rows[at]) + return out diff --git a/workflow/README.md b/workflow/README.md index b774e8ef..00523ab3 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -36,9 +36,12 @@ beside mocks and catalogues under the same blind, since overlaying it with any other version shows the shift. A launch prints one `[custody]` line per catalogue. -Under a blind, every ξ± and pseudo-Cℓ_EE value is shifted by t(hidden) − -t(fiducial) before it is first written; COSEBIs and pure-E/B computed from the -shifted ξ± carry the blind, and B-modes stay usable. Every SACC records the +Under a blind, every ξ±, pseudo-Cℓ_EE and γt value is shifted by t(hidden) − +t(fiducial) before it is first written, t being that data type's theory in +`sp_validation.theory.THEORY` (CCL through `cs_util.cosmo.get_cosmo` by +default); COSEBIs and pure-E/B computed from the shifted ξ± carry the blind, and +B-modes stay usable. A new statistic is refused under a blind until it has a +rule in `sacc_io` and, if shifted, a theory. Every SACC records the custody it was born under, and rule params carry the custody token, so a flip reruns what it touches. diff --git a/workflow/tests/test_toy_run.py b/workflow/tests/test_toy_run.py index b22998d7..1e80c7cd 100644 --- a/workflow/tests/test_toy_run.py +++ b/workflow/tests/test_toy_run.py @@ -152,12 +152,9 @@ def launch(*args): root, "python", "-c", - "import dataclasses, sys, yaml\n" + "import sys, yaml\n" "from sp_validation import blinding\n" - "from sp_validation.blinding_theory import TheoryConfig\n" - "fast = TheoryConfig(transfer_function='eisenstein_hu')\n" - "blinding.init('toy', yaml.safe_load(open(sys.argv[1]))," - " fiducial=dataclasses.asdict(fast))", + "blinding.init('toy', yaml.safe_load(open(sys.argv[1])))", str(cat_config), ) catalogues = yaml.safe_load(cat_config.read_text()) From 283fa1ea83c701f6fbb62b7bc8451e1aa7a1626c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 03:52:39 +0200 Subject: [PATCH 140/160] =?UTF-8?q?Tests:=20B-modes=20under=20a=20blind=20?= =?UTF-8?q?on=20synthetic=20=CE=BE=C2=B1;=20a=20custody=20flip=20reruns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_blinding_bmodes seals the default theory's ξ± on the production grids, noise-free and with one shape-noise draw, under a blind at each envelope corner. COSEBIs B_n move by exactly the transform's response to the E-mode shift, under a tenth of σ. Pure-E/B ξ_B does the same on noise-free input. On noisy input it is a strict xfail: cosmo_numba's adaptive quadrature does not converge on a noisy 1000-bin ξ±, so the kernel is not additive there. The pure-E/B transform pin on committed ξ± is back. test_dag checks that flipping a catalogue's declaration reruns its parts, as a params change. The toy run launches once under a blind drawn up front and checks its parts' stamps. The secrecy test also looks for Ω_c, and at logs, warnings and str/format. The README custody section is shorter and states what a blind does to each B-mode statistic. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GJJyGbKSuEjjX9m2zdv6yi --- src/sp_validation/tests/test_b_modes.py | 77 ++++++- src/sp_validation/tests/test_blinding.py | 18 +- .../tests/test_blinding_bmodes.py | 190 ++++++++++++++++++ workflow/README.md | 41 ++-- workflow/tests/test_dag.py | 40 +++- workflow/tests/test_toy_run.py | 108 ++++------ 6 files changed, 368 insertions(+), 106 deletions(-) create mode 100644 src/sp_validation/tests/test_blinding_bmodes.py diff --git a/src/sp_validation/tests/test_b_modes.py b/src/sp_validation/tests/test_b_modes.py index 67c64376..20a9aa67 100644 --- a/src/sp_validation/tests/test_b_modes.py +++ b/src/sp_validation/tests/test_b_modes.py @@ -1,8 +1,9 @@ """VALUE-DRIFT CHARACTERIZATION TESTS FOR THE B-MODE ESTIMATORS. This module pins the numeric behavior of the pure E/B-mode helpers in -``sp_validation.b_modes`` against fixed, deterministic, in-memory inputs -(seeded RNG and hand-built arrays — no cluster data, no catalogue files). +``sp_validation.b_modes`` against fixed, deterministic inputs (seeded RNG, +hand-built arrays and one committed ξ± fixture — no cluster data, no catalogue +files). Every pinned literal was produced by an actual run of the estimator inside the container; a future refactor that changes the numbers must fail. @@ -293,6 +294,78 @@ def test_calculate_eb_statistics_has_teeth(): assert loud_pte < 0.05 # louder B-modes are clearly rejected +# --------------------------------------------------------------------------- +# 5. pure_eb_from_xi on committed ξ± (the transform pin) +# --------------------------------------------------------------------------- + +# pure_eb_from_xi(**fixture); regenerated only when the transform is meant to move. +_PURE_EB_PINS = { + "xip_E": [ + -2.9831529669542025e-06, + -1.5008524620265777e-05, + 3.221623968725757e-07, + 1.1797672310858565e-05, + 5.715510692557323e-06, + 8.825804523824443e-07, + ], + "xim_E": [ + -4.737558091773235e-05, + -0.00010853189443993388, + -9.094825175032069e-05, + -5.826599101284694e-05, + -4.646405415748759e-05, + -1.9978028925333273e-05, + ], + "xip_B": [ + 1.7069121242262332e-05, + 3.059889782373755e-05, + -4.8805399253844115e-06, + -6.999262696335271e-06, + -1.2672006989728095e-05, + -1.214149138979614e-06, + ], + "xim_B": [ + -0.00011478091634539627, + -5.445112002141066e-05, + -3.100806652947907e-05, + -1.0940424256759085e-05, + -5.755185146643215e-06, + -1.628217762504557e-06, + ], + "xip_amb": [ + 0.00014017621792612224, + 0.0001378482153667787, + 0.0001339573019551001, + 0.00012745126271361765, + 0.00011662385105911986, + 9.851844704443032e-05, + ], + "xim_amb": [ + -4.389203999135455e-05, + 5.279277664928643e-05, + 5.800339397836051e-05, + 4.4242350610114584e-05, + 2.9902912946755567e-05, + 1.912262132836568e-05, + ], +} + + +def test_pure_eb_from_xi_reproduces_pins_on_committed_xi(pure_eb_xi): + """The pure-E/B transform of the committed ξ± reproduces its pins. + + With ξ± frozen, these pins move only when the transform does. rtol=1e-6 is + far above the 1e-12 reduction-order noise across thread counts. + """ + modes = b_modes.pure_eb_from_xi(**pure_eb_xi) + for key in b_modes._EB_KEYS: + npt.assert_allclose(modes[key], _PURE_EB_PINS[key], rtol=1e-6, err_msg=key) + + # Teeth: widening the integration interval by 1% leaves the pins. + moved = b_modes.pure_eb_from_xi(**{**pure_eb_xi, "tmax": 1.01 * pure_eb_xi["tmax"]}) + assert not np.allclose(moved["xip_E"], _PURE_EB_PINS["xip_E"], rtol=1e-6, atol=0) + + # --------------------------------------------------------------------------- # 6. Grid edges and the COSEBIs covariance seam # --------------------------------------------------------------------------- diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 27f1f8bf..e119345d 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -8,6 +8,7 @@ import dataclasses import json +import logging import stat import traceback @@ -289,28 +290,29 @@ def failing(params, *args): return "".join(trace.format()) -def test_the_hidden_cosmology_never_shows(tmp_path, capfd): - """Neither the seed nor the hidden S8, Ωm or σ8 reaches a repr, the - terminal or a traceback's locals.""" +def test_the_hidden_cosmology_never_shows(tmp_path, capfd, caplog, recwarn): + """Neither the seed nor the hidden S8, Ωm, σ8 or Ω_c reaches a repr, the + terminal, a log, a warning or a traceback's locals.""" + caplog.set_level(logging.DEBUG) cats = _catalogues(tmp_path) blind = bd.init("toy", cats) record = json.loads(blind.path.read_text()) hidden = bd._hidden(blind) sigma8 = hidden["S8"] / np.sqrt(hidden["Omega_m"] / 0.3) + omega_c = theory.cosmology(hidden)["Omega_c"] needles = [record["seed"], record["seed"][:16]] + [ form(x) - for x in (hidden["S8"], hidden["Omega_m"], sigma8) + for x in (hidden["S8"], hidden["Omega_m"], sigma8, omega_c) for form in (repr, "{:.4g}".format, "{:.6g}".format) ] bd.show("toy", cats) sio.seal(part(cl=False), cu.custody_of(cats, "TOY")) failure = _hidden_failure(blind) haystacks = [ - repr(blind), - repr(hidden), - str(hidden), - f"{hidden}", + *(form(x) for x in (blind, hidden) for form in (repr, str, "{}".format)), "".join(capfd.readouterr()), + caplog.text, + "".join(str(w.message) for w in recwarn), failure, ] for i, haystack in enumerate(haystacks): diff --git a/src/sp_validation/tests/test_blinding_bmodes.py b/src/sp_validation/tests/test_blinding_bmodes.py new file mode 100644 index 00000000..2f286bec --- /dev/null +++ b/src/sp_validation/tests/test_blinding_bmodes.py @@ -0,0 +1,190 @@ +"""B-modes under a blind, on synthetic ξ±. + +A blind shifts ξ± by one E-mode signal, so it moves COSEBIs B_n and pure-E/B +ξ_B only by each transform's response to a pure E-mode vector. The input is the +default theory's ξ± at the fiducial on the production reporting and integration +grids, noise-free and with one seeded shape-noise draw, sealed under a blind +whose hidden point is each corner of the envelope in turn. Every tolerance is +the same ceiling: a tenth of a bin's standard deviation. +""" + +import numpy as np +import pytest + +from sp_validation import b_modes, theory +from sp_validation import blinding as bd +from sp_validation import custody as cu +from sp_validation import sacc_io as sio + +REPORTING = (1.0, 250.0, 20) +INTEGRATION = (0.08, 300.0, 1000) +COSEBIS_CUT, NMODES = (12.0, 83.0), 20 +# SP_v1.4.6.3's shape noise (its cov_th in cosmo_val/cat_config.yaml). +AREA_DEG2, N_EFF, SIGMA_E = 2894.0, 4.96, 0.378 +CEILING = 0.1 + + +def _grid(lo, hi, n): + edges = np.geomspace(lo, hi, n + 1) + return np.sqrt(edges[:-1] * edges[1:]), edges + + +THETA, EDGES = _grid(*REPORTING) +THETA_INT, EDGES_INT = _grid(*INTEGRATION) + + +def _variance(edges): + """Var ξ± = σ_e⁴ / (2 N_pairs) (Schneider et al. 2002) in each bin.""" + n_gal = N_EFF * AREA_DEG2 * 3600 + return SIGMA_E**4 / (n_gal * N_EFF * np.pi * np.diff(edges**2)) + + +VARIANCE = np.concatenate([_variance(e) for e in (EDGES, EDGES, EDGES_INT, EDGES_INT)]) + + +def _fiducial_xi(): + """The default theory's ξ± on both grids, with the shape-noise covariance.""" + z = np.linspace(0.01, 3.0, 300) + s = sio.new_sacc({0: (z, np.exp(-(((z - 0.7) / 0.3) ** 2)))}) + for theta, grid in ((THETA, "reporting"), (THETA_INT, "integration")): + sio.add_xi(s, (0, 0), theta, 0 * theta, 0 * theta, grid=grid, theta_nom=theta) + for dp, value in zip(s.data, theory.predict(s, theory.fiducial())): + dp.value = float(value) + s.add_covariance(VARIANCE) + return s + + +def _vector(s): + """(ξ+, ξ−) on the reporting grid, then on the integration grid.""" + views = [sio.xi_correlation(s, grid=g) for g in ("reporting", "integration")] + return np.concatenate([np.r_[v.xip, v.xim] for v in views]) + + +@pytest.fixture(scope="module") +def shifts(tmp_path_factory): + """Each input's ξ± and, per envelope corner, its sealed ξ±'s shift.""" + root = tmp_path_factory.mktemp("bmodes") + catalogues = { + "paths": {"blinds": str(root / "blinds")}, + "SYN": {"shear": {"path": str(root / "SYN.fits")}, "blind": "syn"}, + } + bd.init("syn", catalogues) + custody = cu.custody_of(catalogues, "SYN") + clean = _fiducial_xi() + noisy = clean.copy() + noise = np.random.default_rng(3).normal(0.0, np.sqrt(VARIANCE)) + for dp, n in zip(noisy.data, noise): + dp.value += float(n) + fiducial = theory.fiducial() + corners = [ + {**fiducial, "S8": fiducial["S8"] + a, "Omega_m": fiducial["Omega_m"] + b} + for a in (-bd.ENVELOPE["S8"], bd.ENVELOPE["S8"]) + for b in (-bd.ENVELOPE["Omega_m"], bd.ENVELOPE["Omega_m"]) + ] + out = {"noise-free": (_vector(clean), []), "noisy": (_vector(noisy), [])} + with pytest.MonkeyPatch.context() as m: + for corner in corners: + m.setattr(bd, "_hidden", lambda blind, corner=corner: corner) + for name, s in (("noise-free", clean), ("noisy", noisy)): + out[name][1].append(_vector(sio.seal(s, custody)) - out[name][0]) + return out + + +def _as_b(delta): + """The pure-B counterpart of an E-mode ξ± vector: (δξ+, −δξ−) per grid.""" + n, n_int = len(THETA), len(THETA_INT) + sign = np.r_[np.ones(n), -np.ones(n), np.ones(n_int), -np.ones(n_int)] + return sign * delta + + +def _pure_b(x): + """Pure-E/B (ξ+_B, ξ−_B) on the reporting grid.""" + n, n_int = len(THETA), len(THETA_INT) + xip, xim = x[:n], x[n : 2 * n] + xip_int, xim_int = x[2 * n : 2 * n + n_int], x[2 * n + n_int :] + modes = b_modes.pure_eb_from_xi( + THETA, xip, xim, THETA_INT, xip_int, xim_int, EDGES[0], EDGES[-1] + ) + return np.r_[modes["xip_B"], modes["xim_B"]] + + +@pytest.fixture(scope="module") +def cosebis(): + """The fiducial cut's COSEBIs B_n of an integration-grid ξ±, and σ(B_n). + + The transform :func:`b_modes.cosebis_scan_from_xi` runs, built once. + """ + from cosmo_numba.B_modes.cosebis import COSEBIS + + n, n_int = len(THETA), len(THETA_INT) + cut = np.flatnonzero((THETA_INT >= COSEBIS_CUT[0]) & (THETA_INT <= COSEBIS_CUT[1])) + theta = THETA_INT[cut] + transform = COSEBIS( + theta_min=theta.min(), theta_max=theta.max(), N_max=NMODES, precision=120 + ) + rows = np.r_[2 * n + cut, 2 * n + n_int + cut] + covariance = transform.cosebis_covariance_from_xipm_covariance( + theta, np.diag(VARIANCE[rows]) + ) + + def b_n(x): + return transform.cosebis_from_xipm( + theta, x[2 * n + cut], x[2 * n + n_int + cut], parallel=True + )[1] + + return b_n, np.sqrt(np.diag(covariance)[NMODES:]) + + +@pytest.fixture(scope="module") +def sigma_pure_b(shifts): + """σ of pure-E/B ξ_B over shape-noise draws about the fiducial.""" + clean, _ = shifts["noise-free"] + rng = np.random.default_rng(5) + draws = [_pure_b(clean + rng.normal(0.0, np.sqrt(VARIANCE))) for _ in range(50)] + return np.std(draws, axis=0) + + +@pytest.mark.parametrize("noise", ["noise-free", "noisy"]) +def test_cosebis_b_modes_move_only_by_the_transforms_response(shifts, cosebis, noise): + """COSEBIs are linear with fixed weights: ΔB_n = B_n(δ) to round-off, and + B_n(δ), the transform's B response to a pure E-mode δ, stays under the + ceiling; δ's pure-B counterpart moves B_n well past it.""" + b_n, sigma = cosebis + x, deltas = shifts[noise] + for delta in deltas: + before, after = b_n(x), b_n(x + delta) + roundoff = 1e-10 * np.max(np.abs(np.r_[before, after, b_n(delta)])) + np.testing.assert_allclose(after - before, b_n(delta), rtol=0, atol=roundoff) + assert np.max(np.abs(b_n(delta)) / sigma) < CEILING + assert np.max(np.abs(b_n(_as_b(delta))) / sigma) > 10 * CEILING + + +@pytest.mark.parametrize( + "noise", + [ + "noise-free", + pytest.param( + "noisy", + marks=pytest.mark.xfail( + strict=True, + reason="pure-eb-bmode-numerics: cosmo_numba's adaptive quadrature " + "does not converge on a noisy 1000-bin ξ±, so the kernel is not " + "additive there", + ), + ), + ], +) +def test_pure_eb_b_modes_move_only_by_the_transforms_response( + shifts, sigma_pure_b, noise +): + """ΔB = B(x + δ) − B(x) equals B(δ), the kernel's B response to a pure + E-mode δ, and both stay under the ceiling; δ's pure-B counterpart moves B + well past it.""" + x, deltas = shifts[noise] + before = _pure_b(x) + for delta in deltas: + moved = _pure_b(x + delta) - before + response = _pure_b(delta) + assert np.max(np.abs(moved - response) / sigma_pure_b) < CEILING + assert np.max(np.abs(response) / sigma_pure_b) < CEILING + assert np.max(np.abs(_pure_b(_as_b(delta))) / sigma_pure_b) > 10 * CEILING diff --git a/workflow/README.md b/workflow/README.md index 00523ab3..d2e344eb 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -27,32 +27,25 @@ each namespaces cleanly under `results//`. ## Custody: which catalogues are blinded -Every entry of `cosmo_val/cat_config.yaml` declares `blind: none` (public), -`blind: mock`, or the name of the blind its signal is concealed under; an entry -without one is refused. `_leak_corr` and `_seed` versions take their entry's. -Entries reading one shear file declare one blind, and the repository config is -authoritative for the files it names. A run may show a blinded catalogue only -beside mocks and catalogues under the same blind, since overlaying it with any -other version shows the shift. A launch prints one `[custody]` line per -catalogue. - -Under a blind, every ξ±, pseudo-Cℓ_EE and γt value is shifted by t(hidden) − -t(fiducial) before it is first written, t being that data type's theory in -`sp_validation.theory.THEORY` (CCL through `cs_util.cosmo.get_cosmo` by -default); COSEBIs and pure-E/B computed from the shifted ξ± carry the blind, and -B-modes stay usable. A new statistic is refused under a blind until it has a -rule in `sacc_io` and, if shifted, a theory. Every SACC records the -custody it was born under, and rule params carry the custody token, so a flip -reruns what it touches. +Every entry of `cosmo_val/cat_config.yaml` declares `blind: none`, `blind: mock` +or the name of a blind; an entry without one is refused, and `_leak_corr` and +`_seed` versions take their entry's. Entries reading one shear file declare +one blind (the repository config is authoritative), and a blinded catalogue is +shown only beside mocks and its own blind, since any other overlay shows the +shift. A launch prints one `[custody]` line per catalogue. + +Under a blind, ξ±, pseudo-Cℓ_EE and γt are shifted by t(hidden) − t(fiducial) +before they are first written, t being the data type's theory in +`sp_validation.theory.THEORY`; statistics derived from them carry the blind, and +a data type with no blinding rule in `sacc_io` is refused. COSEBIs B_n move only +by the transform's response to an E-mode shift; pure-E/B ξ_B also moves where +cosmo_numba's quadrature does not converge on noisy ξ± (`test_blinding_bmodes`). +Rule params carry the custody token, so a flip reruns what it touches. A blind is one record, `/.blind.json`, outside any git -worktree; read access to it is access to the blind. Draw one, once: - -```bash -spv-container exec python -m sp_validation.blinding init -``` - -`… blinding show ` prints its public record. For an entry under a blind, +worktree; read access to it is access to the blind. +`spv-container exec python -m sp_validation.blinding init ` draws one, +once; `… show ` prints its public record. For an entry under a blind, measure signal only through `CosmologyValidation` or the workflow; never set `blind: none` to get a run through; never print, paste or commit a `.blind.json`. diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index c7862d12..428abe65 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -33,7 +33,7 @@ def grids(toy): def test_xi_leaves_a_measurement_only_as_a_part(toy, forced, grids): - """[P12] No job reads or writes a ξ± text dump; the ξ± figures draw the parts.""" + """No job reads or writes a ξ± text dump; the ξ± figures draw the parts.""" jobs = forced[1] dumps = [ f for j in jobs for f in j.input + j.output if re.search(r"_xi_.*\.txt$", f) @@ -62,7 +62,7 @@ def test_one_integration_grid(toy, forced, grids): def test_custody_is_the_checkouts_and_no_rule_touches_a_blind(toy, forced, tmp_path): - """[P9, P10, P8] A catalogue and its variant share one custody line; even a + """A catalogue and its variant share one custody line; even a forced run schedules nothing that reads or writes the registry; and a config file declaring the catalogue public changes nothing.""" output, jobs = forced @@ -85,8 +85,40 @@ def test_custody_is_the_checkouts_and_no_rule_touches_a_blind(toy, forced, tmp_p assert line in result.stdout +def test_a_custody_flip_reruns_the_catalogues_parts(toy, grids, tmp_path): + """A part's params carry its catalogue's custody token, so declaring + the catalogue public reruns the part and nothing else does.""" + env = toy.env | {"COSMO_VAL": str(tmp_path / "cosmo_val")} + reporting = toy.common.grid_binning(grids["reporting"]) + part = tmp_path / "cosmo_val" / f"{VERSIONS[0]}_xi_{reporting}.sacc" + # Records the job's params; the output itself must exist to be touched. + touched = toy.snakemake("--touch", str(part), env=env) + assert touched.returncode == 0, touched.stdout + part.parent.mkdir(exist_ok=True) + part.touch() + + def scheduled(): + result = toy.snakemake("-n", str(part), env=env) + assert result.returncode == 0, result.stdout + return [j.rule for j in parse_jobs(result.stdout)], result.stdout + + rules, output = scheduled() + assert rules == [], output + cat_config = toy.root / "cosmo_val" / "cat_config.yaml" + declared = cat_config.read_text() + flipped = yaml.safe_load(declared) + flipped[VERSIONS[0]]["blind"] = "none" + cat_config.write_text(yaml.safe_dump(flipped)) + try: + rules, output = scheduled() + finally: + cat_config.write_text(declared) + assert rules == ["xi"], output + assert "params have changed" in output.lower(), output + + LAUNCH_REFUSALS = { - # [P6] a blind the registry does not hold, an entry declaring none, and a + # A blind the registry does not hold, an entry declaring none, and a # blinded catalogue overlaid with a public one "no_blind": ([UNCOVERED], [f"{UNCOVERED} is blinded under gone", "init gone"]), "undeclared": ([UNDECLARED], ["declares no `blind:`"]), @@ -106,7 +138,7 @@ def test_a_launch_the_dag_cannot_honour_stops_with_the_fix(toy, case): def test_outputs_stay_in_the_output_roots(toy, forced): - """[P2] Nothing the suite declares lands outside the configured output roots.""" + """Nothing the suite declares lands outside the configured output roots.""" roots = [ r.resolve() for r in (toy.cosmo_val, toy.cosmo_inference, toy.rundir / "results") diff --git a/workflow/tests/test_toy_run.py b/workflow/tests/test_toy_run.py index 1e80c7cd..a00a0db9 100644 --- a/workflow/tests/test_toy_run.py +++ b/workflow/tests/test_toy_run.py @@ -1,12 +1,10 @@ -"""Rule xi, run for real through apptainer, before and after a blind is drawn. +"""Rule xi, run for real through apptainer, under a throwaway blind. The launch is the README's with the machine-independent default profile: the host Snakemake, the image `spv-container` manages, jobs on this node. A toy checkout under your home directory (which the profile binds) carries copies of workflow/, papers/cosmo_val/ and src/, and one synthetic catalogue declared -public. The first launch makes its reporting parts and their figure; then a -blind is drawn and declared on the catalogue, and the same launch re-measures -the parts concealed. +under a blind drawn in the image with the default theory. """ import json @@ -58,7 +56,7 @@ def _stamps(image, root, parts): def _toy_checkout(root, image): - """A checkout with one synthetic catalogue, SP_v0.1, declared public.""" + """A checkout with one synthetic catalogue, SP_v0.1, under the blind `toy`.""" skip = shutil.ignore_patterns(".snakemake", "__pycache__", "tests") shutil.copytree(REPO / "workflow", root / "workflow", ignore=skip) shutil.copytree( @@ -78,7 +76,17 @@ def _toy_checkout(root, image): "from pathlib import Path\n" "from _synthetic import write_synthetic_catalogs\n" f"write_synthetic_catalogs(Path({str(root / 'cosmo_val')!r})," - f" catalogues={{{VERSIONS[0]!r}: 'none'}})", + f" catalogues={{{VERSIONS[0]!r}: 'toy'}})", + ) + _in_image( + image, + root, + "python", + "-c", + "import sys, yaml\n" + "from sp_validation import blinding\n" + "blinding.init('toy', yaml.safe_load(open(sys.argv[1])))", + str(root / "cosmo_val" / "cat_config.yaml"), ) config_path = root / "papers" / "cosmo_val" / "config" / "config.yaml" @@ -92,16 +100,12 @@ def _toy_checkout(root, image): @pytest.mark.candide @on_candide -def test_xi_before_and_after_its_catalogue_is_blinded(): +def test_xi_parts_are_stamped_with_the_blinds_commitment(): image, kind = container.resolve_image() assert kind != "tag", "no local image; run `spv-container pull`" root = Path(tempfile.mkdtemp(prefix="toy_run_", dir=Path.home())) _toy_checkout(root, image) out = root / "out" - cat_config = root / "cosmo_val" / "cat_config.yaml" - parts = [out / f"{v}_xi_{BINNING}.sacc" for v in VERSIONS] - target = str(out / "snakemake_sentinels" / "plot_2pcf.done") - env = { k: v for k, v in os.environ.items() @@ -114,65 +118,33 @@ def test_xi_before_and_after_its_catalogue_is_blinded(): PYTHONNOUSERSITE="1", PYTHONUNBUFFERED="1", ) - - def launch(*args): - return subprocess.run( - [ - sys.executable, - "-m", - "snakemake", - "--profile", - str(root / "workflow" / "profiles" / "default"), - "--cores", - "4", - *args, - "--config", - f"container={image}", - "--", - target, - ], - cwd=root / "papers" / "cosmo_val", - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=1800, - check=False, - ) - - # Declared public: parts stamped public, and their figure drawn. - result = launch() - assert result.returncode == 0, result.stdout - assert [s["blind"] for s in _stamps(image, root, parts)] == ["none"] * 2 - assert (out / "xi_p.png").is_file() - - # A blind drawn and declared: the parts' params changed. - _in_image( - image, - root, - "python", - "-c", - "import sys, yaml\n" - "from sp_validation import blinding\n" - "blinding.init('toy', yaml.safe_load(open(sys.argv[1])))", - str(cat_config), + result = subprocess.run( + [ + sys.executable, + "-m", + "snakemake", + "--profile", + str(root / "workflow" / "profiles" / "default"), + "--cores", + "4", + "--config", + f"container={image}", + "--", + str(out / "snakemake_sentinels" / "plot_2pcf.done"), + ], + cwd=root / "papers" / "cosmo_val", + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=1800, + check=False, ) - catalogues = yaml.safe_load(cat_config.read_text()) - catalogues[VERSIONS[0]]["blind"] = "toy" - cat_config.write_text(yaml.safe_dump(catalogues, sort_keys=False)) - dry = launch("-n") - assert dry.returncode == 0, dry.stdout - assert "Params have changed" in dry.stdout, dry.stdout - - result = launch() assert result.returncode == 0, result.stdout + record = json.loads((root / "cosmo_val/blinds/toy.blind.json").read_text()) - for stamp in _stamps(image, root, parts): - assert stamp == { - "blind": "toy", - "blind_commitment": custody.commitment(record), - } - assert not list(out.rglob("*_xi_*.txt")) - assert not list((root / "cosmo_val" / "output").iterdir()) + stamp = {"blind": "toy", "blind_commitment": custody.commitment(record)} + parts = [out / f"{v}_xi_{BINNING}.sacc" for v in VERSIONS] + assert _stamps(image, root, parts) == [stamp] * len(parts) shutil.rmtree(root) # kept on failure, for post-mortem From 750794cc42027eeac78b9d0e3b63fd3c934b158a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 05:03:12 +0200 Subject: [PATCH 141/160] Custody token is checked, stamped as one key; theory fixes; tighter tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CosmologyValidation refuses a custody token whose blind differs from the catalogue config's declaration. - A SACC's custody stamp is its token, under one metadata key; the 2pt-FITS converter reads only stamped SACCs. - Default theory: CCL's ξ± Hankel transform extrapolates to ℓ = 10⁷; γt takes each lens sample's bias from its tracer (add_lens(bias=)), not the blind's fiducial; a map-based pseudo-Cℓ window includes pw²(ℓ). - Tests: explicit seal/derivation cases replace hypothesis (dropped from the test extra); pure-E/B additivity asserted at the quadrature floor, the noisy xfail states the measured 0.4σ; ⟨M_ap²⟩ pinned to TreeCorr; a token round-trip replaces the host/job test; develop's assemble and unnamed-output DAG tests restored. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 1 - src/sp_validation/cosmo_val/core.py | 32 +++-- src/sp_validation/cosmo_val/pseudo_cl.py | 16 ++- src/sp_validation/cosmo_val/sacc_writers.py | 9 +- src/sp_validation/custody.py | 31 ++--- src/sp_validation/pseudo_cl.py | 9 +- src/sp_validation/sacc_io.py | 28 ++-- src/sp_validation/tests/test_blinding.py | 123 +++++++++--------- .../tests/test_blinding_bmodes.py | 21 +-- src/sp_validation/tests/test_cosmo_val.py | 30 ++++- src/sp_validation/tests/test_custody.py | 46 +++---- .../tests/test_sacc_io_realdata.py | 2 + .../tests/test_sacc_io_twopoint.py | 17 ++- src/sp_validation/tests/test_sacc_writers.py | 10 ++ src/sp_validation/theory.py | 24 ++-- uv.lock | 66 ---------- workflow/tests/test_dag.py | 48 ++++++- workflow/tests/test_toy_run.py | 6 +- 18 files changed, 275 insertions(+), 244 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 98166f2a..25ca0d54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,7 +122,6 @@ environments = ["sys_platform == 'linux'"] [project.optional-dependencies] test = [ - "hypothesis", "pytest", "pytest-cov", "ruff" diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index fd5ba794..0dd2a38f 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -132,8 +132,9 @@ class CosmologyValidation( cosmo_params : dict, optional Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. custody : dict, optional - ``{version: custody token}`` (a workflow job's ``params.custody``), - replacing the catalogue config's declaration for those versions. + ``{version: custody token}`` (a workflow job's ``params.custody``): + the host's resolution of the catalogue config's declaration, whose + blind must match it. Attributes ---------- @@ -336,14 +337,7 @@ def ensure_version_exists(ver): final_versions.append(ver) self.versions = final_versions - _custody.check_mix( - { - v: _custody.parse(self._tokens[v]).blind - if v in self._tokens - else _custody.declared(self._declared, v) - for v in self.versions - } - ) + _custody.check_mix({v: self._blind(v) for v in self.versions}) if output_dir is not None: cc["paths"]["output"] = output_dir @@ -459,10 +453,24 @@ def results_objectwise(self): self._results_objectwise = self.init_results(objectwise=True) return self._results_objectwise + def _blind(self, version): + """The blind the catalogue config declares for ``version``; a given + token naming another is refused.""" + blind = _custody.declared(self._declared, version) + if version in self._tokens: + given = _custody.parse(self._tokens[version]).blind + if given != blind: + raise _custody.CustodyError( + f"{version} was given custody {given!r}, but the catalogue " + f"config declares blind: {blind}" + ) + return blind + def custody(self, version): """The custody every SACC this object writes for ``version`` is sealed - under: the token it was given for ``version`` (a workflow job's - ``params.custody``), else the catalogue config's declaration.""" + under: its given token (whose commitment ``open_blind`` checks against + the record), else the catalogue config's declaration.""" + self._blind(version) if version in self._tokens: return _custody.parse(self._tokens[version], self._declared) return _custody.custody_of(self._declared, version) diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index c3593b71..4c613770 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -600,7 +600,9 @@ def calculate_pseudo_cl_map(self, ver, nside, out_path): cl_shear = cl_shear - cl_noise self.print_cyan("Saving pseudo-Cl's...") - sealed = self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) + sealed = self.pseudo_cl_to_sacc_part( + ver, out_path, ell_eff, cl_shear, wsp, nside=nside + ) self._pseudo_cls[ver]["pseudo_cl"] = _spectra(sealed) def calculate_pseudo_cl_catalog(self, ver, out_path): @@ -704,13 +706,16 @@ def apply_random_rotation(self, e1, e2, rng=None): """ return apply_random_rotation(e1, e2, rng) - def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): + def pseudo_cl_to_sacc_part( + self, version, out_path, ell_eff, cl_all, wsp, nside=None + ): """Write the pseudo-Cl SACC part (EE/BB/EB + shared bandpower window). ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); - the writer takes the shared bandpower window from ``wsp``. No covariance - is attached here. Returns the part as sealed under the version's - custody. + the writer takes the shared bandpower window from ``wsp``, with the + pixel window of maps at ``nside`` for a map-based spectrum. No + covariance is attached here. Returns the part as sealed under the + version's custody. """ s = pseudo_cl_to_sacc( self.sacc_nz(version), @@ -718,6 +723,7 @@ def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): ell_eff, cl_all, wsp, + nside=nside, ) return sacc_io.save(s, out_path, custody=self.custody(version)) diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index aef72b33..8dcdc9c2 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -71,14 +71,15 @@ def xi_to_sacc( return s -def pseudo_cl_to_sacc(nz, metadata, ell_eff, cl_all, wsp, covariance=None): +def pseudo_cl_to_sacc(nz, metadata, ell_eff, cl_all, wsp, covariance=None, nside=None): """One pseudo-Cℓ part: EE/BB/EB with the shared bandpower window. ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); the - window comes from :func:`bandpower_window_from_workspace`. ``covariance``, - when given, is the dense ``[EE; BB; EB]``-ordered block matching insertion. + window comes from :func:`bandpower_window_from_workspace`, with the pixel + window of maps at ``nside`` when given. ``covariance``, when given, is the + dense ``[EE; BB; EB]``-ordered block matching insertion. """ - window_ells, window_weights = bandpower_window_from_workspace(wsp) + window_ells, window_weights = bandpower_window_from_workspace(wsp, nside) s = sio.new_sacc(nz, metadata) sio.add_pseudo_cl( s, diff --git a/src/sp_validation/custody.py b/src/sp_validation/custody.py index 0f58f3c4..9a65bfc9 100644 --- a/src/sp_validation/custody.py +++ b/src/sp_validation/custody.py @@ -11,7 +11,7 @@ A blind is the record ``/.blind.json``, drawn by :mod:`sp_validation.blinding`. Its commitment, the fork's ``seed_commitment`` of the whole canonical record, is public: it names the -blind in custody tokens and SACC stamps. +blind in custody tokens, and a SACC's stamp is its custody token. The host Snakemake loads this module by path, so it imports only the standard library (and PyYAML, which Snakemake carries, for the repository config). @@ -30,7 +30,7 @@ REPO_CAT_CONFIG = Path(__file__).parents[2] / "cosmo_val" / "cat_config.yaml" # smokescreen.COMMITMENT_DOMAIN, spelt here for the host; test_custody pins it. COMMITMENT_DOMAIN = b"smokescreen-seed-commitment-v1|" -STAMP_KEYS = ("blind", "blind_commitment") +STAMP_KEY = "custody" BLIND_NAME = re.compile(r"[a-z0-9][a-z0-9_.-]*") _SEED_SUFFIX = re.compile(r"_seed\d+$") _LEAK_SUFFIX = "_leak_corr" @@ -57,13 +57,6 @@ def token(self): """One string that changes whenever the custody does.""" return f"{self.blind}:{self.commitment}" if self.blinded else self.blind - @property - def stamp(self): - """The metadata a SACC born under this custody carries.""" - if self.blinded: - return {"blind": self.blind, "blind_commitment": self.commitment} - return {"blind": self.blind} - def parse(token, catalogues=None): """The custody a token (:attr:`Custody.token`) names, with the registry of @@ -265,17 +258,17 @@ def summary(catalogues, versions): def read_stamp(metadata): - """The custody a SACC's stamp records; a missing or malformed stamp raises.""" - blind = metadata.get("blind") - if not isinstance(blind, str) or not BLIND_NAME.fullmatch(blind): + """The custody a SACC's stamp (its ``custody`` token) records; a missing or + malformed stamp raises.""" + token = metadata.get(STAMP_KEY) + custody = parse(token) if isinstance(token, str) else None + if ( + custody is None + or not BLIND_NAME.fullmatch(custody.blind) + or custody.blinded != (custody.commitment is not None) + ): raise CustodyError( - f"no valid custody stamp (blind={blind!r}): every SACC is born " + f"no valid custody stamp ({STAMP_KEY}={token!r}): every SACC is born " "through sacc_io.save" ) - custody = Custody(blind, metadata.get("blind_commitment")) - if custody.blinded != (custody.commitment is not None): - raise CustodyError( - f"malformed custody stamp: blind={blind}, " - f"blind_commitment={custody.commitment}" - ) return custody diff --git a/src/sp_validation/pseudo_cl.py b/src/sp_validation/pseudo_cl.py index 34cbc68d..bdcc8d67 100644 --- a/src/sp_validation/pseudo_cl.py +++ b/src/sp_validation/pseudo_cl.py @@ -286,7 +286,7 @@ def get_pseudo_cls_catalog( _NMT_EE = 0 -def bandpower_window_from_workspace(wsp): +def bandpower_window_from_workspace(wsp, nside=None): """Extract the bandpower window matrix ``W`` for a spin-2×spin-2 workspace. NaMaster's ``get_bandpower_windows()`` returns a four-index array @@ -303,9 +303,14 @@ def bandpower_window_from_workspace(wsp): ``compute_coupled_cell``. window_weights : np.ndarray ``W`` of shape ``(n_ell, n_bpw)`` — one column per bandpower, the layout - :func:`sp_validation.sacc_io.add_pseudo_cl` expects. + :func:`sp_validation.sacc_io.add_pseudo_cl` expects. For a spectrum of + HEALPix maps at ``nside``, ``W`` includes the pixel window pw²(ℓ), so it + maps a C_ℓ to the measured bandpower. """ bpw = wsp.get_bandpower_windows() # (n_cl_out, n_bpw, n_cl_in, n_ell) diagonal = bpw[_NMT_EE, :, _NMT_EE, :] # (n_bpw, n_ell) window_ells = np.arange(diagonal.shape[1], dtype=float) + if nside is not None: + pixwin = hp.pixwin(nside, lmax=diagonal.shape[1] - 1) + diagonal = diagonal * pixwin**2 return window_ells, diagonal.T diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 9783a4fc..d59f00e6 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -425,13 +425,17 @@ def add_pure_eb( _add_theta_series(s, PURE_TYPES[key], tracers, theta, arr, grid=grid) -def add_lens(s, name, nz=None, *, quantity="galaxy_density"): - """Add a γt lens sample: galaxies with their ``(z, n(z))``, or, as a - tracer without one, ``quantity="stars"`` or ``"randoms"``.""" +def add_lens(s, name, nz=None, *, quantity="galaxy_density", bias=None): + """Add a γt lens sample: galaxies with their ``(z, n(z))`` and linear + ``bias``, or, as a tracer without either, ``quantity="stars"`` or + ``"randoms"``.""" if nz is None: s.add_tracer("Misc", name, quantity=quantity) else: - s.add_tracer("NZ", name, *map(np.asarray, nz), quantity=quantity) + metadata = {} if bias is None else {"bias": float(bias)} + s.add_tracer( + "NZ", name, *map(np.asarray, nz), quantity=quantity, metadata=metadata + ) def add_gamma_t(s, source_bin, lens, theta, gamma_t, gamma_x=None, **tags): @@ -1049,13 +1053,11 @@ def _refuse_unruled(s): def _stamped(s): - return any(key in s.metadata for key in _custody.STAMP_KEYS) + return _custody.STAMP_KEY in s.metadata -def _mint(s, stamp): - for key in _custody.STAMP_KEYS: - s.metadata.pop(key, None) - s.metadata.update(stamp) +def _mint(s, custody): + s.metadata[_custody.STAMP_KEY] = custody.token def seal(s, custody): @@ -1089,7 +1091,7 @@ def seal(s, custody): out = blinding.conceal(s, blinding.open_blind(custody)) else: out = s.copy() - _mint(out, custody.stamp) + _mint(out, custody) return out @@ -1125,7 +1127,7 @@ def _derive(s, parts, custody): "save(custody=)" ) out = s.copy() - _mint(out, stamp.stamp) + _mint(out, stamp) return out @@ -1368,10 +1370,12 @@ def sacc_to_twopoint_fits( Raises ------ ValueError - If the SACC has no ξ points; if ``n_bins != 1`` or the SACC's ξ tracer + If the SACC carries no custody stamp (it was not born through + :func:`save`); if it has no ξ points; if ``n_bins != 1`` or its ξ tracer pairs are anything other than exactly ``{(source_0, source_0)}`` (the single-bin contract); or if exactly one of the ρ/τ sidecars is supplied. """ + _custody.read_stamp(s.metadata) if (rho_stats_hdu is None) != (tau_stats_hdu is None): raise ValueError( "rho_stats_hdu and tau_stats_hdu must be supplied together " diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index e119345d..1bf716b8 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -15,15 +15,13 @@ import numpy as np import pytest from _synthetic import TOY_THEORY -from hypothesis import example, given, settings -from hypothesis import strategies as st from sp_validation import blinding as bd from sp_validation import custody as cu from sp_validation import sacc_io as sio from sp_validation import theory -VERSIONS = ("TOY", "OTHER", "TOY_OPEN", "OTHER_OPEN", "TOY_MOCK") +VERSIONS = ("TOY", "OTHER", "TOY_OPEN", "TOY_MOCK") DEFAULTS = theory.THEORY @@ -40,7 +38,6 @@ def _catalogues(root): "TOY": "toy", "OTHER": "other", "TOY_OPEN": "none", - "OTHER_OPEN": "none", "TOY_MOCK": "mock", } config = {"paths": {"blinds": str(root / "blinds")}} @@ -51,7 +48,7 @@ def _catalogues(root): @pytest.fixture(scope="module") def blinds(tmp_path_factory): - """The five custodies: TOY under blind `toy`, OTHER under `other`.""" + """The four custodies: TOY under blind `toy`, OTHER under `other`.""" cats = _catalogues(tmp_path_factory.mktemp("registry")) for name in ("toy", "other"): bd.init(name, cats) @@ -105,8 +102,10 @@ def part( if rho: sio.add_rho(s, 0, theta, np.arange(1, 7) * 1e-7, np.arange(1, 7) * 2e-7) for quantity in gt: - nz = _nz(0.3) if quantity == "galaxy_density" else None - sio.add_lens(s, quantity, nz, quantity=quantity) + if quantity == "galaxy_density": + sio.add_lens(s, quantity, _nz(0.3), quantity=quantity, bias=1.5) + else: + sio.add_lens(s, quantity, quantity=quantity) sio.add_gamma_t(s, 1, quantity, theta, 1e-4 * (theta / 10) ** -0.7, 0 * theta) if derived == "cosebis": sio.add_cosebis(s, (0, 0), np.arange(1, 6) * 1e-10, (12.0, 83.0), Bn=np.ones(5)) @@ -126,53 +125,42 @@ def _values(s): # --------------------------------------------------------------------------- # -# Born sealed: a blind shifts exactly the ξ± and Cℓ_EE rows, and nothing else +# Born sealed: a blind shifts exactly the ξ±, Cℓ_EE and γt rows, nothing else # --------------------------------------------------------------------------- # -PARTS = st.fixed_dictionaries( - { - "xi_tags": st.lists( - st.sampled_from(["reporting", "integration", "cosebis", None]), - max_size=2, - unique=True, - ), - "cl": st.booleans(), - "rho": st.booleans(), - "gt": st.lists( - st.sampled_from(["galaxy_density", "stars", "randoms"]), unique=True - ), - "derived": st.sampled_from([None, "cosebis", "pure_eb"]), - "unruled": st.booleans(), - } -).filter(lambda p: p["xi_tags"] or p["cl"] or p["rho"]) - - -@settings(max_examples=10, deadline=None) -@given(content=PARTS, version=st.sampled_from(["TOY", "TOY_OPEN", "TOY_MOCK"])) -@example( - content=dict( - xi_tags=["mystery", None], - cl=True, - rho=True, - gt=["galaxy_density", "stars"], - derived=None, - unruled=False, - ), - version="TOY", +EVERYTHING = dict( + xi_tags=("reporting", "mystery", None), + rho=True, + gt=("galaxy_density", "stars", "randoms"), +) + + +@pytest.mark.parametrize( + "content, version, refused", + [ + (EVERYTHING, "TOY", False), + (EVERYTHING, "TOY_OPEN", False), + (EVERYTHING, "TOY_MOCK", False), + (dict(derived="cosebis"), "TOY", True), + (dict(derived="pure_eb"), "TOY", True), + (dict(unruled=True), "TOY", True), + (dict(derived="pure_eb", unruled=True), "TOY_OPEN", False), + ], ) -def test_seal_shifts_only_the_signal_it_has_a_rule_for(blinds, content, version): +def test_seal_shifts_only_the_signal_it_has_a_rule_for( + blinds, content, version, refused +): """Under a blind every ξ±, Cℓ_EE and galaxy-lens γt row moves, whatever its grid tag, and every other value (γ×, γt around stars or randoms) and - the covariance stay bitwise; a birth carrying - derived rows or a type with no blinding rule is refused. Unblinded and - mock births keep their values. Each is stamped with its custody.""" + the covariance stay bitwise; a blinded birth carrying derived rows or a + type with no blinding rule is refused. Unblinded and mock births keep + their values. Each is stamped with its custody.""" s, custody = part(**content), blinds[version] - blinded = custody.blinded - if blinded and (content["derived"] or content["unruled"]): + if refused: with pytest.raises(ValueError, match="derived_from|no blinding rule"): sio.seal(s, custody) return sealed = sio.seal(s, custody) - assert cu.read_stamp(sealed.metadata).stamp == custody.stamp + assert cu.read_stamp(sealed.metadata) == custody shiftable = np.array( [ dp.data_type in sio.SHIFTABLE and not {"stars", "randoms"} & {*dp.tracers} @@ -180,7 +168,7 @@ def test_seal_shifts_only_the_signal_it_has_a_rule_for(blinds, content, version) ] ) moved = _values(sealed) != _values(s) - assert np.array_equal(moved, shiftable & blinded) + assert np.array_equal(moved, shiftable & custody.blinded) assert np.array_equal(sealed.covariance.dense, s.covariance.dense) @@ -204,19 +192,22 @@ def parts(blinds): return {v: sio.seal(part(cl=False), blinds[v]) for v in VERSIONS} -@settings(max_examples=25, deadline=None) -@given( - inputs=st.lists(st.sampled_from(VERSIONS), min_size=1, max_size=3), - declared=st.sampled_from([None, *VERSIONS]), - content=st.sampled_from(["copy", "cosebis", "plaintext"]), +@pytest.mark.parametrize( + "inputs, declared, content, allowed", + [ + (["TOY"], None, "copy", True), + (["TOY"], None, "cosebis", True), + (["TOY_OPEN"], None, "plaintext", True), + (["TOY", "TOY"], "TOY", "copy", True), # an assembly + (["TOY"], None, "plaintext", False), # laundering + (["OTHER"], "TOY", "copy", False), # another blind + (["TOY_OPEN"], "TOY", "copy", False), # unblinded into blinded + (["TOY", "OTHER"], None, "cosebis", False), # mixed stamps + (["TOY_MOCK", "TOY_OPEN"], None, "copy", False), # mixed stamps + ], ) -@example(inputs=["TOY"], declared=None, content="plaintext") # laundering -@example(inputs=["TOY", "TOY"], declared="TOY", content="copy") # an assembly -@example(inputs=["OTHER"], declared="TOY", content="copy") # another blind -@example(inputs=["TOY_OPEN"], declared="TOY", content="copy") # unblinded into blinded -@example(inputs=["TOY", "OTHER"], declared=None, content="cosebis") def test_a_derivation_carries_its_inputs_one_stamp( - blinds, parts, tmp_path_factory, inputs, declared, content + blinds, parts, tmp_path, inputs, declared, content, allowed ): """``save(s, derived_from=inputs, custody=declared)`` writes ``s`` under the inputs' stamp exactly when they share one, it is the declared @@ -227,13 +218,7 @@ def test_a_derivation_carries_its_inputs_one_stamp( "cosebis": lambda: part(xi_tags=(), cl=False, derived="cosebis"), "plaintext": lambda: part(cl=False), }[content]() - stamp = blinds[inputs[0]].stamp - allowed = ( - all(blinds[v].stamp == stamp for v in inputs) - and (declared is None or blinds[declared].stamp == stamp) - and not (blinds[inputs[0]].blinded and content == "plaintext") - ) - path = tmp_path_factory.mktemp("derived") / "d.sacc" + path = tmp_path / "d.sacc" kwargs = dict( derived_from=[parts[v] for v in inputs], custody=blinds[declared] if declared else None, @@ -245,7 +230,7 @@ def test_a_derivation_carries_its_inputs_one_stamp( return sio.save(s, path, **kwargs) loaded = sio.load(path) - assert cu.read_stamp(loaded.metadata).stamp == stamp + assert cu.read_stamp(loaded.metadata) == blinds[inputs[0]] assert np.array_equal(_values(loaded), _values(s)) @@ -336,6 +321,14 @@ def test_conceal_refuses_a_shift_it_cannot_make(blinds): bd.conceal(s, blind, theory=flat) +def test_gamma_t_needs_its_lens_samples_bias(): + s = sio.new_sacc({0: _nz(0.5)}) + sio.add_lens(s, "lens", _nz(0.3)) + sio.add_gamma_t(s, 0, "lens", np.geomspace(2.0, 200.0, 4), np.ones(4)) + with pytest.raises(ValueError, match="no linear bias"): + DEFAULTS[sio.GAMMA_T](theory.fiducial(), s, s.indices(sio.GAMMA_T)) + + @pytest.mark.slow def test_the_ccl_defaults_shift_with_s8(blinds, monkeypatch): """The default theories, at a hidden point with S8 above the fiducial, diff --git a/src/sp_validation/tests/test_blinding_bmodes.py b/src/sp_validation/tests/test_blinding_bmodes.py index 2f286bec..6c7a5c70 100644 --- a/src/sp_validation/tests/test_blinding_bmodes.py +++ b/src/sp_validation/tests/test_blinding_bmodes.py @@ -4,8 +4,10 @@ ξ_B only by each transform's response to a pure E-mode vector. The input is the default theory's ξ± at the fiducial on the production reporting and integration grids, noise-free and with one seeded shape-noise draw, sealed under a blind -whose hidden point is each corner of the envelope in turn. Every tolerance is -the same ceiling: a tenth of a bin's standard deviation. +whose hidden point is each corner of the envelope in turn. The transform's +response to δ must stay under a tenth of a bin's standard deviation; pure-E/B's +additivity, B(x + δ) − B(x) = B(δ), holds noise-free to its quadrature's floor, +measured at ≤ 10⁻³σ and asserted at 10⁻²σ. """ import numpy as np @@ -22,6 +24,7 @@ # SP_v1.4.6.3's shape noise (its cov_th in cosmo_val/cat_config.yaml). AREA_DEG2, N_EFF, SIGMA_E = 2894.0, 4.96, 0.378 CEILING = 0.1 +QUADRATURE_FLOOR = 0.01 def _grid(lo, hi, n): @@ -167,9 +170,11 @@ def test_cosebis_b_modes_move_only_by_the_transforms_response(shifts, cosebis, n "noisy", marks=pytest.mark.xfail( strict=True, - reason="pure-eb-bmode-numerics: cosmo_numba's adaptive quadrature " - "does not converge on a noisy 1000-bin ξ±, so the kernel is not " - "additive there", + reason="pure-eb-bmode-numerics: on a noisy 1000-bin ξ± " + "cosmo_numba's adaptive quadrature is not additive: an E-mode δ " + "moves ξ_B by up to 0.4σ at the envelope's corners, unchanged by " + "tightening epsabs/epsrel from 1e-10 to 1e-13, so blinded and " + "unblinded pure-E/B alike carry this error", ), ), ], @@ -178,13 +183,13 @@ def test_pure_eb_b_modes_move_only_by_the_transforms_response( shifts, sigma_pure_b, noise ): """ΔB = B(x + δ) − B(x) equals B(δ), the kernel's B response to a pure - E-mode δ, and both stay under the ceiling; δ's pure-B counterpart moves B - well past it.""" + E-mode δ, to the quadrature's floor, and B(δ) stays under the ceiling; + δ's pure-B counterpart moves B well past it.""" x, deltas = shifts[noise] before = _pure_b(x) for delta in deltas: moved = _pure_b(x + delta) - before response = _pure_b(delta) - assert np.max(np.abs(moved - response) / sigma_pure_b) < CEILING + assert np.max(np.abs(moved - response) / sigma_pure_b) < QUADRATURE_FLOOR assert np.max(np.abs(response) / sigma_pure_b) < CEILING assert np.max(np.abs(_pure_b(_as_b(delta))) / sigma_pure_b) > 10 * CEILING diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index c5467fd4..e90d7d2a 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -536,7 +536,35 @@ def test_a_blinded_catalogues_xi_leaves_concealed(blinded_and_twin): np.testing.assert_allclose( blinded.mean - twin.mean, shift, rtol=1e-8, atol=1e-12 * np.abs(shift).max() ) - assert blinded.metadata["blind"] == "toy" + assert blinded.metadata["custody"] == cv.custody("TOY").token assert cv.xi_parts["TOY", "reporting"] is blinded with pytest.raises(CustodyError, match="shows the blind's shift"): CosmologyValidation(versions=["TOY", "TOY_OPEN"], **grid) + + +def test_map2_transform_is_treecorrs_calculate_map_sq(): + """⟨M_ap²⟩, ⟨M_ײ⟩ from the ξ± transform equal TreeCorr's own sum.""" + import treecorr + + from sp_validation.cosmo_val.real_space import _map2_transform + + rng = np.random.default_rng(1) + n = 5000 + cat = treecorr.Catalog( + ra=rng.uniform(0, 5, n), + dec=rng.uniform(0, 5, n), + g1=rng.normal(0, 0.3, n), + g2=rng.normal(0, 0.3, n), + ra_units="deg", + dec_units="deg", + ) + gg = treecorr.GGCorrelation( + min_sep=0.5, max_sep=200, nbins=200, sep_units="arcmin", bin_slop=0 + ) + gg.process(cat) + radii = np.geomspace(1, 60, 8) + mapsq, _, mxsq, _, _ = gg.calculateMapSq(R=radii, m2_uform="Schneider") + ours = _map2_transform(radii, gg.meanr, gg.bin_size) @ np.r_[gg.xip, gg.xim] + treecorrs = np.r_[mapsq, mxsq] + atol = 1e-12 * np.max(np.abs(treecorrs)) + np.testing.assert_allclose(ours, treecorrs, rtol=1e-12, atol=atol) diff --git a/src/sp_validation/tests/test_custody.py b/src/sp_validation/tests/test_custody.py index f95a64e8..1ce93717 100644 --- a/src/sp_validation/tests/test_custody.py +++ b/src/sp_validation/tests/test_custody.py @@ -4,7 +4,6 @@ no real seed. """ -import importlib.util import json from pathlib import Path @@ -137,39 +136,28 @@ def test_every_catalogue_in_the_repository_resolves(): assert blinds and set(blinds.values()) <= {cu.NONE, cu.MOCK}, blinds -def test_host_and_job_resolve_the_same_custody(tmp_path): - """`common.custody_token(v)` is what a job's CosmologyValidation seals under.""" +def test_a_token_is_the_custody_it_names_and_must_match_the_declaration(tmp_path): + """A job seals under its token's custody, which is the declared one; a + token naming another blind is refused.""" from sp_validation.cosmo_val import CosmologyValidation - (tmp_path / "workflow").mkdir() - (tmp_path / "workflow" / "common.py").write_text( - (REPO / "workflow" / "common.py").read_text() - ) - (tmp_path / "src").symlink_to(REPO / "src") - (tmp_path / "cosmo_val").mkdir() cats = _catalogues(tmp_path, TOY="toy", TOY_OPEN="none", TOY_MOCK="mock") _record(tmp_path, "toy") - (tmp_path / "cosmo_val" / "cat_config.yaml").write_text(yaml.safe_dump(cats)) - spec = importlib.util.spec_from_file_location( - "toy_common", tmp_path / "workflow" / "common.py" - ) - common = importlib.util.module_from_spec(spec) - spec.loader.exec_module(common) - common.CATALOG_CONFIG = yaml.safe_load(Path(common.CAT_CONFIG).read_text()) - - for version in ("TOY", "TOY_leak_corr", "TOY_OPEN", "TOY_MOCK"): - token = common.custody_token(version) - job = CosmologyValidation( + config = tmp_path / "cat_config.yaml" + config.write_text(yaml.safe_dump(cats)) + + def job(version, token): + return CosmologyValidation( versions=[version], - catalog_config=common.CAT_CONFIG, + catalog_config=str(config), output_dir=str(tmp_path / "out"), custody={version: token}, ) - interactive = CosmologyValidation( - versions=[version], - catalog_config=common.CAT_CONFIG, - output_dir=str(tmp_path / "out"), - ) - assert job.custody(version) == interactive.custody(version), version - assert job.custody(version).token == token - assert common.custody_token("TOY").startswith("toy:") + + for version in ("TOY", "TOY_OPEN", "TOY_MOCK"): + custody = cu.custody_of(cats, version) + assert cu.parse(custody.token, cats) == custody + assert job(version, custody.token).custody(version) == custody + for token in ("none", "mock", "other:" + "0" * 64): + with pytest.raises(cu.CustodyError, match="declares blind: toy"): + job("TOY", token) diff --git a/src/sp_validation/tests/test_sacc_io_realdata.py b/src/sp_validation/tests/test_sacc_io_realdata.py index abe1f230..97f7541f 100644 --- a/src/sp_validation/tests/test_sacc_io_realdata.py +++ b/src/sp_validation/tests/test_sacc_io_realdata.py @@ -29,6 +29,7 @@ from astropy.io import fits from sp_validation import sacc_io +from sp_validation.custody import Custody _DATA = Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") _REAL_FILES = { @@ -123,6 +124,7 @@ def _sacc_from_2pt_fits(hdul): for key in ("t0m", "t2m"): full[np.ix_(idx[key], idx[key])] = np.eye(n) s.add_covariance(full) + s = sacc_io.seal(s, Custody("mock")) tau_sidecar = fits.BinTableHDU.from_columns( fits.ColDefs( diff --git a/src/sp_validation/tests/test_sacc_io_twopoint.py b/src/sp_validation/tests/test_sacc_io_twopoint.py index 78f6809d..0187e8da 100644 --- a/src/sp_validation/tests/test_sacc_io_twopoint.py +++ b/src/sp_validation/tests/test_sacc_io_twopoint.py @@ -37,6 +37,9 @@ from astropy.io import fits from sp_validation import sacc_io +from sp_validation.custody import Custody + +MOCK = Custody("mock") _SCRIPT = ( Path(__file__).resolve().parents[3] @@ -262,7 +265,7 @@ def _sacc(inp, *, cl=False, rho_tau=False): idx = s.indices(dtype, (SOURCE, PSF)) full[np.ix_(idx, idx)] = np.eye(N_ANG) s.add_covariance(full) - return s + return sacc_io.seal(s, MOCK) def _sidecar_hdus(tmp_path, inp): @@ -450,6 +453,7 @@ def test_integration_grid_points_ignored(tmp_path): full[np.ix_(xi_int_all, xi_int_all)] = _spd(2 * N_ANG, 43) s_aug.add_covariance(full) + s_aug = sacc_io.seal(s_aug, MOCK) out_aug = tmp_path / "aug.fits" sacc_io.sacc_to_twopoint_fits( @@ -488,6 +492,7 @@ def test_tomographic_sacc_raises(tmp_path): for pair in [(0, 0), (0, 1), (1, 1)]: sacc_io.add_xi(s, pair, inp["theta"], inp["xip"], inp["xim"], grid="reporting") s.add_covariance(np.eye(len(s.mean))) + s = sacc_io.seal(s, MOCK) with pytest.raises(ValueError, match="single-bin only"): sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) @@ -511,6 +516,7 @@ def test_sacc_without_xi_raises(tmp_path): window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), ) s.add_covariance(np.eye(len(s.mean))) + s = sacc_io.seal(s, MOCK) with pytest.raises(ValueError, match="nothing to convert"): sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) @@ -577,3 +583,12 @@ def _mask_idx(dtype, tracers): np.testing.assert_array_equal( hdul["COVMAT_CELL"].data, encoded[np.ix_(cell_idx, cell_idx)] ) + + +def test_an_unstamped_sacc_is_refused(tmp_path): + """Only a SACC born through the door (stamped) is converted.""" + s = _sacc(_inputs(seed=50)) + del s.metadata["custody"] + with pytest.raises(ValueError, match="custody stamp"): + sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) + assert not (tmp_path / "x.fits").exists() diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index 602e874a..c68ca5ac 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -175,6 +175,16 @@ def test_pseudo_cl_to_sacc_real_namaster(tmp_path): assert np.array_equal(ee, cl_all[0]) and np.array_equal(bb, cl_all[3]) # window columns correspond to the bandpowers, one per ell_eff assert window.weight.shape[1] == len(ell_eff) + # a map-based spectrum's window carries the pixel window pw²(ℓ) + import healpy as hp + + mapped = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell_eff, cl_all, wsp, nside=nside) + pw2 = hp.pixwin(nside, lmax=len(window.values) - 1) ** 2 + np.testing.assert_allclose( + mapped.get_bandpower_windows(mapped.indices(sio.CL_EE)).weight, + pw2[:, None] * window.weight, + rtol=1e-12, + ) def test_cosebis_to_sacc(tmp_path): diff --git a/src/sp_validation/theory.py b/src/sp_validation/theory.py index 38f5c15e..dee8bb4f 100644 --- a/src/sp_validation/theory.py +++ b/src/sp_validation/theory.py @@ -32,11 +32,7 @@ def fiducial(): - """cs_util's Planck 2018 point, with feedback, no IA and unit lens bias. - - ``b_lens`` is the linear bias of every γt lens sample: the γt shift scales - as ``b_lens`` over the sample's true bias. - """ + """cs_util's Planck 2018 point, with feedback and no IA.""" from cs_util.cosmo import PLANCK18 as p return { @@ -47,7 +43,6 @@ def fiducial(): }, "logT_AGN": 7.5, "A_IA": 0.0, - "b_lens": 1.0, } @@ -61,7 +56,7 @@ def _cosmology(point): from cs_util.cosmo import get_cosmo p = dict(zip(_COSMOLOGY, point)) - return get_cosmo( + cosmo = get_cosmo( Omega_m=p["Omega_m"], Omega_b=p["Omega_b"], h=p["h"], @@ -79,6 +74,11 @@ def _cosmology(point): } }, ) + # CCL's Hankel transform extrapolates C_ℓ to ELL_MAX_CORR; its default + # (6·10⁴) rings in ξ− below a few arcmin. + cosmo.cosmo.spline_params.ELL_MAX_CORR = 10_000_000 + cosmo.cosmo.spline_params.N_ELL_CORR = 5_000 + return cosmo def tag(s, rows, name): @@ -147,13 +147,17 @@ def shear_cl(params, s, rows): def gamma_t(params, s, rows): - """γt of a source bin around a lens sample of linear bias ``b_lens``.""" + """γt of a source bin around a lens sample of the linear bias its tracer + carries (``sacc_io.add_lens(..., bias=)``).""" import pyccl as ccl - cosmo = cosmology(params) source, lens = source_lens(s, rows) + b = (s.tracers[lens].metadata or {}).get("bias") + if b is None: + raise ValueError(f"lens tracer {lens} carries no linear bias") + cosmo = cosmology(params) z, n = nz(s, lens) - bias = (z, np.full_like(z, params["b_lens"])) + bias = (z, np.full_like(z, float(b))) counts = ccl.NumberCountsTracer(cosmo, has_rsd=False, dndz=(z, n), bias=bias) cl = ccl.angular_cl(cosmo, counts, _lensing(cosmo, params, s, source), ELL) theta = tag(s, rows, "theta") / 60.0 diff --git a/uv.lock b/uv.lock index 88df41c5..9c23efa4 100644 --- a/uv.lock +++ b/uv.lock @@ -1256,60 +1256,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, ] -[[package]] -name = "hypothesis" -version = "6.168.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sortedcontainers" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/10/5795cc413d2ba5ff9c81e5d1b27ba3c652ac6dfac6b1d67072052d338391/hypothesis-6.168.2.tar.gz", hash = "sha256:df61fcada928a3fb3fba4602be5b2d569eb50c36fddae0d2e2a5f51419d0ce9b", size = 510987, upload-time = "2026-09-27T05:22:19.959Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/79/989e20cebec9fef907c43e34dae9c0938b49e2d3bb8537a79990692471c0/hypothesis-6.168.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8268bcc733f2b93e77f64e54d6fa61db79af91806e2d26b950f5d1cca47b4bfa", size = 1124033, upload-time = "2026-09-27T05:20:17.522Z" }, - { url = "https://files.pythonhosted.org/packages/ec/44/3b5edc918128eccea902eb6ee91f0c58ad575274b99c565107fc575483a9/hypothesis-6.168.2-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c138d9848c210b8a1e511ab0884f16bc71470400b11255e8b595df320809afc", size = 1147868, upload-time = "2026-09-27T05:20:44.835Z" }, - { url = "https://files.pythonhosted.org/packages/2a/54/04d30544681626fb7fa25cc1041c12ebfad79aa4995438f6f3e577541111/hypothesis-6.168.2-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:420479d419d73dd7bb2ed0cfb57f35b1dc0bc246c30b250904c3987c5cae9463", size = 1149472, upload-time = "2026-09-27T05:21:49.503Z" }, - { url = "https://files.pythonhosted.org/packages/14/86/a7abe9660b4b997305537aa860fb932f97223125713655a7a1e9a302f629/hypothesis-6.168.2-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:01f74944abe6988a6213dd036bd2dbf1e4f0513af0cf66b7780773fdf0c6f5e2", size = 1191885, upload-time = "2026-09-27T05:22:15.685Z" }, - { url = "https://files.pythonhosted.org/packages/31/d0/24c77c25ec62941022edcbb98a052b98c57b60fee7d289cabea73523a727/hypothesis-6.168.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe84b8218a4fc33f79e64840d23fe9c627e7acb7cad1f70bf6c0f163c7f4f01f", size = 1169899, upload-time = "2026-09-27T05:20:13.379Z" }, - { url = "https://files.pythonhosted.org/packages/cb/23/dd129e570f44051de911761cada83db5695bbbe3db450bc905d449ea8aa2/hypothesis-6.168.2-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:ce74728a05d5e5dd4d28bb4fbaa0c1f5e809ebf6be54d343128d970ee1025697", size = 1129354, upload-time = "2026-09-27T05:20:10.768Z" }, - { url = "https://files.pythonhosted.org/packages/71/65/6d911e5b270c7f9560929b07304869a1f77e2440e22db107600788975bbb/hypothesis-6.168.2-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:29e12bcfcef363a9dbd0d3d62fd9608853268d15319ae3ecaf802555497392a9", size = 1160386, upload-time = "2026-09-27T05:22:05.475Z" }, - { url = "https://files.pythonhosted.org/packages/26/87/5cb26f7a2323ecdb4881f06c8eec19c4087e04b23cc7998b74f2b565a66c/hypothesis-6.168.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3cbe12b2427a86d769fdc93242ee3207d3a1523798b856630f16b674ebae17b1", size = 1299894, upload-time = "2026-09-27T05:20:02.55Z" }, - { url = "https://files.pythonhosted.org/packages/d7/f3/eb40e5c554b78b1b500b93172829b4b16c677acadd3f10f979117a82f851/hypothesis-6.168.2-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:3b62ce43973f4425530e590751439bacceec19921af0edf9fae3a63e173ea146", size = 1425524, upload-time = "2026-09-27T05:20:58.58Z" }, - { url = "https://files.pythonhosted.org/packages/2d/1e/03819636fa86ade2db91ad9953851b49f885a3481a22f0b1361d0c5dfc8d/hypothesis-6.168.2-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:51df0068106dd2db4fc5a1e56801adf5fc1d354792584289a9cc8ac09273d376", size = 1377070, upload-time = "2026-09-27T05:21:04.963Z" }, - { url = "https://files.pythonhosted.org/packages/5b/80/505146f225bbd24759651a16c17cd44aa6f082fa4a14761646e8ec37f374/hypothesis-6.168.2-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:e54638a0f66c608840af13f3ad665710e706bda5c52f706351bb83b5744bec63", size = 1281230, upload-time = "2026-09-27T05:20:04.007Z" }, - { url = "https://files.pythonhosted.org/packages/ff/7b/3c9f0f1cf0733b6d692dd5dca92037f6b707082bfb4c29712f2d0258049e/hypothesis-6.168.2-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:84c5b9891f979e851d140618783e9c3624352760dd78cba4759b263686cdad16", size = 1300413, upload-time = "2026-09-27T05:20:14.695Z" }, - { url = "https://files.pythonhosted.org/packages/81/dc/56c1626009545b86fefa71ee71d273fe518d1374beca40bc9b784818a58e/hypothesis-6.168.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6a6ac0836daba2da486a4fefc22fcdd794dac350bd787a1bb896748cdc47e05e", size = 1336267, upload-time = "2026-09-27T05:21:23.381Z" }, - { url = "https://files.pythonhosted.org/packages/1b/c4/853cc464739bfd076e0f6bac55c4fcff362effdcfc10873bd1a119e80994/hypothesis-6.168.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e28cb8912d7811dcec91b2389d9dcf3b34523c043cfb7454f9c62c8e5a40b2a", size = 1123030, upload-time = "2026-09-27T05:20:38.642Z" }, - { url = "https://files.pythonhosted.org/packages/9c/6a/506d3c0e45bf8cfb6bce2842b0642df37db07f2d5695f4614fa410e854cc/hypothesis-6.168.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3630d664a343354701618a409d3b9d563cb97cfcf749c7a37c5fb32cb3c7084c", size = 1169073, upload-time = "2026-09-27T05:20:55.361Z" }, - { url = "https://files.pythonhosted.org/packages/82/52/244eff24dbaa1f9520dcb42164fb3769e6c08a2e252455af2e8bd23305e5/hypothesis-6.168.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a0df395d27012b07eeed98b2e3f2a17f693236b0518d8a086679dc657801aead", size = 1298741, upload-time = "2026-09-27T05:20:32.609Z" }, - { url = "https://files.pythonhosted.org/packages/2e/99/cdcb4fa1b738ef07827464bdf5ff83897f8303e8eca569207cc52f5be2b2/hypothesis-6.168.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:778f043b8af618baabe79cdb15f6f7855ccf950c6911eea84acabca5b7637cde", size = 1335252, upload-time = "2026-09-27T05:20:09.229Z" }, - { url = "https://files.pythonhosted.org/packages/a5/49/63f0a6b5b709e2bdd0d1d0041c48c6d4614323e27c677a0d44d9748dd4cb/hypothesis-6.168.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97437cf297feb9645c23fd187734c137989c503303542a4034eb43d07f8f151e", size = 1123008, upload-time = "2026-09-27T05:20:01.27Z" }, - { url = "https://files.pythonhosted.org/packages/a5/21/284046912c05973f91d41bd7ed79e3bf1917b1332b70e677c3cf2872cea8/hypothesis-6.168.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5bd1b6334eaccde40ab56ffb96788f2af993869b08aebc4f1b3ff28f3edcda9", size = 1168965, upload-time = "2026-09-27T05:20:18.818Z" }, - { url = "https://files.pythonhosted.org/packages/6a/b7/afe659dccd780393ca6beceabc3a418f11ff2539a83bc4ce9e6c8a0ab589/hypothesis-6.168.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac134d9fe4a1623b57b2a1681e3f51c2c5b324e690c1264dc01a3c57c445b08a", size = 1298607, upload-time = "2026-09-27T05:20:21.424Z" }, - { url = "https://files.pythonhosted.org/packages/e3/e0/d6300216c75fe878da9bc8bfc5e266baa1d350ea145001272d20b3ff3843/hypothesis-6.168.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:03ba2a0767955fd6491cb93e6917db1d5b718a89f9a4de33b0827b7fce1df289", size = 1335166, upload-time = "2026-09-27T05:20:53.745Z" }, - { url = "https://files.pythonhosted.org/packages/5e/2a/11e427ed3f09f0e1d548428144e97fb9bfbdb2f41b6871880625acf41411/hypothesis-6.168.2-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db13338672ba689203f405c6258ef62b0590e3d193714b30809e53e8bfc66e84", size = 1123318, upload-time = "2026-09-27T05:21:30.573Z" }, - { url = "https://files.pythonhosted.org/packages/89/03/80911aca76e411cb1d1e7a5deaea0a49b47321ccd85092ceba971100b921/hypothesis-6.168.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:170a6d2d6d9d209992db1e671d0c2ad17367c8046482ebff2cbc9e14a9698be7", size = 1169146, upload-time = "2026-09-27T05:20:20.159Z" }, - { url = "https://files.pythonhosted.org/packages/b3/19/243f18213e39924db4be21b53e3999c183d12954745254e717a639fc3490/hypothesis-6.168.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9310d3f997f3d5e8e31d5c5f9946bcd3ccf6eb62c92a1b728f2dd57fd62fda05", size = 1299106, upload-time = "2026-09-27T05:21:06.885Z" }, - { url = "https://files.pythonhosted.org/packages/6a/47/abc5881cf0cdce97663b877f486ff064d834850ea19171ba556c7dd835ae/hypothesis-6.168.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d66e68a0dc9e216f9bdda2c9f066c09e36849cbcb49fbcb1860087560709bb8d", size = 1335374, upload-time = "2026-09-27T05:20:46.633Z" }, - { url = "https://files.pythonhosted.org/packages/d1/f9/08d4bba84eab0404b42971af4ee21c4f9f2064444dc7c8040a14f722ffea/hypothesis-6.168.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f98f7581f68acc659510f8786c39c5cf140dbc3dacba6cd0ed059b0e296a158b", size = 624273, upload-time = "2026-09-27T05:20:16.14Z" }, - { url = "https://files.pythonhosted.org/packages/bb/e0/92fc641a751af530859023ea6ae6f73c2294587240b78884aa155fccbe5a/hypothesis-6.168.2-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:798ae9a91ab4cd8cfadd7991f46dc9c12fd43d2997f29744b7eae755ff36027a", size = 1121651, upload-time = "2026-09-27T05:21:40.104Z" }, - { url = "https://files.pythonhosted.org/packages/a9/1a/405c4c9b888721fc3b650ffc3c86d8507450e1ea105e88c97e2e5f277aa6/hypothesis-6.168.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00f8728aaef10c91688dab34df8a59aed83c76ade869b1812f01ac80808bc5cc", size = 1167748, upload-time = "2026-09-27T05:21:41.918Z" }, - { url = "https://files.pythonhosted.org/packages/b1/70/2f7559b159f98e14d950914a0aeb25b0e2ed473f35705192de4a2c20b3e8/hypothesis-6.168.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:63b183d5436688e54389f54b66d1611aef349f36345b6dc0a79ccef0dbc736c8", size = 1297293, upload-time = "2026-09-27T05:21:03.359Z" }, - { url = "https://files.pythonhosted.org/packages/f5/c3/b197f8353c3026b76da82d49a119849cd9eac7b0be96239414d1d748156c/hypothesis-6.168.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:28dfe52bcf6e6621153cfbb5574223d117a3e66b24ffeacb4c9bc8f751190dc6", size = 1334246, upload-time = "2026-09-27T05:21:27.001Z" }, - { url = "https://files.pythonhosted.org/packages/63/98/55f83a1a9cb643866c27211a991cf56a012126a6d5548a02877df1334ab0/hypothesis-6.168.2-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3001d9e6013ebb82d9d7daa97efd75a383e628bf2b5d754512b56191898e377", size = 1121134, upload-time = "2026-09-27T05:21:32.576Z" }, - { url = "https://files.pythonhosted.org/packages/87/dd/2a642db35341893ee604d8cd0b8e6232ddc848e9fe04d6a17fd30cdbc586/hypothesis-6.168.2-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:44b7c3d431cca39342881761dfe3e34b39c9b5e7b0ad5c8c4cb7d92514f3b94c", size = 1144046, upload-time = "2026-09-27T05:20:06.616Z" }, - { url = "https://files.pythonhosted.org/packages/61/13/78e7b3192d4ccd19bbf36ebc0a678a8291f40093baee47fdded4240d2ca7/hypothesis-6.168.2-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e78ce758211e2fd049c358c979c0c1312fc0f6e13a829cf2a4828a9ebb1423f8", size = 1146678, upload-time = "2026-09-27T05:21:34.447Z" }, - { url = "https://files.pythonhosted.org/packages/e4/6a/daa33a3e9eb999da96bd7315d1f5446c5e28866191e2007aad9b1fb6616f/hypothesis-6.168.2-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9aec441b5d7156e64285817fda72b0aad37aa3dd262693708e825858930b7b48", size = 1189433, upload-time = "2026-09-27T05:21:55.422Z" }, - { url = "https://files.pythonhosted.org/packages/28/49/138fa2c61ec12ea7b17a1693a17ce0567ede4e61dac65d3304664493641b/hypothesis-6.168.2-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3510ca51dfd5aa7eb24f39d65714f6a91297fa5382866124b931bf56acd207fd", size = 1167045, upload-time = "2026-09-27T05:22:13.561Z" }, - { url = "https://files.pythonhosted.org/packages/e5/04/38474d9c6bc119700b6eb6fdb73ff8775202af79d5a7ee3bd06619d1b5a0/hypothesis-6.168.2-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:8d7943a4c8dcca314bfd918247acea4a36fd3476167372b19d8db3070cdefa4f", size = 1126814, upload-time = "2026-09-27T05:21:43.835Z" }, - { url = "https://files.pythonhosted.org/packages/99/be/048888e2f1bfe412c46788443079bb03b6c299c0da85f11fdb536f27dfa4/hypothesis-6.168.2-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:903cc76f9ebd36d555039aac270f5652cf0a93cfbbac344915ee3529e917efa3", size = 1155839, upload-time = "2026-09-27T05:20:22.76Z" }, - { url = "https://files.pythonhosted.org/packages/3a/67/0a607ac12fedbd8b2be26b360013df3b064e6b80d7313db635bfb5cc75de/hypothesis-6.168.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:d6f060f4bcf10ac087c34da08fc9f3be979dcfb70d45a238a6fe6298f1cc1ae9", size = 1296640, upload-time = "2026-09-27T05:21:28.848Z" }, - { url = "https://files.pythonhosted.org/packages/d0/f8/f59636b8b9b8c6de5f02b752faba3c7592a13dd767ddc96bf582b8870720/hypothesis-6.168.2-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:8574573eea6693792966eb6c8da079fe63d8dafb884c613c9857b3a13052df45", size = 1422025, upload-time = "2026-09-27T05:20:07.72Z" }, - { url = "https://files.pythonhosted.org/packages/a0/f4/e36c4a95b7997e213cd85c3be1232c8ac19e1ecf20d99512ea279eade321/hypothesis-6.168.2-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:5714ac1e72ba5faa645f593de353af34a8b1de8bcff811f1d3da4c1b8628e1c3", size = 1374166, upload-time = "2026-09-27T05:21:01.762Z" }, - { url = "https://files.pythonhosted.org/packages/18/9e/b9c3e282e9833e75875bc9163f80c49f8153d0f96ce5ba034a7997f13850/hypothesis-6.168.2-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:dffa4c97f0b473f1da456807ff57b348fb675f7728a15faab4eea87dffb6c77a", size = 1278387, upload-time = "2026-09-27T05:20:05.125Z" }, - { url = "https://files.pythonhosted.org/packages/af/96/84ed26fd0855ad8d6d67413893c835e90a77294c1255e3e78c13fe082f0f/hypothesis-6.168.2-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:29c1d761ef45e063eba2f8287e0572976f6b758b6832d75d7e03d02a610d3a91", size = 1297784, upload-time = "2026-09-27T05:20:28.101Z" }, - { url = "https://files.pythonhosted.org/packages/94/19/c94f1bce7e72950f8f3c5ee9d34ba98a7da70548453a96befa86281bfd94/hypothesis-6.168.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c7ab9ecf409143ef26a0d20a5f373d2add8871be58028cdbd3a515b0ef43ac45", size = 1333959, upload-time = "2026-09-27T05:21:16.771Z" }, -] - [[package]] name = "idna" version = "3.18" @@ -3583,15 +3529,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl", hash = "sha256:7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752", size = 104164, upload-time = "2026-06-03T00:56:38.614Z" }, ] -[[package]] -name = "sortedcontainers" -version = "2.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, -] - [[package]] name = "soupsieve" version = "2.8.4" @@ -3650,7 +3587,6 @@ dependencies = [ [package.optional-dependencies] develop = [ - { name = "hypothesis" }, { name = "myst-parser" }, { name = "numpydoc" }, { name = "pytest" }, @@ -3674,7 +3610,6 @@ glass = [ { name = "glass-ext-camb" }, ] test = [ - { name = "hypothesis" }, { name = "pytest" }, { name = "pytest-cov" }, { name = "ruff" }, @@ -3705,7 +3640,6 @@ requires-dist = [ { name = "h5py" }, { name = "healpy" }, { name = "healsparse" }, - { name = "hypothesis", marker = "extra == 'test'" }, { name = "importlib-metadata" }, { name = "joblib", specifier = ">=0.13" }, { name = "jupyter" }, diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 428abe65..0aad6726 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -32,6 +32,34 @@ def grids(toy): return toy.common.xi_grids(toy.config, toy.config["fiducial"]) +def test_assemble_resolves(toy): + """Each terminal file gathers every part and the analytic covariances. + + The ξ± block takes the CosmoCov covariance on the reporting grid, and the + harmonic block is the part on the fiducial harmonic binning with the + NaMaster covariance of that same binning. + """ + result = toy.snakemake("-n", "assemble_sacc_all") + assert result.returncode == 0, result.stdout + jobs = [j for j in parse_jobs(result.stdout) if j.rule == "assemble_sacc"] + grids = toy.common.xi_grids(toy.config, toy.config["fiducial"]) + reporting = toy.common.grid_binning(grids["reporting"]) + harmonic = toy.common.pseudo_cl_tag(toy.config) + assert sorted(j.wildcards["version"] for j in jobs) == sorted(VERSIONS) + for job in jobs: + version = job.wildcards["version"] + assert [Path(o).name for o in job.output] == [f"{version}.sacc"] + assert {Path(f).name for f in job.input} == { + f"{version}_xi_{reporting}.sacc", + toy.covariances[version, "ng"].name, + f"pseudo_cl_{version}_{harmonic}.sacc", + f"pseudo_cl_cov_{version}_{harmonic}.fits", + f"{version}_cosebis.sacc", + f"{version}_pure_eb.sacc", + f"rho_tau_{version}_{reporting}.sacc", + }, job.input + + def test_xi_leaves_a_measurement_only_as_a_part(toy, forced, grids): """No job reads or writes a ξ± text dump; the ξ± figures draw the parts.""" jobs = forced[1] @@ -137,14 +165,22 @@ def test_a_launch_the_dag_cannot_honour_stops_with_the_fix(toy, case): assert "rule assemble_sacc" not in result.stdout -def test_outputs_stay_in_the_output_roots(toy, forced): - """Nothing the suite declares lands outside the configured output roots.""" +@pytest.mark.parametrize("named", [True, False], ids=["named", "unnamed"]) +def test_outputs_stay_in_the_output_roots(toy, named): + """Nothing the suite declares lands outside the configured output roots. + + A launch that names no COSMO_VAL writes into its own checkout's + cosmo_val/output. + """ + env = toy.env if named else {k: v for k, v in toy.env.items() if k != "COSMO_VAL"} + cosmo_val = toy.cosmo_val if named else toy.root / "cosmo_val" / "output" + result = toy.snakemake("-n", "all", env=env) + assert result.returncode == 0, result.stdout roots = [ - r.resolve() - for r in (toy.cosmo_val, toy.cosmo_inference, toy.rundir / "results") + r.resolve() for r in (cosmo_val, toy.cosmo_inference, toy.rundir / "results") ] - outputs = [Path(o) for j in forced[1] for o in j.output] - assert outputs + outputs = [Path(o) for j in parse_jobs(result.stdout) for o in j.output] + assert outputs, result.stdout strays = [ o for o in outputs diff --git a/workflow/tests/test_toy_run.py b/workflow/tests/test_toy_run.py index a00a0db9..76249dd2 100644 --- a/workflow/tests/test_toy_run.py +++ b/workflow/tests/test_toy_run.py @@ -49,7 +49,7 @@ def _stamps(image, root, parts): script = ( "import json, sys\n" "from sp_validation import custody, sacc_io\n" - "print(json.dumps([custody.read_stamp(sacc_io.load(p).metadata).stamp" + "print(json.dumps([custody.read_stamp(sacc_io.load(p).metadata).token" " for p in sys.argv[1:]]))" ) return json.loads(_in_image(image, root, "python", "-c", script, *map(str, parts))) @@ -143,8 +143,8 @@ def test_xi_parts_are_stamped_with_the_blinds_commitment(): assert result.returncode == 0, result.stdout record = json.loads((root / "cosmo_val/blinds/toy.blind.json").read_text()) - stamp = {"blind": "toy", "blind_commitment": custody.commitment(record)} + token = f"toy:{custody.commitment(record)}" parts = [out / f"{v}_xi_{BINNING}.sacc" for v in VERSIONS] - assert _stamps(image, root, parts) == [stamp] * len(parts) + assert _stamps(image, root, parts) == [token] * len(parts) shutil.rmtree(root) # kept on failure, for post-mortem From c62e3b3e6ac7858850adffdb7feb5541dc56956a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 11:11:38 +0200 Subject: [PATCH 142/160] =?UTF-8?q?Pseudo-C=E2=84=93=20windows=20as=20on?= =?UTF-8?q?=20develop;=20the=20pixel=20window=20is=20fix/pseudo-cl-pixel-w?= =?UTF-8?q?indow's?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- src/sp_validation/cosmo_val/pseudo_cl.py | 16 +++++----------- src/sp_validation/cosmo_val/sacc_writers.py | 9 ++++----- src/sp_validation/pseudo_cl.py | 9 ++------- src/sp_validation/tests/test_sacc_writers.py | 10 ---------- 4 files changed, 11 insertions(+), 33 deletions(-) diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 4c613770..c3593b71 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -600,9 +600,7 @@ def calculate_pseudo_cl_map(self, ver, nside, out_path): cl_shear = cl_shear - cl_noise self.print_cyan("Saving pseudo-Cl's...") - sealed = self.pseudo_cl_to_sacc_part( - ver, out_path, ell_eff, cl_shear, wsp, nside=nside - ) + sealed = self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) self._pseudo_cls[ver]["pseudo_cl"] = _spectra(sealed) def calculate_pseudo_cl_catalog(self, ver, out_path): @@ -706,16 +704,13 @@ def apply_random_rotation(self, e1, e2, rng=None): """ return apply_random_rotation(e1, e2, rng) - def pseudo_cl_to_sacc_part( - self, version, out_path, ell_eff, cl_all, wsp, nside=None - ): + def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): """Write the pseudo-Cl SACC part (EE/BB/EB + shared bandpower window). ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); - the writer takes the shared bandpower window from ``wsp``, with the - pixel window of maps at ``nside`` for a map-based spectrum. No - covariance is attached here. Returns the part as sealed under the - version's custody. + the writer takes the shared bandpower window from ``wsp``. No covariance + is attached here. Returns the part as sealed under the version's + custody. """ s = pseudo_cl_to_sacc( self.sacc_nz(version), @@ -723,7 +718,6 @@ def pseudo_cl_to_sacc_part( ell_eff, cl_all, wsp, - nside=nside, ) return sacc_io.save(s, out_path, custody=self.custody(version)) diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index 8dcdc9c2..aef72b33 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -71,15 +71,14 @@ def xi_to_sacc( return s -def pseudo_cl_to_sacc(nz, metadata, ell_eff, cl_all, wsp, covariance=None, nside=None): +def pseudo_cl_to_sacc(nz, metadata, ell_eff, cl_all, wsp, covariance=None): """One pseudo-Cℓ part: EE/BB/EB with the shared bandpower window. ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); the - window comes from :func:`bandpower_window_from_workspace`, with the pixel - window of maps at ``nside`` when given. ``covariance``, when given, is the - dense ``[EE; BB; EB]``-ordered block matching insertion. + window comes from :func:`bandpower_window_from_workspace`. ``covariance``, + when given, is the dense ``[EE; BB; EB]``-ordered block matching insertion. """ - window_ells, window_weights = bandpower_window_from_workspace(wsp, nside) + window_ells, window_weights = bandpower_window_from_workspace(wsp) s = sio.new_sacc(nz, metadata) sio.add_pseudo_cl( s, diff --git a/src/sp_validation/pseudo_cl.py b/src/sp_validation/pseudo_cl.py index bdcc8d67..34cbc68d 100644 --- a/src/sp_validation/pseudo_cl.py +++ b/src/sp_validation/pseudo_cl.py @@ -286,7 +286,7 @@ def get_pseudo_cls_catalog( _NMT_EE = 0 -def bandpower_window_from_workspace(wsp, nside=None): +def bandpower_window_from_workspace(wsp): """Extract the bandpower window matrix ``W`` for a spin-2×spin-2 workspace. NaMaster's ``get_bandpower_windows()`` returns a four-index array @@ -303,14 +303,9 @@ def bandpower_window_from_workspace(wsp, nside=None): ``compute_coupled_cell``. window_weights : np.ndarray ``W`` of shape ``(n_ell, n_bpw)`` — one column per bandpower, the layout - :func:`sp_validation.sacc_io.add_pseudo_cl` expects. For a spectrum of - HEALPix maps at ``nside``, ``W`` includes the pixel window pw²(ℓ), so it - maps a C_ℓ to the measured bandpower. + :func:`sp_validation.sacc_io.add_pseudo_cl` expects. """ bpw = wsp.get_bandpower_windows() # (n_cl_out, n_bpw, n_cl_in, n_ell) diagonal = bpw[_NMT_EE, :, _NMT_EE, :] # (n_bpw, n_ell) window_ells = np.arange(diagonal.shape[1], dtype=float) - if nside is not None: - pixwin = hp.pixwin(nside, lmax=diagonal.shape[1] - 1) - diagonal = diagonal * pixwin**2 return window_ells, diagonal.T diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index c68ca5ac..602e874a 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -175,16 +175,6 @@ def test_pseudo_cl_to_sacc_real_namaster(tmp_path): assert np.array_equal(ee, cl_all[0]) and np.array_equal(bb, cl_all[3]) # window columns correspond to the bandpowers, one per ell_eff assert window.weight.shape[1] == len(ell_eff) - # a map-based spectrum's window carries the pixel window pw²(ℓ) - import healpy as hp - - mapped = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell_eff, cl_all, wsp, nside=nside) - pw2 = hp.pixwin(nside, lmax=len(window.values) - 1) ** 2 - np.testing.assert_allclose( - mapped.get_bandpower_windows(mapped.indices(sio.CL_EE)).weight, - pw2[:, None] * window.weight, - rtol=1e-12, - ) def test_cosebis_to_sacc(tmp_path): From d2bfc73e5c19b07994f655495fefcfdcbfe2a562 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 11:20:23 +0200 Subject: [PATCH 143/160] rho/tau script, workflow README and docstrings as on develop but for custody The scaffolding removal, the private COSMO_INFERENCE paragraph and the non-custody docstrings are chore/cosmo-val-cleanup's; run_rho_tau passes its rule's custody token and nothing else changes. Co-Authored-By: Claude Opus 5.5 --- .../cosmo_val/psf_systematics.py | 1 - src/sp_validation/cosmo_val/sacc_writers.py | 3 -- workflow/README.md | 10 +--- workflow/scripts/run_rho_tau.py | 52 +++++++++++++++---- 4 files changed, 44 insertions(+), 22 deletions(-) diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index 5f6b8cbd..a4828189 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -26,7 +26,6 @@ class PSFSystematicsMixin: def calculate_rho_tau_stats(self): - """Measure ρ/τ statistics per version and write each version's SACC part.""" out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" if not os.path.exists(out_dir): os.mkdir(out_dir) diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index aef72b33..8b7f57b7 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -7,9 +7,6 @@ own covariance as its one block. :func:`assemble_analysis_sacc` rebuilds the single ``{version}.sacc`` analysis file from these parts. -The integration-grid ξ± part is an intermediate consumed by COSEBIs and -pure-E/B; it does not join ``{version}.sacc``. - Everything here is single-bin today (``bins=(0, 0)``); the interface is tomography-native so a future round supplies real bin pairs unchanged. """ diff --git a/workflow/README.md b/workflow/README.md index d2e344eb..6979c540 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -126,15 +126,7 @@ halves of one commit, split. The catalogue config is the launched checkout's `cosmo_val/cat_config.yaml`. `COSMO_VAL` defaults to the launched checkout's `cosmo_val/output`, so writing into another checkout's products means naming it; `COSMO_INFERENCE` defaults to -the shared candide tree, which holds the CosmoCov covariances and only its owner -can write. Anyone else launches with `COSMO_INFERENCE=` of their own, -holding a link to the shared tree's `data/mask/` (the one input the covariance -rules take from it); the CosmoCov chain then runs there: - -```bash -mkdir -p /data -ln -s /n17data/cdaley/unions/code/sp_validation/cosmo_inference/data/mask /data/ -``` +the shared candide tree. **Caveat:** `rerun-triggers: code` watches rule bodies and `script:` files, not `src/`. Editing a module under `src/` does not by itself mark outputs stale — diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index 26f0b349..4334a742 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -1,16 +1,40 @@ -"""Rule rho_tau_stats: ρ/τ PSF statistics for one version. +# %% +# if interactive +import os +import sys +from pathlib import Path -Writes the ρ/τ FITS tables and the version's ρ/τ SACC part, sealed under the -catalogue's custody. -""" +from IPython import get_ipython -from cv_runner import _unbuffer_streams, verify_outputs +ipython = get_ipython() -from sp_validation.cosmo_val import CosmologyValidation +# enable autoreload for interactive sessions +if ipython is not None: + ipython.run_line_magic("load_ext", "autoreload") + ipython.run_line_magic("autoreload", "2") +else: + # Force unbuffered stdout and stderr + sys.stdout = os.fdopen(sys.stdout.fileno(), "w", buffering=1) # line-buffered + sys.stderr = os.fdopen(sys.stderr.fileno(), "w", buffering=1) -_unbuffer_streams() -params = snakemake.params # noqa: F821 — injected by Snakemake's script: directive +from sp_validation.cosmo_val import CosmologyValidation # noqa: E402 +print("Finished imports") + +if ipython is not None: + from snakemake_helpers import snakemake_interactive + + snakemake = snakemake_interactive( + "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/output/rho_tau_stats/rho_stats_SP_v1.4.5.fits", + "/home/cdaley/n17data/unions/pure_eb", + ) + +params = snakemake.params # type: ignore + +# %% +print("Starting CosmologyValidation") + +# Use parameters passed from Snakemake rule cv = CosmologyValidation( versions=[params["ver"]], theta_min=float(params["min_sep"]), @@ -21,5 +45,15 @@ output_dir=params["output_dir"], custody={params["ver"]: params["custody"]}, ) + cv.calculate_rho_tau_stats() -verify_outputs(snakemake) # noqa: F821 + +# Confirm CosmologyValidation produced the requested outputs: the rho/tau FITS +# and the born-as-SACC rho_tau part. +outputs = snakemake.output # type: ignore +for label in ("rho_stats", "tau_stats", "rho_tau"): + target = Path(outputs[label]) + if not target.exists(): + raise FileNotFoundError( + f"Expected {label} file not found after CosmologyValidation run: {target}" + ) From ea7e615a35ae28f1844954547d455245ee61184a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 11:55:49 +0200 Subject: [PATCH 144/160] pure-E/B non-additivity: state the measured range, not a cause Co-Authored-By: Claude Opus 5.5 --- src/sp_validation/tests/test_blinding_bmodes.py | 4 ++-- workflow/README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sp_validation/tests/test_blinding_bmodes.py b/src/sp_validation/tests/test_blinding_bmodes.py index 6c7a5c70..89623763 100644 --- a/src/sp_validation/tests/test_blinding_bmodes.py +++ b/src/sp_validation/tests/test_blinding_bmodes.py @@ -171,8 +171,8 @@ def test_cosebis_b_modes_move_only_by_the_transforms_response(shifts, cosebis, n marks=pytest.mark.xfail( strict=True, reason="pure-eb-bmode-numerics: on a noisy 1000-bin ξ± " - "cosmo_numba's adaptive quadrature is not additive: an E-mode δ " - "moves ξ_B by up to 0.4σ at the envelope's corners, unchanged by " + "the pure-E/B transform is not additive: an E-mode δ moves ξ_B by " + "0.1-1.2σ in isolated bins across noise draws and corners, unchanged by " "tightening epsabs/epsrel from 1e-10 to 1e-13, so blinded and " "unblinded pure-E/B alike carry this error", ), diff --git a/workflow/README.md b/workflow/README.md index 6979c540..cf30d9c1 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -38,8 +38,8 @@ Under a blind, ξ±, pseudo-Cℓ_EE and γt are shifted by t(hidden) − t(fiduc before they are first written, t being the data type's theory in `sp_validation.theory.THEORY`; statistics derived from them carry the blind, and a data type with no blinding rule in `sacc_io` is refused. COSEBIs B_n move only -by the transform's response to an E-mode shift; pure-E/B ξ_B also moves where -cosmo_numba's quadrature does not converge on noisy ξ± (`test_blinding_bmodes`). +by the transform's response to an E-mode shift; pure-E/B ξ_B is not additive on noisy ξ± +and also moves in isolated bins (`test_blinding_bmodes`). Rule params carry the custody token, so a flip reruns what it touches. A blind is one record, `/.blind.json`, outside any git From 41bc44b07bfbe6d55b24974d465236933da3859f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 17:07:36 +0200 Subject: [PATCH 145/160] Blinding: one table of standard estimators; any other type by a theory you pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit blinding.STANDARD declares each estimator cosmo_val computes and its rule once: ξ± and Cℓ_EE shifted by the default PyCCL theory, Cℓ_BB/EB unshifted, COSEBIs and pure-E/B derived from ξ±, ρ/τ signal-free. Any other data type is shifted when its birth passes a function, sacc_io.save(s, path, custody=c, theory={data_type: f}), and is refused under a blind without one; its sealed rows pass later derivations as copies. γt leaves until cosmo_val calculates it: GAMMA_T/GAMMA_X, add_lens, add_gamma_t, the lens bias, source_lens and the null-lens exemption go, with their tests. The secrecy test goes; Blind still holds only a name and path. The README and CLAUDE.md state the calculate-then-save rule. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GJJyGbKSuEjjX9m2zdv6yi --- CLAUDE.md | 2 +- src/sp_validation/blinding.py | 105 +++++++++--- src/sp_validation/sacc_io.py | 162 ++++++------------ src/sp_validation/tests/_synthetic.py | 11 +- src/sp_validation/tests/conftest.py | 8 +- src/sp_validation/tests/test_blinding.py | 157 +++++------------ .../tests/test_blinding_bmodes.py | 2 +- src/sp_validation/theory.py | 54 +----- workflow/README.md | 28 +-- 9 files changed, 221 insertions(+), 308 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 57dad8db..2ee79569 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,7 +82,7 @@ Main configuration in `scripts/calibration/params.py` with parameters: ## Blinded catalogues Each `cosmo_val/cat_config.yaml` entry declares `blind: none`, `mock` or a blind's name. -- For an entry with `blind: `, measure signal (ξ±, Cℓ, γt, COSEBIs, M_ap, maps) only through `CosmologyValidation` or the workflow, never from the file directly. +- For an entry with `blind: `, signal (ξ±, Cℓ, COSEBIs, M_ap, maps, any cosmological statistic) leaves the function that measures it only sealed: compute it and `sacc_io.save(..., custody=)` it in that function and return the sealed part, as `CosmologyValidation` and the workflow do. Never measure it from the file and keep the raw values. - Never set `blind: none` to get a run through. - Never print, paste or commit a `.blind.json`. diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 860baf20..fac14e40 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -1,13 +1,19 @@ """The blind: a secret seed, drawn once, and the shift it conceals signal by. +@sc standard-estimators +:data:`STANDARD` names each estimator cosmo_val computes and its blinding +rule, once. Any other data type is shifted when its birth passes a theory +function, ``sacc_io.save(s, path, custody=c, theory={data_type: f})``, with +``f(params, s, rows) -> values`` as in :mod:`sp_validation.theory`; without +one it is refused under a blind. + @sc blind-record A blind is one read-only file, ``/.blind.json``, outside any git worktree: its seed (unencrypted: registry access is blind access), the envelope the hidden point is drawn in, the fiducial and the fork's draw scheme. -Nothing here prints the seed or the hidden cosmology: :class:`Blind` holds a -name and a path, :func:`_hidden` returns a mapping whose repr is -````, and a theory failure at the hidden point is reported -by exception type alone. +:class:`Blind` holds a name and a path, never a hidden value; :func:`_hidden` +returns a mapping whose repr is ````, and a theory failure at +the hidden point is reported by exception type alone. @sc hidden-draw-uniform-s8-om The hidden point is the fiducial (:func:`sp_validation.theory.fiducial`) moved @@ -28,16 +34,79 @@ from collections.abc import Mapping from contextlib import redirect_stderr, redirect_stdout from pathlib import Path +from types import MappingProxyType import numpy as np from . import custody as _custody -from . import sacc_io +from . import sacc_io as sio from . import theory as _theory ENVELOPE = {"S8": 0.075, "Omega_m": 0.1} SECRET = "secret: never print, paste or commit" +# --------------------------------------------------------------------------- # +# The standard estimators cosmo_val computes, and how a blind treats each +# --------------------------------------------------------------------------- # +UNSHIFTED = "unshifted" # signal a pure E-mode shift leaves unchanged +DERIVED = "derived" # computed from ξ±, so shifted through the rows it reads +SIGNAL_FREE = "signal-free" # PSF diagnostics, no cosmological signal + +STANDARD = MappingProxyType( + { + # Shifted by t(hidden) − t(fiducial), t the default PyCCL theory. + sio.XI_PLUS: _theory.shear_xi, + sio.XI_MINUS: _theory.shear_xi, + sio.CL_EE: _theory.shear_cl, + sio.CL_BB: UNSHIFTED, + sio.CL_EB: UNSHIFTED, + sio.COSEBI_EE: DERIVED, + sio.COSEBI_BB: DERIVED, + **dict.fromkeys(sio.PURE_TYPES.values(), DERIVED), + **{ + t.format(k=k): SIGNAL_FREE + for ts, ks in ( + ((sio.RHO_PLUS, sio.RHO_MINUS), range(6)), + ((sio.TAU_PLUS, sio.TAU_MINUS), (0, 2, 5)), + ) + for t in ts + for k in ks + }, + } +) + + +def rule(data_type, theory=None): + """``data_type``'s rule: its theory function if a blind shifts it, else + :data:`UNSHIFTED`, :data:`DERIVED`, :data:`SIGNAL_FREE`, or None (refused + under a blind). ``theory`` gives functions for custom types and may + replace a standard one's.""" + standard = STANDARD.get(data_type) + if theory and data_type in theory: + if standard is not None and not callable(standard): + raise ValueError(f"{data_type} is {standard} under a blind; no theory") + return theory[data_type] + return standard + + +def shiftable(s, theory=None): + """Indices of the rows of ``s`` a blind shifts.""" + rules = [rule(dp.data_type, theory) for dp in s.data] + return np.array([i for i, r in enumerate(rules) if callable(r)], dtype=int) + + +def refuse_unruled(s, theory=None): + """Refuse rows of a data type with no blinding rule.""" + unruled = sorted( + {dp.data_type for dp in s.data if rule(dp.data_type, theory) is None} + ) + if unruled: + raise ValueError( + f"{unruled}: no blinding rule for these data types, so a blinded " + "catalogue's SACC cannot hold them; pass each a theory function, " + "save(..., theory={data_type: f})" + ) + class BlindingError(RuntimeError): """Concealment failed; the message carries no hidden value.""" @@ -120,7 +189,7 @@ def evaluate(): quiet = io.StringIO() with warnings.catch_warnings(), redirect_stdout(quiet), redirect_stderr(quiet): warnings.simplefilter("ignore") - return _theory.predict(s, _hidden(blind), rows, theory) + return _theory.predict(s, _hidden(blind), theory, rows) try: return evaluate() @@ -132,24 +201,18 @@ def evaluate(): def conceal(s, blind, theory=None): - """A copy of ``s`` with its shiftable rows moved by the blind's shift. + """A copy of ``s`` with its :func:`shiftable` rows moved by the blind's shift. - The shift is t(hidden) − t(fiducial), from ``theory`` (default - :data:`sp_validation.theory.THEORY`), evaluated at the fiducial first. A - shiftable type without a theory, and a data type and tracer pair the blind - leaves unmoved or moves to a non-finite value, are refused. + The shift is t(hidden) − t(fiducial), t each row's theory (:data:`STANDARD`, + or ``theory`` for a custom type), evaluated at the fiducial first. A data + type and tracer pair the blind leaves unmoved or moves to a non-finite + value is refused. """ - theory = _theory.THEORY if theory is None else theory - rows = sacc_io.shiftable(s) - types = {s.data[i].data_type for i in rows} - if types - set(theory): - raise BlindingError( - f"no theory for the shiftable {sorted(types - set(theory))}; " - "give it a function in sp_validation.theory.THEORY" - ) + rows = shiftable(s, theory) + functions = {s.data[i].data_type: rule(s.data[i].data_type, theory) for i in rows} fiducial = json.loads(blind.path.read_text())["fiducial"] - at_fiducial = _theory.predict(s, fiducial, rows, theory) - delta = _at_hidden(s, rows, theory, blind) - at_fiducial + at_fiducial = _theory.predict(s, fiducial, functions, rows) + delta = _at_hidden(s, rows, functions, blind) - at_fiducial groups = {} for n, i in enumerate(rows): groups.setdefault((s.data[i].data_type, s.data[i].tracers), []).append(n) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index d59f00e6..6ec8694d 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -87,7 +87,6 @@ import functools import os -import re import types import numpy as np @@ -107,8 +106,6 @@ CL_EB = "galaxy_shear_cl_eb" COSEBI_EE = "galaxy_shear_cosebi_ee" COSEBI_BB = "galaxy_shear_cosebi_bb" -GAMMA_T = "galaxy_shearDensity_xi_t" -GAMMA_X = "galaxy_shearDensity_xi_x" # Custom data-type strings (all parse under sacc.parse_data_type_name). PURE_TYPES = { @@ -162,13 +159,7 @@ def new_sacc(nz, metadata=None): items = nz.items() if isinstance(nz, dict) else enumerate(nz) s = sacc.Sacc() for i, (z, nz_i) in items: - s.add_tracer( - "NZ", - source_name(i), - np.asarray(z), - np.asarray(nz_i), - quantity="galaxy_shear", - ) + s.add_tracer("NZ", source_name(i), np.asarray(z), np.asarray(nz_i)) # The PSF star sample sits alongside the source bins because a Sacc has a # single tracer namespace — every data point references tracers from one # flat list. This is bookkeeping, not physics: psf_stars is a Misc tracer @@ -425,28 +416,6 @@ def add_pure_eb( _add_theta_series(s, PURE_TYPES[key], tracers, theta, arr, grid=grid) -def add_lens(s, name, nz=None, *, quantity="galaxy_density", bias=None): - """Add a γt lens sample: galaxies with their ``(z, n(z))`` and linear - ``bias``, or, as a tracer without either, ``quantity="stars"`` or - ``"randoms"``.""" - if nz is None: - s.add_tracer("Misc", name, quantity=quantity) - else: - metadata = {} if bias is None else {"bias": float(bias)} - s.add_tracer( - "NZ", name, *map(np.asarray, nz), quantity=quantity, metadata=metadata - ) - - -def add_gamma_t(s, source_bin, lens, theta, gamma_t, gamma_x=None, **tags): - """Add γt (and optionally γ×) of source bin ``source_bin`` around ``lens``.""" - _check_ascending("theta", theta) - tracers = (source_name(source_bin), lens) - _add_theta_series(s, GAMMA_T, tracers, theta, gamma_t, **tags) - if gamma_x is not None: - _add_theta_series(s, GAMMA_X, tracers, theta, gamma_x, **tags) - - def add_rho(s, k, theta, rho_p, rho_m, *, grid="reporting"): """Add a ρ_k PSF statistic (ρ+ then ρ−) on the ``psf_stars`` tracer. @@ -1006,52 +975,8 @@ def _check_grid_consistency(s, angle): # --------------------------------------------------------------------------- # # The file door: every SACC is born sealed, derived with one stamp, or refused # --------------------------------------------------------------------------- # -# The blinding rule of each data type a blinded catalogue's SACC may hold. A -# data type with none is refused under a blind, so a new statistic fails closed. -# Signal the blind shifts, each by its theory in sp_validation.theory.THEORY: -SHIFTABLE = (XI_PLUS, XI_MINUS, CL_EE, GAMMA_T) -# signal a pure E-mode shift leaves unchanged: -UNSHIFTED = (CL_BB, CL_EB, GAMMA_X) -# signal derived from ξ±, which moves only through the rows it is computed from: -DERIVED = (COSEBI_EE, COSEBI_BB, *PURE_TYPES.values()) -SIGNAL = SHIFTABLE + UNSHIFTED + DERIVED -# and no signal: the PSF's ρ and the galaxy–PSF τ statistics. -_SIGNAL_FREE = re.compile( - "|".join(t.format(k=r"\d+") for t in (RHO_PLUS, RHO_MINUS, TAU_PLUS, TAU_MINUS)) -) -# γt around a lens tracer of one of these quantities is a null test, not signal. -NULL_LENSES = ("stars", "randoms") - - -def shiftable(s): - """Indices of the rows of ``s`` a blind shifts.""" - return np.array( - [ - i - for i, dp in enumerate(s.data) - if dp.data_type in SHIFTABLE - and not any(s.tracers[t].quantity in NULL_LENSES for t in dp.tracers) - ], - dtype=int, - ) - - -def _refuse_unruled(s): - """Refuse rows of a data type with no blinding rule (under a blind).""" - unruled = sorted( - t - for t in {dp.data_type for dp in s.data} - if t not in SIGNAL and not _SIGNAL_FREE.fullmatch(t) - ) - if unruled: - raise ValueError( - f"{unruled}: no blinding rule for these data types, so a blinded " - "catalogue's SACC cannot hold them. Signal is SHIFTABLE in sacc_io, " - "with a function in sp_validation.theory.THEORY; UNSHIFTED is only for " - "signal that a pure E-mode shift leaves unchanged" - ) - - +# Which rows a blind shifts, leaves or refuses is sp_validation.blinding's +# table of standard estimators, imported only when a file is sealed or derived. def _stamped(s): return _custody.STAMP_KEY in s.metadata @@ -1060,37 +985,43 @@ def _mint(s, custody): s.metadata[_custody.STAMP_KEY] = custody.token -def seal(s, custody): +def seal(s, custody, theory=None): """A stamped copy of ``s``, concealed first when the catalogue is blinded. @sc born-sealed A catalogue-born SACC leaves memory only through here. Under a blinded - custody every :func:`shiftable` row (ξ±, Cℓ_EE, γt) is shifted on a copy + custody every shiftable row (:func:`sp_validation.blinding.shiftable`: + ξ±, Cℓ_EE, and each type ``theory`` gives a function) is shifted on a copy before the stamp is minted; a SACC with none is stamped without opening the blind; a derived statistic (COSEBIs, pure-E/B) is refused here and - saved with ``derived_from``; a data type with no blinding rule is refused. - An already-stamped SACC is re-written only as a derivation. + saved with ``derived_from``; any other type is refused. An already-stamped + SACC is re-written only as a derivation. """ if _stamped(s): raise ValueError( "this SACC is already stamped; a loaded or sealed SACC is re-written " "only as a derivation (save(..., derived_from=[...]))" ) - if custody.blinded: - _refuse_unruled(s) - types = {dp.data_type for dp in s.data} - derived = types & set(DERIVED) - if custody.blinded and derived: - raise ValueError( - f"a blinded catalogue's {sorted(derived)} rows are derived " - "statistics: save them with derived_from=[their input parts]" - ) - if custody.blinded and len(shiftable(s)): + if not custody.blinded: + out = s.copy() + else: from . import blinding - out = blinding.conceal(s, blinding.open_blind(custody)) - else: - out = s.copy() + blinding.refuse_unruled(s, theory) + derived = { + dp.data_type + for dp in s.data + if blinding.rule(dp.data_type) == blinding.DERIVED + } + if derived: + raise ValueError( + f"a blinded catalogue's {sorted(derived)} rows are derived " + "statistics: save them with derived_from=[their input parts]" + ) + if len(blinding.shiftable(s, theory)): + out = blinding.conceal(s, blinding.open_blind(custody), theory) + else: + out = s.copy() _mint(out, custody) return out @@ -1102,6 +1033,8 @@ def _row_key(dp): def _derive(s, parts, custody): """A copy of ``s`` under its inputs' one stamp (and ``custody``'s, if given).""" + from . import blinding + if not parts: raise ValueError("a derivation needs its input parts") stamps = [_custody.read_stamp(p.metadata) for p in parts] @@ -1116,22 +1049,30 @@ def _derive(s, parts, custody): f"parts are stamped {stamp.token}, but the catalogue is declared " f"{custody.token}" ) - if stamp.blinded: - _refuse_unruled(s) - inputs = {_row_key(p.data[i]) for p in parts for i in shiftable(p)} - stray = [i for i in shiftable(s) if _row_key(s.data[i]) not in inputs] + + def born(x): + """Rows only a birth makes: shiftable ones, and under a blind custom types.""" + rules = [blinding.rule(dp.data_type) for dp in x.data] + return [ + i + for i, r in enumerate(rules) + if callable(r) or (stamp.blinded and r is None) + ] + + inputs = {_row_key(p.data[i]) for p in parts for i in born(p)} + stray = [i for i in born(s) if _row_key(s.data[i]) not in inputs] if stray: raise ValueError( - f"{len(stray)} shiftable rows of a derivation are not copies of its " - "inputs' rows; a catalogue's shiftable signal is born only through " - "save(custody=)" + f"{len(stray)} rows of a derivation are not copies of its inputs' " + "rows, but only a birth makes them (shiftable signal, and under a " + "blind any custom type): save them with custody=" ) out = s.copy() _mint(out, stamp) return out -def save(s, path, *, custody=None, derived_from=None): +def save(s, path, *, custody=None, derived_from=None, theory=None): """Write ``s`` to ``path`` (FITS) through the one door; return what was written. @sc one-door @@ -1139,21 +1080,28 @@ def save(s, path, *, custody=None, derived_from=None): custody stamp is minted: - ``save(s, path, custody=c)``: a birth, sealed by :func:`seal`; + ``theory={data_type: f}`` shifts a type outside the standard estimators + under a blind (:mod:`sp_validation.blinding`); - ``save(s, path, derived_from=parts)``: a derivation, stamped with its inputs' one stamp; - ``save(s, path, derived_from=parts, custody=c)``: an assembly, a derivation whose stamp must also be ``c``'s. + A measurement computes its signal and saves (or seals) it in the same + function, returning the sealed part, so its raw signal never leaves it. + @sc derived-inherit A derivation's inputs must share one stamp, and each of its shiftable rows - must be a copy of an input row (type, tracers, value, tags; windows by - index), so plaintext cannot be saved under a concealed stamp; under a - blinded stamp every data type needs a blinding rule, as at birth. + (under a blind, each row of a custom type too) must be a copy of an input + row (type, tracers, value, tags; windows by index), so plaintext cannot be + saved under a concealed stamp. """ if derived_from is not None: + if theory is not None: + raise ValueError("theory= shifts a birth; a derivation inherits") out = _derive(s, list(derived_from), custody) elif custody is not None: - out = seal(s, custody) + out = seal(s, custody, theory) else: raise ValueError( "save needs custody= (a birth) or derived_from= (a derivation); " diff --git a/src/sp_validation/tests/_synthetic.py b/src/sp_validation/tests/_synthetic.py index e1d1786e..cae90b6a 100644 --- a/src/sp_validation/tests/_synthetic.py +++ b/src/sp_validation/tests/_synthetic.py @@ -6,8 +6,9 @@ registry is ``blinds/`` beside it. Every catalogue entry in the config reads its own copy of the same galaxies and declares its own custody. -``TOY_THEORY`` stands in for :data:`sp_validation.theory.THEORY`, so a blind -shifts without running CAMB. +``TOY_STANDARD`` is :data:`sp_validation.blinding.STANDARD` with the analytic +``toy_theory`` in place of each default theory, so a blind shifts without +running CAMB. """ import copy @@ -16,7 +17,7 @@ import numpy as np import yaml -from sp_validation import sacc_io +from sp_validation import blinding def toy_theory(params, s, rows): @@ -25,7 +26,9 @@ def toy_theory(params, s, rows): return 1e-4 * params["S8"] ** 2 * params["Omega_m"] ** 0.3 * (x / 10.0) ** -0.8 -TOY_THEORY = MappingProxyType({t: toy_theory for t in sacc_io.SHIFTABLE}) +TOY_STANDARD = MappingProxyType( + {t: toy_theory if callable(r) else r for t, r in blinding.STANDARD.items()} +) def write_synthetic_catalogs( diff --git a/src/sp_validation/tests/conftest.py b/src/sp_validation/tests/conftest.py index 781ebb93..b420ee63 100644 --- a/src/sp_validation/tests/conftest.py +++ b/src/sp_validation/tests/conftest.py @@ -21,9 +21,9 @@ def pure_eb_xi(): @pytest.fixture def toy_theory(monkeypatch): - """Blinds shift by the analytic ``_synthetic.TOY_THEORY``.""" - from _synthetic import TOY_THEORY + """Blinds shift by the analytic ``_synthetic.toy_theory``.""" + from _synthetic import TOY_STANDARD - from sp_validation import theory + from sp_validation import blinding - monkeypatch.setattr(theory, "THEORY", TOY_THEORY) + monkeypatch.setattr(blinding, "STANDARD", TOY_STANDARD) diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 1bf716b8..23a0b548 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -3,18 +3,15 @@ A blind is drawn once (``blinding init``) into a registry outside git; ``sacc_io.save`` is the only writer, and conceals a blinded catalogue's shiftable rows in memory before the file exists. Blinds here shift by the -analytic ``TOY_THEORY``; one slow test runs the CCL defaults. +analytic ``toy_theory``; one slow test runs the CCL defaults. """ import dataclasses -import json -import logging import stat -import traceback import numpy as np import pytest -from _synthetic import TOY_THEORY +from _synthetic import TOY_STANDARD, toy_theory from sp_validation import blinding as bd from sp_validation import custody as cu @@ -22,13 +19,14 @@ from sp_validation import theory VERSIONS = ("TOY", "OTHER", "TOY_OPEN", "TOY_MOCK") -DEFAULTS = theory.THEORY +DEFAULTS = {t: f for t, f in bd.STANDARD.items() if callable(f)} +CUSTOM = "galaxy_density_xi" @pytest.fixture(scope="module", autouse=True) def _toy_theory(): with pytest.MonkeyPatch.context() as m: - m.setattr(theory, "THEORY", TOY_THEORY) + m.setattr(bd, "STANDARD", TOY_STANDARD) yield @@ -65,14 +63,12 @@ def part( xi_tags=("reporting",), cl=True, rho=False, - gt=(), derived=None, unruled=False, ): """A two-bin part: ξ± under each of ``xi_tags`` (None: no tag) and - pseudo-Cℓ (EE, BB, EB) on pairs (0,0), (0,1), (1,1), and optionally ρ/τ, - γt and γ× of bin 1 around each lens quantity in ``gt``, a derived - statistic's rows and rows of a type with no blinding rule.""" + pseudo-Cℓ (EE, BB, EB) on pairs (0,0), (0,1), (1,1), and optionally ρ, + a derived statistic's rows and rows of a custom type.""" s = sio.new_sacc({0: _nz(0.5), 1: _nz(0.9)}) theta = np.geomspace(2.0, 200.0, 6) ell = np.array([30.0, 80.0, 150.0, 280.0, 450.0]) @@ -101,21 +97,13 @@ def part( ) if rho: sio.add_rho(s, 0, theta, np.arange(1, 7) * 1e-7, np.arange(1, 7) * 2e-7) - for quantity in gt: - if quantity == "galaxy_density": - sio.add_lens(s, quantity, _nz(0.3), quantity=quantity, bias=1.5) - else: - sio.add_lens(s, quantity, quantity=quantity) - sio.add_gamma_t(s, 1, quantity, theta, 1e-4 * (theta / 10) ** -0.7, 0 * theta) if derived == "cosebis": sio.add_cosebis(s, (0, 0), np.arange(1, 6) * 1e-10, (12.0, 83.0), Bn=np.ones(5)) elif derived == "pure_eb": sio.add_pure_eb(s, (0, 0), theta[:4], **{k: np.ones(4) for k in sio.PURE_KEYS}) if unruled: for x in (5.0, 20.0, 80.0): - s.add_data_point( - "galaxy_density_xi", ("source_0", "source_0"), 1.0, theta=x - ) + s.add_data_point(CUSTOM, ("source_0", "source_0"), 1e-4, theta=x) s.add_covariance(np.abs(np.asarray(s.mean)) ** 2 + 1e-20) return s @@ -125,13 +113,9 @@ def _values(s): # --------------------------------------------------------------------------- # -# Born sealed: a blind shifts exactly the ξ±, Cℓ_EE and γt rows, nothing else +# Born sealed: a blind shifts exactly the ξ± and Cℓ_EE rows, nothing else # --------------------------------------------------------------------------- # -EVERYTHING = dict( - xi_tags=("reporting", "mystery", None), - rho=True, - gt=("galaxy_density", "stars", "randoms"), -) +EVERYTHING = dict(xi_tags=("reporting", "mystery", None), rho=True) @pytest.mark.parametrize( @@ -149,11 +133,11 @@ def _values(s): def test_seal_shifts_only_the_signal_it_has_a_rule_for( blinds, content, version, refused ): - """Under a blind every ξ±, Cℓ_EE and galaxy-lens γt row moves, whatever - its grid tag, and every other value (γ×, γt around stars or randoms) and - the covariance stay bitwise; a blinded birth carrying derived rows or a - type with no blinding rule is refused. Unblinded and mock births keep - their values. Each is stamped with its custody.""" + """Under a blind every ξ± and Cℓ_EE row moves, whatever its grid tag, and + every other value (Cℓ_BB/EB, ρ) and the covariance stay bitwise; a blinded + birth carrying derived rows or a custom type without a theory is refused. + Unblinded and mock births keep their values. Each is stamped with its + custody.""" s, custody = part(**content), blinds[version] if refused: with pytest.raises(ValueError, match="derived_from|no blinding rule"): @@ -161,12 +145,7 @@ def test_seal_shifts_only_the_signal_it_has_a_rule_for( return sealed = sio.seal(s, custody) assert cu.read_stamp(sealed.metadata) == custody - shiftable = np.array( - [ - dp.data_type in sio.SHIFTABLE and not {"stars", "randoms"} & {*dp.tracers} - for dp in s.data - ] - ) + shiftable = np.array([callable(bd.STANDARD.get(dp.data_type)) for dp in s.data]) moved = _values(sealed) != _values(s) assert np.array_equal(moved, shiftable & custody.blinded) assert np.array_equal(sealed.covariance.dense, s.covariance.dense) @@ -235,7 +214,7 @@ def test_a_derivation_carries_its_inputs_one_stamp( # --------------------------------------------------------------------------- # -# The blind: drawn once, opened only under its commitment, never shown +# The blind: drawn once, opened only under its commitment # --------------------------------------------------------------------------- # def test_a_blind_is_drawn_once_and_kept_private(tmp_path): cats = _catalogues(tmp_path) @@ -258,96 +237,46 @@ def test_a_blind_opens_only_under_its_commitment(blinds, monkeypatch): bd.open_blind(custody) -def _hidden_failure(blind): - """The traceback, locals included, of a theory failing at the hidden point - with its parameters in its message and on stdout.""" - - def failing(params, *args): - print(dict(params)) - raise RuntimeError(f"cannot evaluate {dict(params)}") - - s = part(cl=False) - with pytest.raises(bd.BlindingError) as failure: - bd._at_hidden(s, sio.shiftable(s), dict.fromkeys(sio.SHIFTABLE, failing), blind) - trace = traceback.TracebackException.from_exception( - failure.value, capture_locals=True +# --------------------------------------------------------------------------- # +# The shift: from each data type's theory, refused where it cannot be made +# --------------------------------------------------------------------------- # +def test_a_custom_type_is_shifted_by_its_own_theory(blinds, tmp_path): + """Under a blind a custom data type is shifted when its birth passes a + theory function and refused without one; its sealed rows then pass an + assembly. A standard type the blind leaves alone takes no theory.""" + custody, s = blinds["TOY"], part(cl=False, unruled=True) + with pytest.raises(ValueError, match="no blinding rule"): + sio.save(s, tmp_path / "refused.sacc", custody=custody) + sealed = sio.save( + s, tmp_path / "custom.sacc", custody=custody, theory={CUSTOM: toy_theory} ) - return "".join(trace.format()) + rows = s.indices(CUSTOM) + assert np.all(_values(sealed)[rows] != _values(s)[rows]) + sio.save(sealed.copy(), tmp_path / "assembled.sacc", derived_from=[sealed]) + with pytest.raises(ValueError, match="no theory"): + sio.seal(part(), custody, theory={sio.CL_BB: toy_theory}) -def test_the_hidden_cosmology_never_shows(tmp_path, capfd, caplog, recwarn): - """Neither the seed nor the hidden S8, Ωm, σ8 or Ω_c reaches a repr, the - terminal, a log, a warning or a traceback's locals.""" - caplog.set_level(logging.DEBUG) - cats = _catalogues(tmp_path) - blind = bd.init("toy", cats) - record = json.loads(blind.path.read_text()) - hidden = bd._hidden(blind) - sigma8 = hidden["S8"] / np.sqrt(hidden["Omega_m"] / 0.3) - omega_c = theory.cosmology(hidden)["Omega_c"] - needles = [record["seed"], record["seed"][:16]] + [ - form(x) - for x in (hidden["S8"], hidden["Omega_m"], sigma8, omega_c) - for form in (repr, "{:.4g}".format, "{:.6g}".format) - ] - bd.show("toy", cats) - sio.seal(part(cl=False), cu.custody_of(cats, "TOY")) - failure = _hidden_failure(blind) - haystacks = [ - *(form(x) for x in (blind, hidden) for form in (repr, str, "{}".format)), - "".join(capfd.readouterr()), - caplog.text, - "".join(str(w.message) for w in recwarn), - failure, - ] - for i, haystack in enumerate(haystacks): - # The message names the haystack only: a failure must not print a needle. - assert not any(n in haystack for n in needles), f"haystack {i} shows it" +def test_conceal_refuses_a_theory_blind_to_cosmology(blinds): + """A theory that ignores the cosmology is refused rather than stamped as + blinded.""" + def flat(params, s, rows): + return np.ones(len(rows)) -# --------------------------------------------------------------------------- # -# The shift: from each data type's theory, refused where it cannot be made -# --------------------------------------------------------------------------- # -def test_conceal_refuses_a_shift_it_cannot_make(blinds): - """A shiftable type without a theory, and a theory that ignores the - cosmology, are refused rather than stamped as blinded.""" blind = bd.open_blind(blinds["TOY"]) - s = part(cl=False, gt=["galaxy_density"]) - no_gt = {t: f for t, f in TOY_THEORY.items() if t != sio.GAMMA_T} - with pytest.raises(bd.BlindingError, match="no theory"): - bd.conceal(s, blind, theory=no_gt) - flat = {**TOY_THEORY, sio.GAMMA_T: lambda params, s, rows: np.ones(len(rows))} with pytest.raises(bd.BlindingError, match="unmoved"): - bd.conceal(s, blind, theory=flat) - - -def test_gamma_t_needs_its_lens_samples_bias(): - s = sio.new_sacc({0: _nz(0.5)}) - sio.add_lens(s, "lens", _nz(0.3)) - sio.add_gamma_t(s, 0, "lens", np.geomspace(2.0, 200.0, 4), np.ones(4)) - with pytest.raises(ValueError, match="no linear bias"): - DEFAULTS[sio.GAMMA_T](theory.fiducial(), s, s.indices(sio.GAMMA_T)) + bd.conceal(part(cl=False), blind, theory={sio.XI_PLUS: flat}) @pytest.mark.slow def test_the_ccl_defaults_shift_with_s8(blinds, monkeypatch): """The default theories, at a hidden point with S8 above the fiducial, - raise ξ+, Cℓ_EE and γt, and γt's shift does not depend on tracer order.""" + raise ξ+ and Cℓ_EE.""" fiducial = theory.fiducial() monkeypatch.setattr(bd, "_hidden", lambda blind: {**fiducial, "S8": 0.9}) blind = bd.open_blind(blinds["TOY"]) - s = part(gt=["galaxy_density"]) + s = part() delta = bd.conceal(s, blind, theory=DEFAULTS).mean - s.mean - for data_type in (sio.XI_PLUS, sio.CL_EE, sio.GAMMA_T): + for data_type in (sio.XI_PLUS, sio.CL_EE): assert np.all(delta[s.indices(data_type)] > 0), data_type - - swapped = s.copy() - for dp in swapped.data: - if dp.data_type == sio.GAMMA_T: - dp.tracers = dp.tracers[::-1] - rows = s.indices(sio.GAMMA_T) - np.testing.assert_allclose( - (bd.conceal(swapped, blind, theory=DEFAULTS).mean - s.mean)[rows], - delta[rows], - rtol=1e-10, - ) diff --git a/src/sp_validation/tests/test_blinding_bmodes.py b/src/sp_validation/tests/test_blinding_bmodes.py index 89623763..49b4379a 100644 --- a/src/sp_validation/tests/test_blinding_bmodes.py +++ b/src/sp_validation/tests/test_blinding_bmodes.py @@ -51,7 +51,7 @@ def _fiducial_xi(): s = sio.new_sacc({0: (z, np.exp(-(((z - 0.7) / 0.3) ** 2)))}) for theta, grid in ((THETA, "reporting"), (THETA_INT, "integration")): sio.add_xi(s, (0, 0), theta, 0 * theta, 0 * theta, grid=grid, theta_nom=theta) - for dp, value in zip(s.data, theory.predict(s, theory.fiducial())): + for dp, value in zip(s.data, theory.predict(s, theory.fiducial(), bd.STANDARD)): dp.value = float(value) s.add_covariance(VARIANCE) return s diff --git a/src/sp_validation/theory.py b/src/sp_validation/theory.py index dee8bb4f..33a9c6c8 100644 --- a/src/sp_validation/theory.py +++ b/src/sp_validation/theory.py @@ -5,9 +5,9 @@ parameter point ``params``, for rows ``rows`` of SACC ``s``, which share a data type's theory, a tracer pair and a bandpower window; values are aligned to ``rows``. ``params`` is a plain mapping with the keys of :func:`fiducial`, -never a CCL object, so an emulator can stand in for CCL. :data:`THEORY` maps -each data type to its default, and :func:`predict` takes any other mapping as -``theory=``. +never a CCL object, so an emulator can stand in for CCL. :func:`shear_xi` and +:func:`shear_cl` are the defaults :data:`sp_validation.blinding.STANDARD` +shifts ξ± and Cℓ_EE by. @sc theory-ccl-default The defaults build one ``pyccl.Cosmology`` per point through @@ -19,13 +19,12 @@ """ import functools -from types import MappingProxyType import numpy as np -from .sacc_io import CL_EE, GAMMA_T, XI_MINUS, XI_PLUS +from .sacc_io import XI_PLUS -# Multipoles the ξ± and γt Hankel transforms integrate over: every ℓ below 50, +# Multipoles the ξ± Hankel transform integrates over: every ℓ below 50, # then 200 log-spaced up to 6·10⁴. ELL = np.unique(np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)])) _COSMOLOGY = ("S8", "Omega_m", "Omega_b", "h", "n_s", "m_nu", "w0", "wa", "logT_AGN") @@ -99,17 +98,6 @@ def nz(s, name): return np.asarray(tracer.z, float), np.asarray(tracer.nz, float) -def source_lens(s, rows): - """The source and lens tracers of γt-like rows, told apart by quantity.""" - names = s.data[rows[0]].tracers - by = {s.tracers[n].quantity: n for n in names} - if len(names) != 2 or set(by) != {"galaxy_shear", "galaxy_density"}: - raise ValueError( - f"tracers {names} are not one galaxy_shear and one galaxy_density tracer" - ) - return by["galaxy_shear"], by["galaxy_density"] - - def _lensing(cosmo, params, s, name): import pyccl as ccl @@ -146,37 +134,13 @@ def shear_cl(params, s, rows): return np.asarray(window.weight).T @ cl -def gamma_t(params, s, rows): - """γt of a source bin around a lens sample of the linear bias its tracer - carries (``sacc_io.add_lens(..., bias=)``).""" - import pyccl as ccl - - source, lens = source_lens(s, rows) - b = (s.tracers[lens].metadata or {}).get("bias") - if b is None: - raise ValueError(f"lens tracer {lens} carries no linear bias") - cosmo = cosmology(params) - z, n = nz(s, lens) - bias = (z, np.full_like(z, float(b))) - counts = ccl.NumberCountsTracer(cosmo, has_rsd=False, dndz=(z, n), bias=bias) - cl = ccl.angular_cl(cosmo, counts, _lensing(cosmo, params, s, source), ELL) - theta = tag(s, rows, "theta") / 60.0 - return ccl.correlation(cosmo, ell=ELL, C_ell=cl, theta=theta, type="NG") - - -THEORY = MappingProxyType( - {XI_PLUS: shear_xi, XI_MINUS: shear_xi, CL_EE: shear_cl, GAMMA_T: gamma_t} -) - - -def predict(s, params, rows=None, theory=None): +def predict(s, params, theory, rows=None): """The theory of ``rows`` (default: all) of ``s`` at ``params``. - Rows are grouped by theory function, tracer pair and bandpower window, one - call per group, so ξ+ and ξ− of a pair share their C_ℓ. ``theory`` maps - data type to function (default :data:`THEORY`); a row without one raises. + ``theory`` maps data type to function; a row without one raises. Rows are + grouped by function, tracer pair and bandpower window, one call per group, + so ξ+ and ξ− of a pair share their C_ℓ. """ - theory = THEORY if theory is None else theory rows = np.arange(len(s.data)) if rows is None else np.asarray(rows, int) groups = {} for n, i in enumerate(rows): diff --git a/workflow/README.md b/workflow/README.md index cf30d9c1..d168e306 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -34,21 +34,27 @@ one blind (the repository config is authoritative), and a blinded catalogue is shown only beside mocks and its own blind, since any other overlay shows the shift. A launch prints one `[custody]` line per catalogue. -Under a blind, ξ±, pseudo-Cℓ_EE and γt are shifted by t(hidden) − t(fiducial) -before they are first written, t being the data type's theory in -`sp_validation.theory.THEORY`; statistics derived from them carry the blind, and -a data type with no blinding rule in `sacc_io` is refused. COSEBIs B_n move only -by the transform's response to an E-mode shift; pure-E/B ξ_B is not additive on noisy ξ± -and also moves in isolated bins (`test_blinding_bmodes`). -Rule params carry the custody token, so a flip reruns what it touches. +Under a blind, ξ± and pseudo-Cℓ_EE are shifted by t(hidden) − t(fiducial) +before they are first written, t being the default PyCCL theory; +`sp_validation.blinding.STANDARD` lists every standard estimator's rule once +(Cℓ_BB/EB unshifted, COSEBIs and pure-E/B inheriting from ξ±, ρ/τ signal-free). +Any other data type is shifted by a theory function its birth passes, +`sacc_io.save(s, path, custody=c, theory={data_type: f})` with +`f(params, s, rows) -> values`, and is refused under a blind without one. +COSEBIs B_n move only by the transform's response to an E-mode shift; pure-E/B +ξ_B is not additive on noisy ξ± and also moves in isolated bins +(`test_blinding_bmodes`). Rule params carry the custody token, so a flip reruns +what it touches. + +A measurement calculates, then saves: one function computes the signal, saves +or seals it under the catalogue's custody, and returns the sealed part, so raw +signal never leaves it (`CosmologyValidation.calculate_2pcf` is the pattern). A blind is one record, `/.blind.json`, outside any git worktree; read access to it is access to the blind. `spv-container exec python -m sp_validation.blinding init ` draws one, -once; `… show ` prints its public record. For an entry under a blind, -measure signal only through `CosmologyValidation` or the workflow; never set -`blind: none` to get a run through; never print, paste or commit a -`.blind.json`. +once; `… show ` prints its public record. Never set `blind: none` to get +a run through; never print, paste or commit a `.blind.json`. ## Running on the cluster — the candide profile From ae95e781ea109fb2762813f824f1cdd323550a40 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 04:10:34 +0200 Subject: [PATCH 146/160] Blinding is one Smokescreen call over a theory(params, s) vector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit theory.shear predicts ξ± and Cℓ_EE for every row of a SACC (zeros for Cℓ_BB/EB), theory.none gives zeros. blinding.conceal hands the theory to smokescreen's concealing_factor and adds the vector; a failure names only its exception type. sacc_io.save conceals a birth under a Blind and stamps its name, or stamps a derivation with its inputs' one blind. Co-Authored-By: Claude Opus 5.5 --- src/sp_validation/blinding.py | 278 ++++++++-------------------------- src/sp_validation/sacc_io.py | 181 ++++++---------------- src/sp_validation/theory.py | 143 +++++++++-------- 3 files changed, 195 insertions(+), 407 deletions(-) diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index fac14e40..99717403 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -1,281 +1,135 @@ -"""The blind: a secret seed, drawn once, and the shift it conceals signal by. +"""Blinds, and the one call that conceals a data vector under one. -@sc standard-estimators -:data:`STANDARD` names each estimator cosmo_val computes and its blinding -rule, once. Any other data type is shifted when its birth passes a theory -function, ``sacc_io.save(s, path, custody=c, theory={data_type: f})``, with -``f(params, s, rows) -> values`` as in :mod:`sp_validation.theory`; without -one it is refused under a blind. +A blind is a named secret, ``/.blind.json`` holding a +seed, the envelope the hidden point is drawn in and the fiducial point. It is +drawn once, ``python -m sp_validation.blinding init ``, and never +committed (``*.blind.json`` is gitignored). -@sc blind-record -A blind is one read-only file, ``/.blind.json``, outside -any git worktree: its seed (unencrypted: registry access is blind access), the -envelope the hidden point is drawn in, the fiducial and the fork's draw scheme. -:class:`Blind` holds a name and a path, never a hidden value; :func:`_hidden` -returns a mapping whose repr is ````, and a theory failure at -the hidden point is reported by exception type alone. - -@sc hidden-draw-uniform-s8-om -The hidden point is the fiducial (:func:`sp_validation.theory.fiducial`) moved -by the fork's per-key draw from the seed, uniform within the envelope in S8 and -Ωm. - -``python -m sp_validation.blinding init `` draws a blind; ``show `` -prints its public record (everything but the seed). +:func:`conceal` adds the blind's shift, t(hidden) − t(fiducial), to every row +of a SACC; Smokescreen draws the hidden point from the seed and evaluates the +theory t at both points (:mod:`sp_validation.theory`). """ import argparse import dataclasses -import io import json import os +import re import secrets -import warnings -from collections.abc import Mapping -from contextlib import redirect_stderr, redirect_stdout from pathlib import Path -from types import MappingProxyType import numpy as np -from . import custody as _custody -from . import sacc_io as sio from . import theory as _theory ENVELOPE = {"S8": 0.075, "Omega_m": 0.1} -SECRET = "secret: never print, paste or commit" - -# --------------------------------------------------------------------------- # -# The standard estimators cosmo_val computes, and how a blind treats each -# --------------------------------------------------------------------------- # -UNSHIFTED = "unshifted" # signal a pure E-mode shift leaves unchanged -DERIVED = "derived" # computed from ξ±, so shifted through the rows it reads -SIGNAL_FREE = "signal-free" # PSF diagnostics, no cosmological signal - -STANDARD = MappingProxyType( - { - # Shifted by t(hidden) − t(fiducial), t the default PyCCL theory. - sio.XI_PLUS: _theory.shear_xi, - sio.XI_MINUS: _theory.shear_xi, - sio.CL_EE: _theory.shear_cl, - sio.CL_BB: UNSHIFTED, - sio.CL_EB: UNSHIFTED, - sio.COSEBI_EE: DERIVED, - sio.COSEBI_BB: DERIVED, - **dict.fromkeys(sio.PURE_TYPES.values(), DERIVED), - **{ - t.format(k=k): SIGNAL_FREE - for ts, ks in ( - ((sio.RHO_PLUS, sio.RHO_MINUS), range(6)), - ((sio.TAU_PLUS, sio.TAU_MINUS), (0, 2, 5)), - ) - for t in ts - for k in ks - }, - } -) - - -def rule(data_type, theory=None): - """``data_type``'s rule: its theory function if a blind shifts it, else - :data:`UNSHIFTED`, :data:`DERIVED`, :data:`SIGNAL_FREE`, or None (refused - under a blind). ``theory`` gives functions for custom types and may - replace a standard one's.""" - standard = STANDARD.get(data_type) - if theory and data_type in theory: - if standard is not None and not callable(standard): - raise ValueError(f"{data_type} is {standard} under a blind; no theory") - return theory[data_type] - return standard - - -def shiftable(s, theory=None): - """Indices of the rows of ``s`` a blind shifts.""" - rules = [rule(dp.data_type, theory) for dp in s.data] - return np.array([i for i, r in enumerate(rules) if callable(r)], dtype=int) - - -def refuse_unruled(s, theory=None): - """Refuse rows of a data type with no blinding rule.""" - unruled = sorted( - {dp.data_type for dp in s.data if rule(dp.data_type, theory) is None} - ) - if unruled: - raise ValueError( - f"{unruled}: no blinding rule for these data types, so a blinded " - "catalogue's SACC cannot hold them; pass each a theory function, " - "save(..., theory={data_type: f})" - ) +REPO_CAT_CONFIG = Path(__file__).parents[2] / "cosmo_val" / "cat_config.yaml" +_NAME = re.compile(r"[a-z0-9][a-z0-9_.-]*") class BlindingError(RuntimeError): - """Concealment failed; the message carries no hidden value.""" + """A blind cannot be opened or applied; the message carries no hidden value.""" @dataclasses.dataclass(frozen=True) class Blind: - """An opened blind: its name and the path of its record.""" + """A blind's name and the path of its record; ``none`` has no record.""" name: str - path: Path - - -def draw_scheme(): - """The installed fork's shift-draw semantics (``smokescreen.DRAW_SCHEME``).""" - from smokescreen import DRAW_SCHEME - - return int(DRAW_SCHEME) - - -def open_blind(custody): - """The blind a blinded custody names, its record matching the commitment.""" - path = Path(custody.registry or "") / f"{custody.blind}.blind.json" - try: - record = json.loads(path.read_text()) - except (OSError, ValueError): - raise _custody.CustodyError( - f"cannot read blind {custody.blind} at {path}" - ) from None - if _custody.commitment(record) != custody.commitment: - raise _custody.CustodyError( - f"blind {custody.blind}'s record no longer matches the commitment this " - "run was launched under; launch again" - ) - if record["draw_scheme"] != draw_scheme(): - raise _custody.CustodyError( - f"blind {custody.blind} was drawn under draw scheme " - f"{record['draw_scheme']}; this smokescreen draws under {draw_scheme()}" - ) - return Blind(custody.blind, path) - - -class _Hidden(Mapping): - def __init__(self, values): - self._values = values - - def __getitem__(self, key): - return self._values[key] + path: Path | None = None - def __iter__(self): - return iter(self._values) + def record(self): + """The blind's ``{seed, envelope, fiducial}``.""" + return json.loads(self.path.read_text()) - def __len__(self): - return len(self._values) - def __repr__(self): - return "" +NONE = Blind("none") - __str__ = __repr__ +def registry(catalogues): + """The directory a catalogue config keeps its blinds in (``paths.blinds``).""" + return Path(os.path.expanduser(catalogues["paths"]["blinds"])) -def _hidden(blind): - """The blind's hidden point: its fiducial moved by the seed's draw.""" - from smokescreen.param_shifts import draw_param_shifts - record = json.loads(blind.path.read_text()) - shift = draw_param_shifts(dict(record["envelope"]), record["seed"]) - fiducial = record["fiducial"] - return _Hidden({k: v + shift[k] if k in shift else v for k, v in fiducial.items()}) +def open_blind(name, catalogues): + """Blind ``name`` of the catalogue config ``catalogues``; its record must exist.""" + if name == NONE.name: + return NONE + path = registry(catalogues) / f"{name}.blind.json" + if not os.access(path, os.R_OK): + raise BlindingError( + f"cannot read blind {name} at {path}; `python -m sp_validation.blinding " + f"init {name}` draws it, if it is meant to be new" + ) + return Blind(name, path) -def _at_hidden(s, rows, theory, blind): - """The theory of ``rows`` at the hidden point; a failure names only its type. +def conceal(s, blind, theory=None): + """A copy of SACC ``s`` with ``blind``'s shift added to every row. - Warnings and the theory's own prints are silenced, and the error is raised - outside the ``except``, so no context or frame carries the hidden point. + The shift is t(hidden) − t(fiducial), t being ``theory`` (default + :func:`sp_validation.theory.shear`) evaluated on ``s``. A failure raises + :class:`BlindingError` naming only the exception type, so no message or + traceback shows the hidden point. """ + from smokescreen.datavector import concealing_factor - def evaluate(): - quiet = io.StringIO() - with warnings.catch_warnings(), redirect_stdout(quiet), redirect_stderr(quiet): - warnings.simplefilter("ignore") - return _theory.predict(s, _hidden(blind), theory, rows) - + theory = theory or _theory.shear + record, failure = blind.record(), None try: - return evaluate() - except Exception as err: # the message must carry no value + shift = concealing_factor( + record["fiducial"], + record["envelope"], + seed=record["seed"], + theory_fn=lambda params: theory(params, s), + ) + except Exception as err: failure = type(err).__name__ - raise BlindingError( - f"the theory failed at blind {blind.name}'s hidden point ({failure})" - ) - - -def conceal(s, blind, theory=None): - """A copy of ``s`` with its :func:`shiftable` rows moved by the blind's shift. - - The shift is t(hidden) − t(fiducial), t each row's theory (:data:`STANDARD`, - or ``theory`` for a custom type), evaluated at the fiducial first. A data - type and tracer pair the blind leaves unmoved or moves to a non-finite - value is refused. - """ - rows = shiftable(s, theory) - functions = {s.data[i].data_type: rule(s.data[i].data_type, theory) for i in rows} - fiducial = json.loads(blind.path.read_text())["fiducial"] - at_fiducial = _theory.predict(s, fiducial, functions, rows) - delta = _at_hidden(s, rows, functions, blind) - at_fiducial - groups = {} - for n, i in enumerate(rows): - groups.setdefault((s.data[i].data_type, s.data[i].tracers), []).append(n) - for (data_type, pair), at in groups.items(): - if not np.all(np.isfinite(delta[at])) or not delta[at].any(): - raise BlindingError( - f"blind {blind.name} leaves {data_type} of {pair} unmoved or " - "non-finite; its theory must depend on the cosmology" - ) + if failure: # raised outside the except, so nothing is chained + raise BlindingError(f"the theory failed under blind {blind.name} ({failure})") + shift = np.asarray(shift, float) + if shift.shape != (len(s.mean),): + raise BlindingError( + f"the theory returned {shift.shape[0] if shift.ndim else 0} values for " + f"{len(s.mean)} rows" + ) out = s.copy() - for row, d in zip(rows, delta): - out.data[int(row)].value += float(d) + for dp, d in zip(out.data, shift): + dp.value += float(d) return out def init(name, catalogues): """Draw blind ``name`` into the catalogue config's registry, once.""" - if not _custody.BLIND_NAME.fullmatch(name) or name in ( - _custody.NONE, - _custody.MOCK, - ): - raise _custody.CustodyError(f"blind name {name!r}: use a-z, 0-9, - _ .") - where = _custody.registry(catalogues) + if not _NAME.fullmatch(name) or name == NONE.name: + raise BlindingError(f"blind name {name!r}: use a-z, 0-9, - _ .") + where = registry(catalogues) where.mkdir(mode=0o700, parents=True, exist_ok=True) record = { - "_": SECRET, "seed": secrets.token_hex(32), "envelope": ENVELOPE, "fiducial": _theory.fiducial(), - "draw_scheme": draw_scheme(), } path = where / f"{name}.blind.json" try: fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o440) except FileExistsError: - raise _custody.CustodyError( - f"blind {name} exists at {path}; a blind is drawn once" - ) from None + raise BlindingError(f"blind {name} exists at {path}") from None with os.fdopen(fd, "w") as f: json.dump(record, f, indent=1) - print(f"[blinding] drew blind {name}, commitment {_custody.commitment(record)}") + print(f"[blinding] drew blind {name} at {path}") return Blind(name, path) -def show(name, catalogues): - """Print blind ``name``'s public record: everything but the seed.""" - path = _custody.registry(catalogues) / f"{name}.blind.json" - record = json.loads(path.read_text()) - print(f"blind {name}, commitment {_custody.commitment(record)}") - for key in ("envelope", "fiducial", "draw_scheme"): - print(f" {key}: {json.dumps(record[key])}") - - def main(argv=None): import yaml parser = argparse.ArgumentParser(prog="python -m sp_validation.blinding") - parser.add_argument("command", choices=("init", "show")) + parser.add_argument("command", choices=("init",)) parser.add_argument("blind") - parser.add_argument("--cat-config", default=str(_custody.REPO_CAT_CONFIG)) + parser.add_argument("--cat-config", default=str(REPO_CAT_CONFIG)) a = parser.parse_args(argv) - catalogues = yaml.safe_load(Path(a.cat_config).read_text()) - (init if a.command == "init" else show)(a.blind, catalogues) + init(a.blind, yaml.safe_load(Path(a.cat_config).read_text())) return 0 diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 6ec8694d..6688ce9b 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -93,7 +93,6 @@ import sacc from astropy.io import fits -from . import custody as _custody from .statistics import cov_from_one_covariance PSF_TRACER = "psf_stars" @@ -973,157 +972,77 @@ def _check_grid_consistency(s, angle): # --------------------------------------------------------------------------- # -# The file door: every SACC is born sealed, derived with one stamp, or refused +# Saving: a birth is concealed under its catalogue's blind, a derivation +# inherits its inputs' stamp # --------------------------------------------------------------------------- # -# Which rows a blind shifts, leaves or refuses is sp_validation.blinding's -# table of standard estimators, imported only when a file is sealed or derived. -def _stamped(s): - return _custody.STAMP_KEY in s.metadata +STAMP_KEY = "blind" -def _mint(s, custody): - s.metadata[_custody.STAMP_KEY] = custody.token +def stamp(s): + """The blind ``s`` was saved under: its ``blind`` metadata, ``none`` if absent.""" + return s.metadata.get(STAMP_KEY, "none") -def seal(s, custody, theory=None): - """A stamped copy of ``s``, concealed first when the catalogue is blinded. - - @sc born-sealed - A catalogue-born SACC leaves memory only through here. Under a blinded - custody every shiftable row (:func:`sp_validation.blinding.shiftable`: - ξ±, Cℓ_EE, and each type ``theory`` gives a function) is shifted on a copy - before the stamp is minted; a SACC with none is stamped without opening - the blind; a derived statistic (COSEBIs, pure-E/B) is refused here and - saved with ``derived_from``; any other type is refused. An already-stamped - SACC is re-written only as a derivation. - """ - if _stamped(s): - raise ValueError( - "this SACC is already stamped; a loaded or sealed SACC is re-written " - "only as a derivation (save(..., derived_from=[...]))" - ) - if not custody.blinded: +def seal(s, blind, theory=None): + """What :func:`save` writes for a birth, kept in memory: a copy of ``s`` + concealed under ``blind`` with ``theory`` and stamped with its name.""" + if blind.name == "none": out = s.copy() else: - from . import blinding - - blinding.refuse_unruled(s, theory) - derived = { - dp.data_type - for dp in s.data - if blinding.rule(dp.data_type) == blinding.DERIVED - } - if derived: - raise ValueError( - f"a blinded catalogue's {sorted(derived)} rows are derived " - "statistics: save them with derived_from=[their input parts]" - ) - if len(blinding.shiftable(s, theory)): - out = blinding.conceal(s, blinding.open_blind(custody), theory) - else: - out = s.copy() - _mint(out, custody) - return out - - -def _row_key(dp): - tags = tuple(sorted((k, v) for k, v in dp.tags.items() if k != "window")) - return (dp.data_type, tuple(dp.tracers), float(dp.value), tags) - - -def _derive(s, parts, custody): - """A copy of ``s`` under its inputs' one stamp (and ``custody``'s, if given).""" - from . import blinding - - if not parts: - raise ValueError("a derivation needs its input parts") - stamps = [_custody.read_stamp(p.metadata) for p in parts] - stamp = stamps[0] - if any(st != stamp for st in stamps): - raise ValueError( - "input parts carry different custody stamps: " - + "; ".join(f"part {i}: {st.token}" for i, st in enumerate(stamps)) - ) - if custody is not None and custody != stamp: - raise ValueError( - f"parts are stamped {stamp.token}, but the catalogue is declared " - f"{custody.token}" - ) - - def born(x): - """Rows only a birth makes: shiftable ones, and under a blind custom types.""" - rules = [blinding.rule(dp.data_type) for dp in x.data] - return [ - i - for i, r in enumerate(rules) - if callable(r) or (stamp.blinded and r is None) - ] + from .blinding import conceal - inputs = {_row_key(p.data[i]) for p in parts for i in born(p)} - stray = [i for i in born(s) if _row_key(s.data[i]) not in inputs] - if stray: - raise ValueError( - f"{len(stray)} rows of a derivation are not copies of its inputs' " - "rows, but only a birth makes them (shiftable signal, and under a " - "blind any custom type): save them with custody=" - ) - out = s.copy() - _mint(out, stamp) + out = conceal(s, blind, theory) + out.metadata[STAMP_KEY] = blind.name return out -def save(s, path, *, custody=None, derived_from=None, theory=None): - """Write ``s`` to ``path`` (FITS) through the one door; return what was written. +def save(s, path, *, blind=None, theory=None, derived_from=None): + """Write ``s`` to ``path`` (FITS) and return what was written. - @sc one-door - The only writer of a SACC file, and with :func:`seal` the only place a - custody stamp is minted: + - ``save(s, path, blind=b)``: a birth. ``b`` is a + :class:`sp_validation.blinding.Blind`; unless it is ``none``, every row + of ``s`` is shifted by t(hidden) − t(fiducial), t being ``theory``. The + output is stamped with ``b``'s name. + - ``save(s, path, derived_from=parts)``: a derivation (COSEBIs, pure-E/B, + an assembly); the parts must share one :func:`stamp`, which ``s`` takes, + unshifted. With ``blind=b`` too, that stamp must be ``b``'s name. - - ``save(s, path, custody=c)``: a birth, sealed by :func:`seal`; - ``theory={data_type: f}`` shifts a type outside the standard estimators - under a blind (:mod:`sp_validation.blinding`); - - ``save(s, path, derived_from=parts)``: a derivation, stamped with its - inputs' one stamp; - - ``save(s, path, derived_from=parts, custody=c)``: an assembly, a - derivation whose stamp must also be ``c``'s. + ``theory(params, s)`` returns the prediction for every row of ``s``: an + array the length of ``s.mean``, in its order, zero where the cosmology has + no effect. ``params`` is a plain dict with the keys of + :func:`sp_validation.theory.fiducial`; the theory is called at the blind's + fiducial and hidden points, which differ only in ``S8`` and ``Omega_m``. + The default, :func:`sp_validation.theory.shear`, predicts ξ± and Cℓ_EE and + gives zeros for Cℓ_BB and Cℓ_EB; :func:`sp_validation.theory.none` gives + zeros throughout. A measurement computes its signal and saves (or seals) it in the same - function, returning the sealed part, so its raw signal never leaves it. - - @sc derived-inherit - A derivation's inputs must share one stamp, and each of its shiftable rows - (under a blind, each row of a custom type too) must be a copy of an input - row (type, tracers, value, tags; windows by index), so plaintext cannot be - saved under a concealed stamp. + function, returning the sealed part, so on a blinded catalogue its raw + signal never leaves that function. """ if derived_from is not None: - if theory is not None: - raise ValueError("theory= shifts a birth; a derivation inherits") - out = _derive(s, list(derived_from), custody) - elif custody is not None: - out = seal(s, custody, theory) + stamps = {stamp(p) for p in derived_from} + if len(stamps) != 1: + raise ValueError(f"input parts carry different blinds: {sorted(stamps)}") + (name,) = stamps + if blind is not None and blind.name != name: + raise ValueError( + f"parts are stamped {name}, but the catalogue is declared under " + f"{blind.name}" + ) + out = s.copy() + out.metadata[STAMP_KEY] = name + elif blind is not None: + out = seal(s, blind, theory) else: - raise ValueError( - "save needs custody= (a birth) or derived_from= (a derivation); " - "a SACC is never written unstamped" - ) + raise ValueError("save needs blind= (a birth) or derived_from= (a derivation)") out.save_fits(str(path), overwrite=True) return out def load(path): - """Load the SACC at ``path``, refusing a file without a valid custody stamp. - - @sc stamped-or-refused - Every file ``save`` wrote carries a custody stamp; anything else was - not born through the door and is refused, with no escape hatch. - """ - s = sacc.Sacc.load_fits(str(path)) - try: - _custody.read_stamp(s.metadata) - except _custody.CustodyError as err: - raise _custody.CustodyError(f"{path}: {err}") from None - return s + """Load the SACC at ``path``.""" + return sacc.Sacc.load_fits(str(path)) # ============================================================================= @@ -1318,12 +1237,10 @@ def sacc_to_twopoint_fits( Raises ------ ValueError - If the SACC carries no custody stamp (it was not born through - :func:`save`); if it has no ξ points; if ``n_bins != 1`` or its ξ tracer + If the SACC has no ξ points; if ``n_bins != 1`` or the SACC's ξ tracer pairs are anything other than exactly ``{(source_0, source_0)}`` (the single-bin contract); or if exactly one of the ρ/τ sidecars is supplied. """ - _custody.read_stamp(s.metadata) if (rho_stats_hdu is None) != (tau_stats_hdu is None): raise ValueError( "rho_stats_hdu and tau_stats_hdu must be supplied together " diff --git a/src/sp_validation/theory.py b/src/sp_validation/theory.py index 33a9c6c8..0b610eb1 100644 --- a/src/sp_validation/theory.py +++ b/src/sp_validation/theory.py @@ -1,28 +1,38 @@ -"""Theory data vectors: a prediction for the rows of a SACC, per data type. - -@sc theory-plugin -A theory is a function ``f(params, s, rows) -> values``: the prediction, at the -parameter point ``params``, for rows ``rows`` of SACC ``s``, which share a data -type's theory, a tracer pair and a bandpower window; values are aligned to -``rows``. ``params`` is a plain mapping with the keys of :func:`fiducial`, -never a CCL object, so an emulator can stand in for CCL. :func:`shear_xi` and -:func:`shear_cl` are the defaults :data:`sp_validation.blinding.STANDARD` -shifts ξ± and Cℓ_EE by. - -@sc theory-ccl-default -The defaults build one ``pyccl.Cosmology`` per point through -``cs_util.cosmo.get_cosmo``, on the CAMB HMCode2020-feedback route the CosmoSIS -inference runs, with σ8 = S8/√(Ωm/0.3). ``Omega_m`` is ``get_cosmo``'s -argument, from which it subtracts CAMB's Ω_ν for Ω_c. - -pyccl and cs_util are imported only when a default theory runs. +"""Theory data vectors: a prediction for every row of a SACC. + +A theory is a function ``theory(params, s) -> np.ndarray``: + +- ``params`` is a plain dict of cosmological parameters by name, with exactly + the keys of :func:`fiducial`: ``S8``, ``Omega_m``, ``Omega_b``, ``h``, + ``n_s``, ``m_nu``, ``w0``, ``wa``, ``logT_AGN`` and ``A_IA``. Blinding calls + the theory twice, at a blind's fiducial point and at the hidden point drawn + from it; the two differ only in ``S8`` and ``Omega_m``. +- ``s`` is the ``sacc.Sacc`` being blinded. Its rows, ``s.data``, each carry a + data type, a tracer pair and a θ or ℓ tag (Cℓ rows also a bandpower window, + ``s.get_bandpower_windows(rows)``); its tracers, ``s.tracers``, carry the + n(z). +- It returns an array the length of ``s.mean``, in the same order: the + prediction for each row, zero where the cosmology has no effect. + +For a SACC of Cℓ_EE/BB/EB bandpowers, a theory could read:: + + def my_theory(params, s): + out = np.zeros(len(s.mean)) + ee = s.indices(sacc_io.CL_EE) + window = s.get_bandpower_windows(ee) + cl = my_cl_ee(params, window.values) # Cℓ_EE on the window's ℓ + out[ee] = window.weight.T @ cl + return out # Cℓ_BB and Cℓ_EB stay zero + +:func:`shear` is the default, :func:`none` the theory of statistics with no +cosmological signal. """ import functools import numpy as np -from .sacc_io import XI_PLUS +from .sacc_io import CL_BB, CL_EB, CL_EE, XI_MINUS, XI_PLUS # Multipoles the ξ± Hankel transform integrates over: every ℓ below 50, # then 200 log-spaced up to 6·10⁴. @@ -45,8 +55,46 @@ def fiducial(): } +def none(params, s): + """Zeros: the theory of a statistic with no cosmological signal (ρ/τ).""" + return np.zeros(len(s.mean)) + + +def shear(params, s): + """Cosmic-shear ξ± and Cℓ_EE from pyccl; zeros for Cℓ_BB and Cℓ_EB. + + ξ± is evaluated at each row's stored θ (TreeCorr's ``meanr``) and Cℓ_EE + through each row's bandpower window. Any other data type raises: pass your + own theory. pyccl and cs_util are imported only when this runs. + """ + out = np.zeros(len(s.mean)) + groups = {} + for i, dp in enumerate(s.data): + if dp.data_type in (XI_PLUS, XI_MINUS): + key = (_xi, dp.tracers) + elif dp.data_type == CL_EE: + key = (_cl, dp.tracers, id(dp.tags.get("window"))) + elif dp.data_type in (CL_BB, CL_EB): + continue + else: + raise ValueError( + f"theory.shear has no prediction for {dp.data_type}; pass your " + "own theory" + ) + groups.setdefault(key, []).append(i) + # One call per tracer pair (and window), so ξ+ and ξ− share their Cℓ. + for (function, *_), rows in groups.items(): + out[rows] = function(params, s, np.asarray(rows)) + return out + + def cosmology(params): - """The ``pyccl.Cosmology`` at ``params``, built once per point.""" + """The ``pyccl.Cosmology`` at ``params``, built once per point. + + Built through ``cs_util.cosmo.get_cosmo`` on the CAMB HMCode2020-feedback + route the CosmoSIS inference runs, with σ8 = S8/√(Ωm/0.3); ``get_cosmo`` + subtracts CAMB's Ω_ν from ``Omega_m`` for Ω_c. + """ return _cosmology(tuple(float(params[k]) for k in _COSMOLOGY)) @@ -80,28 +128,17 @@ def _cosmology(point): return cosmo -def tag(s, rows, name): - """Tag ``name`` (or ``"data_type"``) of each of ``rows``.""" - return np.array( - [ - s.data[i].data_type if name == "data_type" else s.data[i].tags[name] - for i in rows - ] - ) - - -def nz(s, name): - """The ``(z, n(z))`` of tracer ``name``.""" - tracer = s.tracers[name] - if getattr(tracer, "nz", None) is None: - raise ValueError(f"tracer {name} has no n(z)") - return np.asarray(tracer.z, float), np.asarray(tracer.nz, float) +def _tag(s, rows, name): + return np.array([s.data[i].tags[name] for i in rows]) def _lensing(cosmo, params, s, name): import pyccl as ccl - z, n = nz(s, name) + tracer = s.tracers[name] + if getattr(tracer, "nz", None) is None: + raise ValueError(f"tracer {name} has no n(z)") + z, n = np.asarray(tracer.z, float), np.asarray(tracer.nz, float) ia = None if params["A_IA"] == 0 else (z, np.full_like(z, params["A_IA"])) return ccl.WeakLensingTracer(cosmo, dndz=(z, n), ia_bias=ia) @@ -114,42 +151,22 @@ def _shear_cl(params, s, rows, ell): return ccl.angular_cl(cosmo, a, b, ell) -def shear_xi(params, s, rows): - """ξ± of one shear pair: Limber C_ℓ on :data:`ELL`, then CCL's Hankel transform.""" +def _xi(params, s, rows): + """ξ± of one shear pair: Limber Cℓ on :data:`ELL`, then CCL's Hankel transform.""" import pyccl as ccl cosmo, cl = cosmology(params), _shear_cl(params, s, rows, ELL) - theta = tag(s, rows, "theta") / 60.0 + theta = _tag(s, rows, "theta") / 60.0 xip, xim = ( ccl.correlation(cosmo, ell=ELL, C_ell=cl, theta=theta, type=t) for t in ("GG+", "GG-") ) - return np.where(tag(s, rows, "data_type") == XI_PLUS, xip, xim) + plus = np.array([s.data[i].data_type == XI_PLUS for i in rows]) + return np.where(plus, xip, xim) -def shear_cl(params, s, rows): +def _cl(params, s, rows): """Cℓ_EE of one shear pair through its bandpower window.""" window = s.get_bandpower_windows(rows) cl = _shear_cl(params, s, rows, np.asarray(window.values, float)) return np.asarray(window.weight).T @ cl - - -def predict(s, params, theory, rows=None): - """The theory of ``rows`` (default: all) of ``s`` at ``params``. - - ``theory`` maps data type to function; a row without one raises. Rows are - grouped by function, tracer pair and bandpower window, one call per group, - so ξ+ and ξ− of a pair share their C_ℓ. - """ - rows = np.arange(len(s.data)) if rows is None else np.asarray(rows, int) - groups = {} - for n, i in enumerate(rows): - dp = s.data[i] - if dp.data_type not in theory: - raise ValueError(f"no theory for {dp.data_type}") - window = id(dp.tags.get("window")) - groups.setdefault((theory[dp.data_type], dp.tracers, window), []).append(n) - out = np.empty(len(rows)) - for (function, *_), at in groups.items(): - out[at] = function(params, s, rows[at]) - return out From f56b8842f74006a8611e707659bb07f7c0d50987 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 04:10:35 +0200 Subject: [PATCH 147/160] custody: each catalogue's declared blind and its shear file, nothing more An entry must declare blind: none or blind: ; entries reading one shear file declare one blind. The catalogue-path helpers stay here, stdlib-only for the host. Co-Authored-By: Claude Opus 5.5 --- cosmo_val/cat_config.yaml | 8 +- src/sp_validation/custody.py | 211 ++++------------------------------- 2 files changed, 28 insertions(+), 191 deletions(-) diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index a19357f2..5ac254b1 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -1,5 +1,5 @@ -# One entry per catalogue. `blind:` is its custody: `none` (public), `mock`, or -# the name of the blind its signal is concealed under. See custody.py. +# One entry per catalogue. `blind:` is `none` (public) or the name of the blind +# its signal is concealed under (sp_validation.blinding). DES: blind: none subdir: /n17data/mkilbing/astro/data/DES @@ -372,7 +372,7 @@ SP_v1.4.5_faint: path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits patch_number: 150 SP_v1.4.6_glass_mock: - blind: mock + blind: none subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP colour: darkgreen @@ -1194,7 +1194,7 @@ nz: path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_{pipeline}_A.txt paths: output: ./output - # The blind registry: .blind.json records, outside any git worktree. + # Where the .blind.json records live. blinds: /n17data/UNIONS/WL/blinds SP_v1.6.6: blind: none diff --git a/src/sp_validation/custody.py b/src/sp_validation/custody.py index 9a65bfc9..12bacfe1 100644 --- a/src/sp_validation/custody.py +++ b/src/sp_validation/custody.py @@ -1,101 +1,22 @@ -"""Custody: whether a catalogue's signal is blinded, and under which blind. +"""Which blind each catalogue is declared under, and where its shear file is. -@sc custody-is-declared -Every catalogue entry of the catalogue config declares ``blind: none``, -``blind: mock`` or ``blind: ``, and an entry without one is refused. -``_leak_corr`` and ``_seed`` versions take their entry's. Entries reading -one shear file declare one blind, the repository's ``cosmo_val/cat_config.yaml`` -included whatever config is passed; and a set of versions shown together may -hold a blinded catalogue only beside mocks and catalogues under the same blind. - -A blind is the record ``/.blind.json``, drawn by -:mod:`sp_validation.blinding`. Its commitment, the fork's -``seed_commitment`` of the whole canonical record, is public: it names the -blind in custody tokens, and a SACC's stamp is its custody token. +Every catalogue entry of the catalogue config declares ``blind: none`` or +``blind: `` (:mod:`sp_validation.blinding`); an entry without one is +refused. ``_leak_corr`` and ``_seed`` versions take their entry's, and +entries reading one shear file declare one blind. The host Snakemake loads this module by path, so it imports only the standard -library (and PyYAML, which Snakemake carries, for the repository config). +library. """ -import functools -import hashlib -import json import os import re -from dataclasses import dataclass, field -from pathlib import Path -NONE, MOCK = "none", "mock" NOT_CATALOGUES = ("nz", "paths") -REPO_CAT_CONFIG = Path(__file__).parents[2] / "cosmo_val" / "cat_config.yaml" -# smokescreen.COMMITMENT_DOMAIN, spelt here for the host; test_custody pins it. -COMMITMENT_DOMAIN = b"smokescreen-seed-commitment-v1|" -STAMP_KEY = "custody" -BLIND_NAME = re.compile(r"[a-z0-9][a-z0-9_.-]*") _SEED_SUFFIX = re.compile(r"_seed\d+$") _LEAK_SUFFIX = "_leak_corr" -class CustodyError(ValueError): - """A catalogue's custody cannot be resolved, or a stamp is not one.""" - - -@dataclass(frozen=True) -class Custody: - """``none``, ``mock`` or a blind's name, and that blind's commitment.""" - - blind: str - commitment: str | None = None - registry: Path | None = field(default=None, compare=False) - - @property - def blinded(self): - return self.blind not in (NONE, MOCK) - - @property - def token(self): - """One string that changes whenever the custody does.""" - return f"{self.blind}:{self.commitment}" if self.blinded else self.blind - - -def parse(token, catalogues=None): - """The custody a token (:attr:`Custody.token`) names, with the registry of - ``catalogues`` (a catalogue config) when it is blinded.""" - blind, _, commitment = token.partition(":") - custody = Custody(blind, commitment or None) - if custody.blinded and catalogues is not None: - return Custody(blind, commitment, registry(catalogues)) - return custody - - -def commitment(record): - """sha256 of the domain-prefixed canonical JSON of a blind's record.""" - text = json.dumps(record, sort_keys=True, separators=(",", ":")) - return hashlib.sha256(COMMITMENT_DOMAIN + text.encode("utf-8")).hexdigest() - - -@functools.cache -def _guard(path): - for parent in (path, *path.parents): - if (parent / ".git").exists(): - if parent != path: - raise CustodyError( - f"the blind registry {path} lies inside the git worktree " - f"{parent}; point paths.blinds outside it, or at the root of " - "a dedicated private blinds repository" - ) - break - return path - - -def registry(catalogues): - """The blind registry the catalogue config names (``paths.blinds``).""" - path = (catalogues.get("paths") or {}).get("blinds") - if not path or not os.path.isabs(os.path.expanduser(path)): - raise CustodyError(f"paths.blinds, {path!r}, must be an absolute path") - return _guard(Path(os.path.realpath(os.path.expanduser(path)))) - - def _variant_chain(version): """``version``, then each name that stripping ``_leak_corr``/``_seed`` leaves.""" chain = [version] @@ -116,7 +37,7 @@ def entry_name(catalogues, version): for name in _variant_chain(version): if name in catalogues and name not in NOT_CATALOGUES: return name - raise CustodyError(f"no catalogue {version} in the catalogue config") + raise ValueError(f"no catalogue {version} in the catalogue config") def seed_path(shear, seed): @@ -127,7 +48,7 @@ def seed_path(shear, seed): path = shear.get("path", "") match = re.match(r"(.*seed\D?)\d+", path) if match is None: - raise CustodyError(f"shear path {path!r} has no seed to put a seed in") + raise ValueError(f"shear path {path!r} has no seed to put a seed in") return match.group(1) + seed + path[match.end() :] @@ -155,120 +76,36 @@ def shear_file(catalogues, version): return _reads(entry) -def _blinds_by_file(catalogues): - by_file = {} - for name, entry in catalogues.items(): - if name not in NOT_CATALOGUES and (path := _reads(entry)): - by_file.setdefault(os.path.realpath(path), set()).add(entry.get("blind")) - return by_file - - -@functools.cache -def _repo_by_file(path, mtime): - import yaml - - return _blinds_by_file(yaml.safe_load(Path(path).read_text())) - - -def of_file(path): - """The blinds the repository config declares for shear file ``path``.""" - if not REPO_CAT_CONFIG.exists(): - return set() - by_file = _repo_by_file(str(REPO_CAT_CONFIG), REPO_CAT_CONFIG.stat().st_mtime_ns) - return by_file.get(os.path.realpath(path), set()) - - def declared(catalogues, version): - """The blind ``version`` is declared under: ``none``, ``mock`` or a name.""" + """The blind ``version`` is declared under: ``none`` or a blind's name.""" name = entry_name(catalogues, version) blind = catalogues[name].get("blind") - if blind is None: - raise CustodyError( - f"{name} declares no `blind:`; declare `blind: none` (public), " - "`blind: mock` or `blind: ` in its catalogue config entry" + if not isinstance(blind, str): + raise ValueError( + f"{name} declares no `blind:`; declare `blind: none` (public) or " + "`blind: ` in its catalogue config entry" ) - if not isinstance(blind, str) or not BLIND_NAME.fullmatch(blind): - raise CustodyError(f"{name} declares blind: {blind!r}, not a blind name") - try: - path = shear_file(catalogues, version) - except CustodyError: - return blind # a seed file no entry names: nothing else reads it - if path is None: - return blind - blinds = {blind} | _blinds_by_file(catalogues).get(os.path.realpath(path), set()) - blinds |= of_file(path) - if len(blinds) > 1: - raise CustodyError( - f"{version} reads {path}, which catalogue entries declare under " - f"{sorted(map(str, blinds))}; entries reading one file declare one blind " - f"(and {REPO_CAT_CONFIG} is authoritative for the files it names)" + path = _reads(catalogues[name]) + others = { + other: entry.get("blind") + for other, entry in catalogues.items() + if other not in NOT_CATALOGUES and path and _reads(entry) == path + } + if len(set(others.values())) > 1: + raise ValueError( + f"{path} is read by entries declaring different blinds: {others}" ) return blind -def custody_of(catalogues, version): - """The custody of ``version``: a blinded one reads its blind's commitment.""" - blind = declared(catalogues, version) - if blind in (NONE, MOCK): - return Custody(blind) - where = registry(catalogues) - try: - record = json.loads((where / f"{blind}.blind.json").read_text()) - except (OSError, ValueError): - fix = ( - f"no blind {blind} in {where}; `python -m sp_validation.blinding init " - f"{blind}` draws one, only if you mean to create it" - if where.is_dir() and os.access(where, os.R_OK | os.X_OK) - else f"get read access to {where}" - ) - raise CustodyError( - f"{version} is blinded under {blind}: {fix}. Setting `blind: none` " - "would unblind it." - ) from None - return Custody(blind, commitment(record), where) - - -def check_mix(blinds): - """Refuse ``{version: blind}`` showing a blinded catalogue beside another - real-sky catalogue not under its blind.""" - real = {v: b for v, b in blinds.items() if b != MOCK} - for version, blind in real.items(): - for other, theirs in real.items(): - if blind not in (NONE, theirs): - raise CustodyError( - f"{version} (blinded under {blind}) and {other} " - f"({'public' if theirs == NONE else f'blinded under {theirs}'}) " - "cannot be shown together: overlaying them shows the blind's shift" - ) - - def summary(catalogues, versions): - """One ``[custody]`` line per catalogue entry among ``versions``.""" + """One ``[blind]`` line per catalogue entry among ``versions``.""" by_entry = {} for version in dict.fromkeys(versions): by_entry.setdefault(entry_name(catalogues, version), []).append(version) lines = [] for name, members in by_entry.items(): - custody = custody_of(catalogues, name) also = [v for v in members if v != name] also = f" (+ {', '.join(also)})" if also else "" - state = f"blinded under {custody.blind}" if custody.blinded else custody.blind - lines.append(f"[custody] {name}{also}: {state}") + lines.append(f"[blind] {name}{also}: {declared(catalogues, name)}") return lines - - -def read_stamp(metadata): - """The custody a SACC's stamp (its ``custody`` token) records; a missing or - malformed stamp raises.""" - token = metadata.get(STAMP_KEY) - custody = parse(token) if isinstance(token, str) else None - if ( - custody is None - or not BLIND_NAME.fullmatch(custody.blind) - or custody.blinded != (custody.commitment is not None) - ): - raise CustodyError( - f"no valid custody stamp ({STAMP_KEY}={token!r}): every SACC is born " - "through sacc_io.save" - ) - return custody From 3dc1835f992d445caf747a9112f88f5bf3badd9b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 04:10:35 +0200 Subject: [PATCH 148/160] CosmologyValidation and the workflow save under the declared blind CosmologyValidation.blind(version) opens the blind its catalogue config declares; producer rules carry it as params.blind, so flipping it reruns what it touches, and assembly refuses parts stamped otherwise. A launch prints each catalogue's blind. Co-Authored-By: Claude Opus 5.5 --- src/sp_validation/cosmo_val/core.py | 35 ++++--------------- src/sp_validation/cosmo_val/pseudo_cl.py | 4 +-- .../cosmo_val/psf_systematics.py | 7 ++-- src/sp_validation/cosmo_val/real_space.py | 21 ++++------- workflow/common.py | 23 +++++------- workflow/rules/cosmo_val.smk | 2 +- workflow/rules/twopoint.smk | 6 ++-- workflow/scripts/assemble_sacc.py | 25 ++++++------- workflow/scripts/cv_cosebis.py | 2 +- workflow/scripts/cv_pure_eb.py | 4 +-- workflow/scripts/generate_pseudo_cl.py | 6 ---- workflow/scripts/run_2pcf.py | 9 ++--- workflow/scripts/run_rho_tau.py | 1 - 13 files changed, 47 insertions(+), 98 deletions(-) diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 0dd2a38f..55acf92e 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -9,6 +9,7 @@ from cs_util.cosmo import get_cosmo from shear_psf_leakage import run_object, run_scale +from .. import blinding from .. import custody as _custody from ..b_modes import ( _get_pte_from_scale_cut, @@ -131,10 +132,6 @@ class CosmologyValidation( noise debiasing, making those realizations reproducible run-to-run. cosmo_params : dict, optional Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. - custody : dict, optional - ``{version: custody token}`` (a workflow job's ``params.custody``): - the host's resolution of the catalogue config's declaration, whose - blind must match it. Attributes ---------- @@ -202,7 +199,6 @@ def __init__( cell_seed=8192, path_onecovariance=None, cosmo_params=None, - custody=None, ): self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method @@ -269,7 +265,6 @@ def __init__( self.cc = cc = yaml.load(file, Loader=yaml.FullLoader) # The catalogues as declared, before virtual versions are materialised. self._declared = copy.deepcopy(cc) - self._tokens = dict(custody or {}) def resolve_paths_for_version(ver): """Resolve relative paths for a version using its subdir.""" @@ -337,7 +332,6 @@ def ensure_version_exists(ver): final_versions.append(ver) self.versions = final_versions - _custody.check_mix({v: self._blind(v) for v in self.versions}) if output_dir is not None: cc["paths"]["output"] = output_dir @@ -453,27 +447,12 @@ def results_objectwise(self): self._results_objectwise = self.init_results(objectwise=True) return self._results_objectwise - def _blind(self, version): - """The blind the catalogue config declares for ``version``; a given - token naming another is refused.""" - blind = _custody.declared(self._declared, version) - if version in self._tokens: - given = _custody.parse(self._tokens[version]).blind - if given != blind: - raise _custody.CustodyError( - f"{version} was given custody {given!r}, but the catalogue " - f"config declares blind: {blind}" - ) - return blind - - def custody(self, version): - """The custody every SACC this object writes for ``version`` is sealed - under: its given token (whose commitment ``open_blind`` checks against - the record), else the catalogue config's declaration.""" - self._blind(version) - if version in self._tokens: - return _custody.parse(self._tokens[version], self._declared) - return _custody.custody_of(self._declared, version) + def blind(self, version): + """The blind every SACC this object writes for ``version`` is saved + under, as the catalogue config declares it.""" + return blinding.open_blind( + _custody.declared(self._declared, version), self._declared + ) def basename(self, version, treecorr_config=None, npatch=None): cfg = treecorr_config or self.treecorr_config diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index c3593b71..b52c3917 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -710,7 +710,7 @@ def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); the writer takes the shared bandpower window from ``wsp``. No covariance is attached here. Returns the part as sealed under the version's - custody. + blind. """ s = pseudo_cl_to_sacc( self.sacc_nz(version), @@ -719,7 +719,7 @@ def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): cl_all, wsp, ) - return sacc_io.save(s, out_path, custody=self.custody(version)) + return sacc_io.save(s, out_path, blind=self.blind(version)) def plot_pseudo_cl(self): """Plot the EE/EB/BB pseudo-Cl spectra for every version.""" diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index a4828189..61dba2f7 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -16,7 +16,7 @@ from shear_psf_leakage.rho_tau_stat import PSFErrorFit from uncertainties import ufloat -from .. import sacc_io +from .. import sacc_io, theory from ..rho_tau import ( get_rho_tau_w_cov, get_samples, @@ -54,7 +54,7 @@ def calculate_rho_tau_stats(self): def rho_tau_to_sacc_part( self, version, out_dir, base, rho_stat_handler, tau_stat_handler ): - """Write the ρ/τ SACC part for one version, under the version's custody. + """Write the ρ/τ SACC part for one version, under the version's blind. ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage from the handler tables. The ``CovTauTh`` theory covariance ``cov_tau_{base}_th.npy`` is passed as @@ -77,7 +77,8 @@ def rho_tau_to_sacc_part( tau_cov_th=tau_cov_th, ) out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") - sacc_io.save(s, out_path, custody=self.custody(version)) + # ρ/τ carry no cosmological signal: the blind leaves them unshifted. + sacc_io.save(s, out_path, blind=self.blind(version), theory=theory.none) @property def rho_stat_handler(self): diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 5e6ef277..a20a9729 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -29,14 +29,12 @@ def calculate_2pcf( out=None, **treecorr_config, ): - """ξ± of ``ver`` on one binning, as its SACC part sealed under its custody. + """ξ± of ``ver`` on one binning, as its SACC part sealed under its blind. - @sc signal-leaves-sealed A catalogue's ξ± leaves this object only as a part sealed under the - catalogue's custody (:func:`sp_validation.sacc_io.seal`): concealed - when it is blinded, before it is returned, cached or written. The - blind is opened first, so a blind that cannot open fails before - TreeCorr runs. + catalogue's blind (:func:`sp_validation.sacc_io.seal`), concealed + before it is returned, cached or written. The blind is opened first, + so a blind that cannot open fails before TreeCorr runs. With patches, the catalogue splits at the output tree's ``{ver}_patches_npatch={npatch}.dat``, drawn by TreeCorr's k-means and @@ -57,12 +55,7 @@ def calculate_2pcf( """ self.print_magenta(f"Computing {ver} ξ±") npatch = int(npatch or self.npatch) - custody = self.custody(ver) - if custody.blinded: - from .. import blinding - - blinding.open_blind(custody) - + blind = self.blind(ver) metadata = {**self.sacc_metadata(ver), "npatch": npatch} jackknife = npatch > 1 patch_file = self._output_path(f"{ver}_patches_npatch={npatch}.dat") @@ -106,9 +99,7 @@ def calculate_2pcf( covariance=gg.cov if jackknife else None, variances=None if jackknife else np.concatenate([gg.varxip, gg.varxim]), ) - part = ( - sacc_io.save(s, out, custody=custody) if out else sacc_io.seal(s, custody) - ) + part = sacc_io.save(s, out, blind=blind) if out else sacc_io.seal(s, blind) self.xi_parts[ver, grid] = part self.print_done("Done 2PCF") return part diff --git a/workflow/common.py b/workflow/common.py index 80560270..e6234a95 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -43,7 +43,7 @@ def _load_checkout_module(name): image_revision = _container.image_revision resolve_image = _container.resolve_image -# Catalogue custody, resolved as container jobs resolve it. +# Each catalogue's declared blind and shear file, resolved as container jobs do. _custody = _load_checkout_module("custody") # Every job inherits this launch's environment (the slurm executor submits with @@ -186,33 +186,28 @@ def configure(workflow_config): DEFAULT_MASK_SUFFIX = ( "_masked" if workflow_config["covariance"].get("default_masked", False) else "" ) - announce_custody(workflow_config) + announce_blinds(workflow_config) with open(COSMOLOGY_PARAMS) as f: PLANCK18 = json.load(f) -def announce_custody(workflow_config): - """Print each run catalogue's custody, or stop the launch with why it cannot run. - - Every version the config names must resolve (a blinded one, to its blind), - and the overlaid ``versions`` must pass the mixing rule. - """ +def announce_blinds(workflow_config): + """Print each run catalogue's blind, or stop the launch if one declares none.""" from snakemake.exceptions import WorkflowError versions = workflow_config.get("versions", []) fiducial = [FIDUCIAL.get(k) for k in ("version", "mock_version")] try: - _custody.check_mix({v: _custody.declared(CATALOG_CONFIG, v) for v in versions}) lines = _custody.summary(CATALOG_CONFIG, [*versions, *filter(None, fiducial)]) - except _custody.CustodyError as err: + except ValueError as err: raise WorkflowError(str(err)) from None print("\n".join(lines), file=sys.stderr) -def custody_token(version): - """``version``'s custody token: a producer's ``params.custody``, so a - custody change reruns it, and the custody its job seals under.""" - return _custody.custody_of(CATALOG_CONFIG, version).token +def blind_of(version): + """``version``'s declared blind: a producer's ``params.blind``, so flipping + it reruns exactly the jobs it touches.""" + return _custody.declared(CATALOG_CONFIG, version) def fiducial_binning_suffix(fiducial=None): diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 1ccb2bab..3f966ae8 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -508,7 +508,7 @@ rule assemble_sacc: sacc=cv_analysis_sacc("{version}"), params: version="{version}", - custody=lambda w: custody_token(w.version), + blind=lambda w: blind_of(w.version), # The statistics this rule wired, so a typo'd input keyword cannot # silently drop one. expected=lambda w: [ diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 82a43614..fbc5f151 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -33,7 +33,7 @@ rule xi: cat_config=CAT_CONFIG, output_dir=str(COSMO_VAL), grid=lambda w: grid_of(XI_GRIDS, w), - custody=lambda w: custody_token(w.version), + blind=lambda w: blind_of(w.version), resources: # The fine integration grid needs more memory and wall time than the # ~20-bin reporting one; scale on nbins rather than splitting the rule. @@ -59,7 +59,7 @@ rule rho_tau_stats: npatch="{npatch}", cat_config=CAT_CONFIG, output_dir=str(COSMO_VAL), - custody=lambda w: custody_token(w.version), + blind=lambda w: blind_of(w.version), resources: mem_mb=30000, disk_mb=20000, @@ -87,7 +87,7 @@ rule pseudo_cl: binning="{binning}", nbins=lambda w: int(w.nbins), power=0.5, - custody=lambda w: custody_token(w.version), + blind=lambda w: blind_of(w.version), resources: mem_mb=32000, runtime=120, diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 48f1f012..d756bce2 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -16,10 +16,9 @@ import argparse import numpy as np -import yaml -from sp_validation import custody as _custody from sp_validation import sacc_io +from sp_validation.blinding import Blind from sp_validation.cosmo_val.sacc_writers import assemble_analysis_sacc # NaMaster iNKA covariance FITS: per-spectrum HDU names, in SACC insertion order. @@ -93,17 +92,15 @@ def assemble_sacc( part_paths, out_path, *, - custody, + blind, expected=None, xi_cov=None, pseudo_cl_cov=None, ): """Assemble ``{version}.sacc`` from the per-statistic ``part_paths`` mapping. - @sc one-custody-per-assembly - Every part, signal-bearing or not, must carry one stamp, and it must be - ``custody``: a stale part after a custody flip, a part under another blind, - a mock in data and public parts in a blinded file are all refused. + Every part must be stamped with ``blind``, the catalogue's declared blind, + so a stale part from before a flip is refused. Parameters ---------- @@ -115,8 +112,8 @@ def assemble_sacc( expected : sequence of str, optional Statistics that must be present, from the caller's config toggles. A typo'd input keyword would otherwise silently drop a statistic. - custody : sp_validation.custody.Custody - The custody ``version`` runs under. + blind : str + The name of the blind ``version`` is declared under. xi_cov, pseudo_cl_cov Covariance sourcing — see the module docstring. """ @@ -143,7 +140,7 @@ def assemble_sacc( if not parts: raise ValueError(f"no parts found for {version}: {part_paths}") s = sacc_io.save( - assemble_analysis_sacc(parts), out_path, derived_from=parts, custody=custody + assemble_analysis_sacc(parts), out_path, derived_from=parts, blind=Blind(blind) ) print(f"Assembled {len(parts)} parts -> {out_path}") return s @@ -161,7 +158,7 @@ def _from_snakemake(smk): version=p["version"], part_paths=part_paths, out_path=str(smk.output[0]), - custody=_custody.parse(p["custody"]), + blind=p["blind"], expected=list(p["expected"]), xi_cov=getattr(inp, "xi_cov", None), pseudo_cl_cov=getattr(inp, "pseudo_cl_cov", None), @@ -175,9 +172,7 @@ def _from_cli(argv=None): ap.add_argument("--version", required=True, help="Catalogue version") ap.add_argument("--out", required=True, help="Output {version}.sacc path") ap.add_argument( - "--cat-config", - required=True, - help="cat_config.yaml declaring the catalogue's custody", + "--blind", required=True, help="The blind the catalogue is declared under" ) for name in CANONICAL: ap.add_argument( @@ -193,7 +188,7 @@ def _from_cli(argv=None): version=a.version, part_paths=part_paths, out_path=a.out, - custody=_custody.custody_of(yaml.safe_load(open(a.cat_config)), a.version), + blind=a.blind, xi_cov=a.xi_cov, pseudo_cl_cov=a.pseudo_cl_cov, ) diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index c0962afc..ee9d8aef 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -4,7 +4,7 @@ the same grid — nothing here touches a catalogue. The covariance goes through the same linear kernel as the modes to give the COSEBIs covariance; it is analytic, so no Hartlap debiasing applies. The COSEBIs part is a derivation of -the ξ± part and carries its custody. +the ξ± part and carries its blind stamp. """ import numpy as np diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index 61a5282d..a4d8e803 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -7,7 +7,7 @@ mean, so it is a function of the covariance model and the grids alone, never of the measured vector. A jackknife of the transformed modes would need per-patch realisations, which are never persisted. The pure-E/B part is a -derivation of the two ξ± parts and carries their custody. +derivation of the two ξ± parts and carries their blind stamp. """ import numpy as np @@ -58,7 +58,7 @@ n_samples=p["n_samples"], rng=np.random.default_rng(0), ) -# Written first, so parts under two custodies are refused before any product +# Written first, so parts under two blinds are refused before any product # of them exists. sacc_io.save( pure_eb_to_sacc({0: (z, nz)}, reporting.metadata, theta, modes, covariance=cov), diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index a548f488..bbf96a82 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -37,7 +37,6 @@ def generate_pseudo_cl( binning: str = "linear", nbins: int = None, power: float = 0.5, - custody: str = None, ): """Generate a pseudo-Cl data vector, born as a SACC part at ``out_path``. @@ -63,9 +62,6 @@ def generate_pseudo_cl( Number of ell bins (required) power : float Power for powspace binning (0.5 = sqrt spacing) - custody : str, optional - The custody token Snakemake resolved for ``version`` (the rule's - ``params.custody``), which the part is sealed under. Returns ------- @@ -119,7 +115,6 @@ def generate_pseudo_cl( theta_min=1.0, theta_max=250.0, nbins=20, - custody=None if custody is None else {version: custody}, ) if binning == "linear": cv_kwargs["ell_step"] = ell_step @@ -150,7 +145,6 @@ def _from_snakemake(smk): binning=p["binning"], nbins=int(p["nbins"]), power=float(p.get("power", 0.5)), - custody=p["custody"], ) diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 2f1f4fc4..2617610c 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -15,7 +15,7 @@ The measurement is binning-agnostic: the reporting and the fine integration grids are the same compute with different ``--min-sep/--max-sep/--nbins``. The ξ± is born as a SACC part, named by its binning, tagged with its ``--grid`` and -sealed under the catalogue's custody by ``CosmologyValidation.calculate_2pcf``. +sealed under the catalogue's blind by ``CosmologyValidation.calculate_2pcf``. ``output_dir`` is passed explicitly so lc can point each run at its own ``{output}`` tree. @@ -37,7 +37,6 @@ def run_2pcf( output_dir, sacc_out=None, grid="reporting", - custody=None, ): """Measure ξ±(θ) for ``ver`` and write its sealed SACC part. @@ -47,9 +46,7 @@ def run_2pcf( the catalog configuration; ``output_dir`` overrides ``cat_config['paths']['output']``. ``sacc_out`` is the exact destination for the part (the Snakemake-declared output); it defaults to a binning-derived - name under the resolved output directory for the CLI path. ``custody`` is - the custody token Snakemake resolved for ``ver`` (the rule's - ``params.custody``), which the part is sealed under. + name under the resolved output directory for the CLI path. Returns ------- @@ -60,7 +57,6 @@ def run_2pcf( versions=[ver], catalog_config=cat_config, output_dir=output_dir, - custody=None if custody is None else {ver: custody}, ) out_path = sacc_out or os.path.join( output_dir or cv.cc["paths"]["output"], @@ -91,7 +87,6 @@ def _from_snakemake(smk): output_dir=p["output_dir"], grid=p.get("grid", "reporting"), sacc_out=smk.output["sacc"], - custody=p["custody"], ) diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index 4334a742..4983cffb 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -43,7 +43,6 @@ npatch=int(params["npatch"]), catalog_config=params["cat_config"], output_dir=params["output_dir"], - custody={params["ver"]: params["custody"]}, ) cv.calculate_rho_tau_stats() From 019c06bc2cadb89e3ef932f29b025e922feb46ad Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 04:10:36 +0200 Subject: [PATCH 149/160] Pure E/B takes its covariance from the CosmoCov integration-grid covariance calculate_pure_eb requires cov_path_int; the interactive jackknife path is gone. Co-Authored-By: Claude Opus 5.5 --- src/sp_validation/b_modes.py | 138 +++++++------------------ src/sp_validation/cosmo_val/pure_eb.py | 39 +++---- 2 files changed, 55 insertions(+), 122 deletions(-) diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index 01743b66..711133c7 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -127,10 +127,10 @@ def correlation_from_covariance(covariance): def calculate_pure_eb_correlation( gg, gg_int, - cov_path_int=None, - cosmo_cov=None, + cov_path_int, + cosmo_cov, + z_dist, n_samples=1000, - z_dist=None, ): """ Calculate pure E/B modes from ξ± on the reporting and integration grids. @@ -140,18 +140,18 @@ def calculate_pure_eb_correlation( gg : sacc_io.xi_correlation view ξ± on the reporting binning (coarser binning for final results) gg_int : sacc_io.xi_correlation view - ξ± on the integration binning (fine binning for numerical integration); - without ``cov_path_int``, its jackknife covariance is propagated - (:func:`pure_eb_covariance_from_xi`) - cov_path_int : str, optional - Path to integration covariance matrix for semi-analytical calculation - cosmo_cov : pyccl.Cosmology, optional - Cosmology for theoretical predictions in semi-analytical covariance - n_samples : int, optional - Number of Monte Carlo samples for semi-analytical covariance - z_dist : 2D array, optional + ξ± on the integration binning (fine binning for numerical integration) + cov_path_int : str + Path to the integration-grid ξ± covariance (CosmoCov), from which the + modes' covariance is drawn by Monte Carlo + (:func:`pure_eb_covariance_mc`) + cosmo_cov : pyccl.Cosmology + Cosmology for the theory mean of the Monte Carlo draws + z_dist : 2D array Redshift distribution; z_dist[:, 0] = z, z_dist[:, 1] = n(z) + n_samples : int, optional + Number of Monte Carlo samples for semi-analytical covariance Returns ------- @@ -171,7 +171,7 @@ def calculate_pure_eb_correlation( "theta_int": gg_int.meanr, "xip_int": gg_int.xip, "xim_int": gg_int.xim, - "n_eff": n_samples if cov_path_int is not None else gg_int.npatch1, + "n_eff": n_samples, } results.update( pure_eb_from_xi( @@ -186,33 +186,18 @@ def calculate_pure_eb_correlation( ) ) - if cov_path_int is not None: - if z_dist is None or cosmo_cov is None: - raise ValueError( - "semi-analytical covariance needs both z_dist and cosmo_cov" - ) - cov, eb_samples = pure_eb_covariance_mc( - theta=gg.meanr, - left_edges=gg.left_edges, - right_edges=gg.right_edges, - theta_int=gg_int.meanr, - cov_int=np.loadtxt(cov_path_int), - z=z_dist[:, 0], - nz=z_dist[:, 1], - cosmo=cosmo_cov, - n_samples=n_samples, - ) - results.update({"cov": cov, "eb_samples": eb_samples}) - else: - results["cov"] = pure_eb_covariance_from_xi( - theta=gg.meanr, - left_edges=gg.left_edges, - right_edges=gg.right_edges, - theta_int=gg_int.meanr, - xi_int=np.concatenate([gg_int.xip, gg_int.xim]), - weight_int=gg_int.weight, - cov_int=gg_int.cov, - ) + cov, eb_samples = pure_eb_covariance_mc( + theta=gg.meanr, + left_edges=gg.left_edges, + right_edges=gg.right_edges, + theta_int=gg_int.meanr, + cov_int=np.loadtxt(cov_path_int), + z=z_dist[:, 0], + nz=z_dist[:, 1], + cosmo=cosmo_cov, + n_samples=n_samples, + ) + results.update({"cov": cov, "eb_samples": eb_samples}) # Validate covariance matrix try: @@ -288,7 +273,18 @@ def pure_eb_covariance_mc( """ theta, theta_int = np.asarray(theta), np.asarray(theta_int) nbins_int = len(theta_int) - binning_matrix = _reporting_binning(left_edges, right_edges, theta_int) + + # Each reporting bin averages the integration bins that fall inside it. + reporting_bin_edges = np.concatenate([left_edges, [right_edges[-1]]]) + bin_indices = np.digitize(theta_int, reporting_bin_edges) - 1 + valid_mask = (bin_indices >= 0) & (bin_indices < len(theta)) + row_indices, col_indices = (bin_indices[valid_mask], np.where(valid_mask)[0]) + binning_matrix = sparse.csr_matrix( + (np.ones(len(row_indices)), (row_indices, col_indices)), + shape=(len(theta), nbins_int), + ) + row_sums = np.array(binning_matrix.sum(axis=1)).flatten() + binning_matrix = sparse.diags(1 / row_sums) @ binning_matrix # One n(z) gives one tracer pair: get_theo_xi's single (xi+, xi-) entry. (xi_pm,) = get_theo_xi( @@ -323,64 +319,6 @@ def eb_draw(i): return np.cov(eb_samples.T), eb_samples -def _reporting_binning(left_edges, right_edges, theta_int, weight_int=None): - """The matrix averaging integration-grid ξ± into each reporting bin. - - Each reporting bin averages the integration bins whose centres fall inside - it, weighted by ``weight_int`` (their pair weights: TreeCorr's estimate on - the reporting bin), or uniformly. - """ - edges = np.concatenate([left_edges, [right_edges[-1]]]) - rows = np.digitize(theta_int, edges) - 1 - inside = (rows >= 0) & (rows < len(left_edges)) - weight = np.ones(len(theta_int)) if weight_int is None else np.asarray(weight_int) - matrix = sparse.csr_matrix( - (weight[inside], (rows[inside], np.flatnonzero(inside))), - shape=(len(left_edges), len(theta_int)), - ) - return sparse.diags(1 / np.asarray(matrix.sum(axis=1)).ravel()) @ matrix - - -def pure_eb_covariance_from_xi( - *, theta, left_edges, right_edges, theta_int, xi_int, weight_int, cov_int -): - """Pure-E/B covariance T·C·Tᵀ from the integration-grid ξ± covariance C. - - The modes are linear in ξ±, and the reporting ξ± is the pair-weighted - average of the integration ξ± inside each reporting bin, so T is the - kernel composed with that average. Each column u of a factor C = Σ u uᵀ - is pushed through as K(ξ + u) − K(ξ), about the measured ``xi_int`` - ([ξ+, ξ−]), as a jackknife pushes each patch's ξ±: the kernel's - quadrature is accurate on a ξ± shaped like a measurement, not on a bare - fluctuation. A jackknife C has rank below its patch count, which bounds - the kernel calls. - """ - values, vectors = np.linalg.eigh(np.asarray(cov_int)) - keep = values > values.max() * len(values) * np.finfo(float).eps - factor = vectors[:, keep] * np.sqrt(values[keep]) - binning = _reporting_binning(left_edges, right_edges, theta_int, weight_int) - nbins_int = len(theta_int) - - def transformed(xi): - xip_int, xim_int = xi[:nbins_int], xi[nbins_int:] - modes = pure_eb_from_xi( - theta_report=theta, - xip_report=binning @ xip_int, - xim_report=binning @ xim_int, - theta_int=theta_int, - xip_int=xip_int, - xim_int=xim_int, - tmin=left_edges[0], - tmax=right_edges[-1], - ) - return _eb_vector(modes) - - xi_int = np.asarray(xi_int) - centre = transformed(xi_int) - columns = np.column_stack([transformed(xi_int + u) - centre for u in factor.T]) - return columns @ columns.T - - def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): """ Calculate COSEBIs modes from a correlation function for multiple scale cuts. diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 52ca5e45..17743d07 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -59,12 +59,11 @@ def calculate_pure_eb( nbins_int : int, optional Number of bins for the integration binning. Defaults to 1000. npatch : int, optional - Number of jackknife patches. Defaults to the value in self.npatch if - not provided. - cov_path_int : str, optional - Path to the integration-grid ξ± covariance. When given, the - covariance is Monte Carlo through the kernel from it, instead of the - jackknife. + Number of patches of the ξ± measurements. Defaults to the value in + self.npatch if not provided. + cov_path_int : str + Path to the integration-grid ξ± covariance (CosmoCov), required: + the modes' covariance is Monte Carlo through the kernel from it. cosmo_cov : pyccl.Cosmology, optional Cosmology object to use for theoretical xi+/xi- predictions in the semi-analytical covariance calculation. Defaults to self.cosmo if not @@ -90,20 +89,21 @@ def calculate_pure_eb( - "xip", "xim", "var_xip", "var_xim": Reporting-grid xi+/xi- and their variances. - "theta_int", "xip_int", "xim_int": Integration-grid xi+/xi-. - - "n_eff": Realisation count behind "cov" (jackknife patches or - MC draws), which sets the Hartlap debiasing. - - "eb_samples": (only when using semi-analytical covariance) Semi-analytic - EB samples used for covariance calculation. Shape: (n_samples, 6*nbins) + - "n_eff": MC draws behind "cov", which set the Hartlap debiasing. + - "eb_samples": Semi-analytic EB samples used for covariance + calculation. Shape: (n_samples, 6*nbins) Notes ----- - Both binnings are the version's sealed ξ± parts - (:meth:`calculate_2pcf`), split at the same patch centres, so a - blinded catalogue's modes are concealed. The jackknife covariance is - the integration part's, pushed through the kernel - (:func:`~sp_validation.b_modes.pure_eb_covariance_from_xi`): the - shift, the same in every patch, leaves it unchanged. + (:meth:`calculate_2pcf`), so a blinded catalogue's modes are + concealed. """ + if cov_path_int is None: + raise ValueError( + "calculate_pure_eb needs cov_path_int, the CosmoCov " + "integration-grid ξ± covariance" + ) self.print_start(f"Computing {version} pure E/B") gg, gg_int = ( @@ -123,19 +123,14 @@ def calculate_pure_eb( ) ) - # Get redshift distribution if using analytic covariance - z_dist = ( - np.column_stack(self.get_redshift(version)) - if cov_path_int is not None - else None - ) + z_dist = np.column_stack(self.get_redshift(version)) # Delegate to b_modes module results = calculate_pure_eb_correlation( gg=gg, gg_int=gg_int, cov_path_int=cov_path_int, - cosmo_cov=cosmo_cov, + cosmo_cov=cosmo_cov or self.cosmo, n_samples=n_samples, z_dist=z_dist, ) From 3ed0218b940fa7fac5115f414ea013431aaa6ebf Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 04:10:37 +0200 Subject: [PATCH 150/160] Remove scripts/xip_xim.py and scripts/map2.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in the repository calls them; ξ± and ⟨M_ap²⟩ live in cosmo_val. Co-Authored-By: Claude Opus 5.5 --- scripts/map2.py | 151 ------------------------------------- scripts/xip_xim.py | 182 --------------------------------------------- 2 files changed, 333 deletions(-) delete mode 100755 scripts/map2.py delete mode 100755 scripts/xip_xim.py diff --git a/scripts/map2.py b/scripts/map2.py deleted file mode 100755 index 361fe48e..00000000 --- a/scripts/map2.py +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env python3 - -"""map2.py - -Compute aperture-mass dispersion using ``treecorr``. -Requires input file with xi+ and xi- previously -computed with ``treecorr``. - -:Author: Martin Kilbinger - -""" - -import sys -from optparse import OptionParser - -import numpy as np -import treecorr -from cs_util import logging - - -def params_default(): - - params = { - "input_path": "xip_xim.txt", - "theta_min": 0.5, - "theta_max": 200, - "n_theta": 20, - "n_Theta": 10, - "output_path": "./map2.txt", - } - - short_options = { - "input_path": "-i", - "output_path": "-o", - } - - types = { - "theta_min": "float", - "theta_max": "float", - "n_theta": "int", - "n_Theta": "int", - } - - help_strings = { - "input_path": "input file containing xi+ and xi-, default={}", - "theta_min": "mininum angular scale [arcmin], default={}", - "theta_max": "maximum angular scale [arcmin], default={}", - "n_theta": "number of angular scales on input, default={}", - "n_Theta": "number of angular scales on output, default={}", - "output_path": "output path, default={}", - } - - return params, short_options, types, help_strings - - -def parse_options(p_def, short_options, types, help_strings): - """Parse command line options. - - Parameters - ---------- - p_def : dict - default parameter values - help_strings : dict - help strings for options - - Returns - ------- - options: tuple - Command line options - """ - - usage = "%prog [OPTIONS]" - parser = OptionParser(usage=usage) - - for key in p_def: - if key in help_strings: - if key in short_options: - short = short_options[key] - else: - short = "" - - if key in types: - typ = types[key] - else: - typ = "string" - - parser.add_option( - short, - f"--{key}", - dest=key, - type=typ, - default=p_def[key], - help=help_strings[key].format(p_def[key]), - ) - - parser.add_option( - "-v", "--verbose", dest="verbose", action="store_true", help="verbose output" - ) - - options, args = parser.parse_args() - - return options - - -def main(argv=None): - - params, short_options, types, help_strings = params_default() - - options = parse_options(params, short_options, types, help_strings) - - # Update parameter values - for key in vars(options): - params[key] = getattr(options, key) - - # Save calling command - logging.log_command(argv) - - # Initizlies correlation object - gg = treecorr.GGCorrelation( - min_sep=params["theta_min"], - max_sep=params["theta_max"], - bin_size=params["n_theta"], - ) - - # Open input catalogue - if params["verbose"]: - print(f"Reading xi+ and xi- input file {params['input_path']}...") - gg.read(params["input_path"]) - - # Set up angular smoothing scales on output - R = np.geomspace( - params["theta_min"] * 5, - params["theta_max"] / 2, - params["n_Theta"], - ) - - # Compute correlation - if params["verbose"]: - print("Computing aperture mass dispersion...") - gg.calculateMapSq(R, m2_uform="Schneider") - - # Write to file - if params["verbose"]: - print(f"Writing output file {params['output_path']}") - gg.writeMapSq(params["output_path"], R=R, m2_uform="Schneider") - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/xip_xim.py b/scripts/xip_xim.py deleted file mode 100755 index 9713745c..00000000 --- a/scripts/xip_xim.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python3 - -"""xip_xim.py - -Compute shear correlation functions using ``treecorr``. - -:Author: Martin Kilbinger - -""" - -import sys -from optparse import OptionParser - -import treecorr -from astropy.io import fits -from cs_util import logging - -from sp_validation.custody import MOCK, NONE, of_file - - -def params_default(): - - params = { - "input_path": "shape_catalog_ngmix.fits", - "key_ra": "RA", - "key_dec": "DEC", - "key_e1": "e1", - "key_e2": "e2", - "sign_e1": +1, - "sign_e2": +1, - "theta_min": 0.5, - "theta_max": 200, - "n_theta": 20, - "output_path": "./xip_xim.txt", - } - - short_options = { - "input_path": "-i", - "output_path": "-o", - } - - types = { - "sign_e1": "int", - "sign_e2": "int", - } - - help_strings = { - "input_path": "shear catalogue input path, default={}", - "key_ra": "column name for right ascension, default={}", - "key_dec": "column name for declination, default={}", - "key_e1": "column name for ellipticity component 1, default={}", - "key_e2": "column name for ellipticity component 2, default={}", - "sign_e1": "sign for ellipticity component 1, default={}", - "sign_e2": "sign for ellipticity component 2, default={}", - "theta_min": "mininum angular scale [arcmin], default={}", - "theta_max": "maximum angular scale [arcmin], default={}", - "n_theta": "number of angular scales, default={}", - "output_path": "output path, default={}", - } - - return params, short_options, types, help_strings - - -def parse_options(p_def, short_options, types, help_strings): - """Parse command line options. - - Parameters - ---------- - p_def : dict - default parameter values - help_strings : dict - help strings for options - - Returns - ------- - options: tuple - Command line options - """ - - usage = "%prog [OPTIONS]" - parser = OptionParser(usage=usage) - - for key in p_def: - if key in help_strings: - if key in short_options: - short = short_options[key] - else: - short = "" - - if key in types: - typ = types[key] - else: - typ = "string" - - parser.add_option( - short, - f"--{key}", - dest=key, - type=typ, - default=p_def[key], - help=help_strings[key].format(p_def[key]), - ) - - parser.add_option( - "-v", "--verbose", dest="verbose", action="store_true", help="verbose output" - ) - - options, args = parser.parse_args() - - return options - - -def main(argv=None): - - params, short_options, types, help_strings = params_default() - - options = parse_options(params, short_options, types, help_strings) - - # Update parameter values - for key in vars(options): - params[key] = getattr(options, key) - - # Save calling command - logging.log_command(argv) - - blinds = of_file(params["input_path"]) - {NONE, MOCK} - if blinds: - raise SystemExit( - f"{params['input_path']} is blinded under {', '.join(sorted(blinds))}: " - "measure its signal through CosmologyValidation or the workflow" - ) - - # Open input catalogue - if params["verbose"]: - print(f"Reading catalogue {params['input_path']}...") - data = fits.getdata(params["input_path"]) - - coord_units = "degrees" - if params["verbose"]: - print( - "Signs for ellipticity components =" - + f" ({params['sign_e1']:+d}, {params['sign_e2']:+d})" - ) - g1 = data[params["key_e1"]] * params["sign_e1"] - g2 = data[params["key_e2"]] * params["sign_e2"] - cat = treecorr.Catalog( - ra=data[params["key_ra"]], - dec=data[params["key_dec"]], - g1=g1, - g2=g2, - w=data["w"], - ra_units=coord_units, - dec_units=coord_units, - ) - - # Set treecorr config info for correlation - sep_units = "arcmin" - TreeCorrConfig = { - "ra_units": coord_units, - "dec_units": coord_units, - "min_sep": params["theta_min"], - "max_sep": params["theta_max"], - "sep_units": sep_units, - "nbins": params["n_theta"], - } - gg = treecorr.GGCorrelation(TreeCorrConfig) - - # Compute correlation - if params["verbose"]: - print("Correlating...") - gg.process(cat, cat) - - # Write to file - if params["verbose"]: - print(f"Writing output file {params['output_path']}") - gg.write(params["output_path"]) - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) From 3e22d4ae21f23f2f98b8d7af4dfc84135e959b56 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 04:10:47 +0200 Subject: [PATCH 151/160] Tests for the theory-vector blinding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit conceal moves each row by t(hidden) − t(fiducial), a custom theory blinds a custom type, theory.none stamps without shifting, derivations inherit one stamp, catalogues declare their blind, B-modes respond to the shift only through each transform; DAG tests for the launch line and a blind flip. Co-Authored-By: Claude Opus 5.5 --- src/sp_validation/tests/_synthetic.py | 38 +- src/sp_validation/tests/conftest.py | 8 +- src/sp_validation/tests/test_architecture.py | 48 --- src/sp_validation/tests/test_assemble_sacc.py | 19 +- src/sp_validation/tests/test_blinding.py | 337 +++++++----------- .../tests/test_blinding_bmodes.py | 82 ++--- src/sp_validation/tests/test_cosmo_val.py | 86 ++--- src/sp_validation/tests/test_custody.py | 144 +------- src/sp_validation/tests/test_custody_e2e.py | 152 -------- .../tests/test_cv_init_params.py | 4 +- src/sp_validation/tests/test_masks.py | 7 - src/sp_validation/tests/test_pseudo_cl.py | 2 +- src/sp_validation/tests/test_sacc_io.py | 13 +- .../tests/test_sacc_io_realdata.py | 2 - .../tests/test_sacc_io_twopoint.py | 17 +- src/sp_validation/tests/test_sacc_writers.py | 11 +- src/sp_validation/tests/test_survey.py | 2 - workflow/tests/conftest.py | 30 +- workflow/tests/test_dag.py | 64 +--- workflow/tests/test_toy_run.py | 22 +- 20 files changed, 279 insertions(+), 809 deletions(-) delete mode 100644 src/sp_validation/tests/test_architecture.py delete mode 100644 src/sp_validation/tests/test_custody_e2e.py diff --git a/src/sp_validation/tests/_synthetic.py b/src/sp_validation/tests/_synthetic.py index cae90b6a..72f63f1f 100644 --- a/src/sp_validation/tests/_synthetic.py +++ b/src/sp_validation/tests/_synthetic.py @@ -2,33 +2,33 @@ The glue tests run the real compute seams on it: a shear catalogue (RA/Dec/e1/e2/w), a PSF star catalogue with the columns the leakage and ρ/τ -seams read, a cs_util-readable dndz, and a ``cat_config.yaml`` whose blind -registry is ``blinds/`` beside it. Every catalogue entry in the config reads its -own copy of the same galaxies and declares its own custody. +seams read, a cs_util-readable dndz, and a ``cat_config.yaml`` whose blinds +live in ``blinds/`` beside it. Every catalogue entry in the config reads its +own copy of the same galaxies and declares its own blind. -``TOY_STANDARD`` is :data:`sp_validation.blinding.STANDARD` with the analytic -``toy_theory`` in place of each default theory, so a blind shifts without -running CAMB. +``toy_shear`` stands in for :func:`sp_validation.theory.shear`, so a blind +shifts without running CAMB. """ import copy -from types import MappingProxyType import numpy as np import yaml -from sp_validation import blinding +from sp_validation import sacc_io +SIGNAL = (sacc_io.XI_PLUS, sacc_io.XI_MINUS, sacc_io.CL_EE) -def toy_theory(params, s, rows): - """A power law in θ or ℓ whose amplitude grows with S8 and Ωm.""" - x = np.array([s.data[i].tags.get("theta", s.data[i].tags.get("ell")) for i in rows]) - return 1e-4 * params["S8"] ** 2 * params["Omega_m"] ** 0.3 * (x / 10.0) ** -0.8 - -TOY_STANDARD = MappingProxyType( - {t: toy_theory if callable(r) else r for t, r in blinding.STANDARD.items()} -) +def toy_shear(params, s): + """A power law in θ or ℓ for ξ± and Cℓ_EE, its amplitude growing with S8 + and Ωm; zeros for every other row.""" + out = np.zeros(len(s.mean)) + for i, dp in enumerate(s.data): + if dp.data_type in SIGNAL: + x = dp.tags.get("theta", dp.tags.get("ell")) + out[i] = params["S8"] ** 2 * params["Omega_m"] ** 0.3 * (x / 10.0) ** -0.8 + return 1e-4 * out def write_synthetic_catalogs( @@ -55,8 +55,8 @@ def write_synthetic_catalogs( catalogues : dict, optional ``{version: blind}``, one catalogue entry each, reading its own copy of the shear catalogue. ``blind`` is the entry's ``blind:`` value, or - ``None`` for an entry that declares none. Defaults to one mock, - ``TestCatalog``. + ``None`` for an entry that declares none. Defaults to one public + catalogue, ``TestCatalog``. Returns ------- @@ -66,7 +66,7 @@ def write_synthetic_catalogs( """ from astropy.table import Table - catalogues = catalogues or {"TestCatalog": "mock"} + catalogues = catalogues or {"TestCatalog": "none"} rng = np.random.default_rng(seed) cat_dir = tmp_path / "catalog" diff --git a/src/sp_validation/tests/conftest.py b/src/sp_validation/tests/conftest.py index b420ee63..07b81665 100644 --- a/src/sp_validation/tests/conftest.py +++ b/src/sp_validation/tests/conftest.py @@ -21,9 +21,9 @@ def pure_eb_xi(): @pytest.fixture def toy_theory(monkeypatch): - """Blinds shift by the analytic ``_synthetic.toy_theory``.""" - from _synthetic import TOY_STANDARD + """Blinds shift by the analytic ``_synthetic.toy_shear``.""" + from _synthetic import toy_shear - from sp_validation import blinding + from sp_validation import theory - monkeypatch.setattr(blinding, "STANDARD", TOY_STANDARD) + monkeypatch.setattr(theory, "shear", toy_shear) diff --git a/src/sp_validation/tests/test_architecture.py b/src/sp_validation/tests/test_architecture.py deleted file mode 100644 index c3be2605..00000000 --- a/src/sp_validation/tests/test_architecture.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Architecture on the resolved code tree: every SACC file passes one door. - -``sacc_io.save`` seals and stamps what it writes and ``sacc_io.load`` refuses -what it did not, so no other module may write or read a SACC file directly. -""" - -import ast -from pathlib import Path - -import pytest - -REPO = Path(__file__).resolve().parents[3] -TESTS = Path(__file__).resolve().parent - - -def _sources(): - """Every Python file of the package and the workflow, outside the tests.""" - for top in ( - "src/sp_validation", - "workflow", - "scripts", - "papers", - "cosmo_inference", - ): - for path in (REPO / top).rglob("*.py"): - parts = path.relative_to(REPO).parts - if ( - TESTS not in path.parents - and "tests" not in parts - and ".snakemake" not in parts - ): - yield path - - -@pytest.mark.parametrize("method", ["save_fits", "load_fits"]) -def test_sacc_files_pass_one_door(method): - """[one-door] Only sacc_io writes or reads a SACC file.""" - door = REPO / "src" / "sp_validation" / "sacc_io.py" - callers = sorted( - str(path.relative_to(REPO)) - for path in _sources() - if path != door - for node in ast.walk(ast.parse(path.read_text(), filename=str(path))) - if isinstance(node, ast.Call) - and isinstance(node.func, ast.Attribute) - and node.func.attr == method - ) - assert not callers, callers diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py index 5f974ca4..5c340e24 100644 --- a/src/sp_validation/tests/test_assemble_sacc.py +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -18,8 +18,8 @@ import pytest from sp_validation import sacc_io as sio +from sp_validation.blinding import NONE from sp_validation.cosmo_val import sacc_writers as sw -from sp_validation.custody import Custody def _load_assemble_module(): @@ -52,7 +52,6 @@ def _theta(n=6): META = {"catalogue_version": "vSYNTH", "npatch": 1} -MOCK = Custody("mock") def _xi_cov_txt(tmp_path, n=12, seed=21): @@ -159,7 +158,7 @@ def get_bandpower_windows(self): paths = {} for name, part in parts.items(): p = tmp_path / f"{name}.sacc" - sio.save(part, str(p), custody=MOCK) + sio.save(part, str(p), blind=NONE) paths[name] = str(p) return paths @@ -175,7 +174,7 @@ def test_assemble_sacc_canonical_order(tmp_path): "vSYNTH", paths, str(out), - custody=MOCK, + blind="none", xi_cov=cov_path, pseudo_cl_cov=cl_cov_path, ) @@ -224,7 +223,7 @@ def test_injected_xi_covariance_replaces_the_parts_own(tmp_path): "vSYNTH", paths, str(out), - custody=MOCK, + blind="none", xi_cov=cov_path, pseudo_cl_cov=cl_cov_path, ) @@ -244,7 +243,7 @@ def test_assemble_sacc_injects_pseudo_cl_covariance(tmp_path): out = tmp_path / "vSYNTH.sacc" s = asm.assemble_sacc( - "vSYNTH", paths, str(out), custody=MOCK, xi_cov=cov_path, pseudo_cl_cov=cov_fits + "vSYNTH", paths, str(out), blind="none", xi_cov=cov_path, pseudo_cl_cov=cov_fits ) tr = ("source_0", "source_0") cl_idx = np.concatenate( @@ -265,7 +264,7 @@ def test_missing_injected_covariance_raises(tmp_path): paths = _write_parts(tmp_path, cov_less=()) # every part born with a block out = tmp_path / "vSYNTH.sacc" with pytest.raises(ValueError, match="takes its analysis covariance from"): - asm.assemble_sacc("vSYNTH", paths, str(out), custody=MOCK) + asm.assemble_sacc("vSYNTH", paths, str(out), blind="none") def test_assemble_sacc_respects_pseudo_cl_toggle(tmp_path): @@ -274,7 +273,7 @@ def test_assemble_sacc_respects_pseudo_cl_toggle(tmp_path): assert "pseudo_cl" not in paths cov_path, _xi_cov = _xi_cov_txt(tmp_path) out = tmp_path / "vSYNTH.sacc" - s = asm.assemble_sacc("vSYNTH", paths, str(out), custody=MOCK, xi_cov=cov_path) + s = asm.assemble_sacc("vSYNTH", paths, str(out), blind="none", xi_cov=cov_path) tr = ("source_0", "source_0") assert len(s.indices(sio.CL_EE, tr)) == 0 # Round-trips as a valid BlockDiagonalCovariance over the remaining points. @@ -296,7 +295,7 @@ def test_assemble_sacc_expected_part_missing_raises(tmp_path): "vSYNTH", paths, str(out), - custody=MOCK, + blind="none", expected=["xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau"], xi_cov=cov_path, ) @@ -312,7 +311,7 @@ def test_assemble_sacc_expected_rejects_unknown_name(tmp_path): "vSYNTH", paths, str(out), - custody=MOCK, + blind="none", expected=["cosebi"], xi_cov=cov_path, ) diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 23a0b548..171f2a59 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -1,56 +1,37 @@ -"""The blind and the file door. +"""Blinds, concealment and the blind stamp. -A blind is drawn once (``blinding init``) into a registry outside git; -``sacc_io.save`` is the only writer, and conceals a blinded catalogue's -shiftable rows in memory before the file exists. Blinds here shift by the -analytic ``toy_theory``; one slow test runs the CCL defaults. +Blinds here shift by the analytic ``toy_shear``; one slow test runs the CCL +default. """ -import dataclasses +import json import stat import numpy as np import pytest -from _synthetic import TOY_STANDARD, toy_theory +from _synthetic import toy_shear +from smokescreen.param_shifts import draw_param_shifts from sp_validation import blinding as bd -from sp_validation import custody as cu from sp_validation import sacc_io as sio from sp_validation import theory -VERSIONS = ("TOY", "OTHER", "TOY_OPEN", "TOY_MOCK") -DEFAULTS = {t: f for t, f in bd.STANDARD.items() if callable(f)} -CUSTOM = "galaxy_density_xi" +DELTA_SIGMA = "galaxy_shearDensity_xi_t" -@pytest.fixture(scope="module", autouse=True) -def _toy_theory(): - with pytest.MonkeyPatch.context() as m: - m.setattr(bd, "STANDARD", TOY_STANDARD) - yield +@pytest.fixture(scope="module") +def catalogues(tmp_path_factory): + return {"paths": {"blinds": str(tmp_path_factory.mktemp("blinds"))}} -def _catalogues(root): - """A catalogue config declaring one catalogue per custody.""" - blinds = { - "TOY": "toy", - "OTHER": "other", - "TOY_OPEN": "none", - "TOY_MOCK": "mock", - } - config = {"paths": {"blinds": str(root / "blinds")}} - for name, blind in blinds.items(): - config[name] = {"shear": {"path": str(root / f"{name}.fits")}, "blind": blind} - return config +@pytest.fixture(scope="module") +def toy(catalogues): + return bd.init("toy", catalogues) @pytest.fixture(scope="module") -def blinds(tmp_path_factory): - """The four custodies: TOY under blind `toy`, OTHER under `other`.""" - cats = _catalogues(tmp_path_factory.mktemp("registry")) - for name in ("toy", "other"): - bd.init(name, cats) - return {v: cu.custody_of(cats, v) for v in VERSIONS} +def other(catalogues): + return bd.init("other", catalogues) def _nz(z0): @@ -58,31 +39,17 @@ def _nz(z0): return z, np.exp(-0.5 * ((z - z0) / 0.2) ** 2) -def part( - *, - xi_tags=("reporting",), - cl=True, - rho=False, - derived=None, - unruled=False, -): - """A two-bin part: ξ± under each of ``xi_tags`` (None: no tag) and - pseudo-Cℓ (EE, BB, EB) on pairs (0,0), (0,1), (1,1), and optionally ρ, - a derived statistic's rows and rows of a custom type.""" +def part(*, cl=True, rho=False, delta_sigma=False): + """ξ± and pseudo-Cℓ (EE, BB, EB) on pairs (0,0), (0,1), (1,1), and + optionally ρ and a ΔΣ-like custom type.""" s = sio.new_sacc({0: _nz(0.5), 1: _nz(0.9)}) theta = np.geomspace(2.0, 200.0, 6) ell = np.array([30.0, 80.0, 150.0, 280.0, 450.0]) w_ell = np.arange(2, 501).astype(float) window = np.exp(-0.5 * ((w_ell[:, None] - ell[None, :]) / 40.0) ** 2) for pair in ((0, 0), (0, 1), (1, 1)): - for tag in xi_tags: - xip, xim = 1e-4 * (theta / 10) ** -0.6, 0.5e-4 * (theta / 10) ** -0.9 - if tag is not None: - sio.add_xi(s, pair, theta, xip, xim, grid=tag) - else: - for dtype, values in ((sio.XI_PLUS, xip), (sio.XI_MINUS, xim)): - for th, v in zip(theta, values): - s.add_data_point(dtype, sio._pair(pair), v, theta=th) + xip, xim = 1e-4 * (theta / 10) ** -0.6, 0.5e-4 * (theta / 10) ** -0.9 + sio.add_xi(s, pair, theta, xip, xim, grid="reporting") if cl: ee = 1e-8 * (ell / 100.0) ** -1.2 sio.add_pseudo_cl( @@ -97,186 +64,152 @@ def part( ) if rho: sio.add_rho(s, 0, theta, np.arange(1, 7) * 1e-7, np.arange(1, 7) * 2e-7) - if derived == "cosebis": - sio.add_cosebis(s, (0, 0), np.arange(1, 6) * 1e-10, (12.0, 83.0), Bn=np.ones(5)) - elif derived == "pure_eb": - sio.add_pure_eb(s, (0, 0), theta[:4], **{k: np.ones(4) for k in sio.PURE_KEYS}) - if unruled: - for x in (5.0, 20.0, 80.0): - s.add_data_point(CUSTOM, ("source_0", "source_0"), 1e-4, theta=x) + if delta_sigma: + for r in (0.5, 2.0, 8.0): # tagged theta, as sacc requires + s.add_data_point(DELTA_SIGMA, ("source_0", "source_1"), 10.0, theta=r) s.add_covariance(np.abs(np.asarray(s.mean)) ** 2 + 1e-20) return s -def _values(s): - return np.asarray(s.mean) +def hidden(blind): + """The blind's hidden point, as Smokescreen draws it.""" + record = blind.record() + shift = draw_param_shifts(record["envelope"], record["seed"]) + return {k: v + shift.get(k, 0.0) for k, v in record["fiducial"].items()} # --------------------------------------------------------------------------- # -# Born sealed: a blind shifts exactly the ξ± and Cℓ_EE rows, nothing else +# Concealing: every row moves by its theory's t(hidden) − t(fiducial) # --------------------------------------------------------------------------- # -EVERYTHING = dict(xi_tags=("reporting", "mystery", None), rho=True) - - -@pytest.mark.parametrize( - "content, version, refused", - [ - (EVERYTHING, "TOY", False), - (EVERYTHING, "TOY_OPEN", False), - (EVERYTHING, "TOY_MOCK", False), - (dict(derived="cosebis"), "TOY", True), - (dict(derived="pure_eb"), "TOY", True), - (dict(unruled=True), "TOY", True), - (dict(derived="pure_eb", unruled=True), "TOY_OPEN", False), - ], -) -def test_seal_shifts_only_the_signal_it_has_a_rule_for( - blinds, content, version, refused -): - """Under a blind every ξ± and Cℓ_EE row moves, whatever its grid tag, and - every other value (Cℓ_BB/EB, ρ) and the covariance stay bitwise; a blinded - birth carrying derived rows or a custom type without a theory is refused. - Unblinded and mock births keep their values. Each is stamped with its - custody.""" - s, custody = part(**content), blinds[version] - if refused: - with pytest.raises(ValueError, match="derived_from|no blinding rule"): - sio.seal(s, custody) - return - sealed = sio.seal(s, custody) - assert cu.read_stamp(sealed.metadata) == custody - shiftable = np.array([callable(bd.STANDARD.get(dp.data_type)) for dp in s.data]) - moved = _values(sealed) != _values(s) - assert np.array_equal(moved, shiftable & custody.blinded) - assert np.array_equal(sealed.covariance.dense, s.covariance.dense) - - -def test_nothing_is_written_unstamped(blinds, tmp_path): - """``save`` needs a custody or its inputs; ``load`` refuses a file born - outside the door.""" - with pytest.raises(ValueError, match="custody"): - sio.save(part(), tmp_path / "x.sacc") - assert not (tmp_path / "x.sacc").exists() - part().save_fits(str(tmp_path / "raw.sacc")) - with pytest.raises(ValueError, match="stamp"): - sio.load(tmp_path / "raw.sacc") +def test_conceal_moves_signal_rows_by_the_theory_shift(toy, other): + """ξ± and Cℓ_EE move by t(hidden) − t(fiducial); Cℓ_BB/EB and the + covariance stay identical; the same blind gives the same shift, another + blind another.""" + s = part() + shift = bd.conceal(s, toy, toy_shear).mean - s.mean + fiducial = toy.record()["fiducial"] + expected = toy_shear(hidden(toy), s) - toy_shear(fiducial, s) + np.testing.assert_allclose(shift, expected, rtol=1e-10, atol=0) + signal = np.isin([dp.data_type for dp in s.data], [sio.XI_PLUS, sio.XI_MINUS]) + signal |= np.array([dp.data_type == sio.CL_EE for dp in s.data]) + assert np.all(shift[signal] != 0) and np.all(shift[~signal] == 0) + concealed = bd.conceal(s, toy, toy_shear) + np.testing.assert_array_equal(concealed.covariance.dense, s.covariance.dense) + np.testing.assert_array_equal(concealed.mean - s.mean, shift) + assert not np.allclose(bd.conceal(s, other, toy_shear).mean - s.mean, shift) + + +def test_a_theory_failure_names_no_value(toy): + """Any failure, including a vector of the wrong length, is a + BlindingError carrying neither the hidden point nor the original message.""" + + def broken(params, s): + raise RuntimeError(f"S8 = {params['S8']}") + + with pytest.raises(bd.BlindingError) as err: + bd.conceal(part(), toy, broken) + assert "S8" not in str(err.value) and "RuntimeError" in str(err.value) + assert err.value.__cause__ is None and err.value.__context__ is None + with pytest.raises(bd.BlindingError, match="values for"): + bd.conceal(part(), toy, lambda params, s: np.zeros(3)) + + +def test_a_custom_theory_blinds_a_custom_type(toy): + """A ΔΣ-like type is shifted by a theory that knows it; the default + theory refuses it; theory.none leaves every value but stamps.""" + + def delta_sigma(params, s): + out = toy_shear(params, s) + for i, dp in enumerate(s.data): + if dp.data_type == DELTA_SIGMA: + out[i] = 10.0 * params["S8"] / dp.tags["theta"] + return out + + s = part(cl=False, delta_sigma=True) + rows = s.indices(DELTA_SIGMA) + sealed = sio.seal(s, toy, delta_sigma) + assert np.all(sealed.mean[rows] != s.mean[rows]) + assert sio.stamp(sealed) == "toy" + with pytest.raises(bd.BlindingError, match="ValueError"): + sio.seal(s, toy) # theory.shear: no prediction for this type + rho = part(cl=False, rho=True) + kept = sio.seal(rho, toy, theory.none) + np.testing.assert_array_equal(kept.mean, rho.mean) + assert sio.stamp(kept) == "toy" + + +def test_none_stamps_without_concealing(tmp_path): + s = part() + saved = sio.save(s, tmp_path / "x.sacc", blind=bd.NONE) + np.testing.assert_array_equal(saved.mean, s.mean) + assert sio.stamp(sio.load(tmp_path / "x.sacc")) == "none" + assert sio.stamp(part()) == "none" # absent reads as none + with pytest.raises(ValueError, match="blind= .* or derived_from="): + sio.save(s, tmp_path / "y.sacc") # --------------------------------------------------------------------------- # -# Derived: a derivation carries its inputs' one stamp, never plaintext +# Derivations inherit their inputs' one stamp # --------------------------------------------------------------------------- # -@pytest.fixture(scope="module") -def parts(blinds): - """A ξ± part born under each custody.""" - return {v: sio.seal(part(cl=False), blinds[v]) for v in VERSIONS} - - @pytest.mark.parametrize( - "inputs, declared, content, allowed", + "inputs, declared, allowed", [ - (["TOY"], None, "copy", True), - (["TOY"], None, "cosebis", True), - (["TOY_OPEN"], None, "plaintext", True), - (["TOY", "TOY"], "TOY", "copy", True), # an assembly - (["TOY"], None, "plaintext", False), # laundering - (["OTHER"], "TOY", "copy", False), # another blind - (["TOY_OPEN"], "TOY", "copy", False), # unblinded into blinded - (["TOY", "OTHER"], None, "cosebis", False), # mixed stamps - (["TOY_MOCK", "TOY_OPEN"], None, "copy", False), # mixed stamps + (["toy"], None, True), + (["toy", "toy"], "toy", True), # an assembly + (["none"], None, True), + (["toy", "other"], None, False), # mixed stamps + (["toy", "none"], None, False), + (["none"], "toy", False), # a stale public part under a blind + (["other"], "toy", False), ], ) def test_a_derivation_carries_its_inputs_one_stamp( - blinds, parts, tmp_path, inputs, declared, content, allowed + toy, other, tmp_path, inputs, declared, allowed ): - """``save(s, derived_from=inputs, custody=declared)`` writes ``s`` under - the inputs' stamp exactly when they share one, it is the declared - custody's (if any), and, under a blind, ``s``'s ξ± rows are copies of - theirs. Otherwise nothing is written.""" - s = { - "copy": lambda: parts[inputs[0]].copy(), - "cosebis": lambda: part(xi_tags=(), cl=False, derived="cosebis"), - "plaintext": lambda: part(cl=False), - }[content]() + blinds = {"toy": toy, "other": other, "none": bd.NONE} + parts = [sio.seal(part(cl=False), blinds[b], toy_shear) for b in inputs] path = tmp_path / "d.sacc" - kwargs = dict( - derived_from=[parts[v] for v in inputs], - custody=blinds[declared] if declared else None, - ) + kwargs = dict(derived_from=parts, blind=bd.Blind(declared) if declared else None) if not allowed: with pytest.raises(ValueError): - sio.save(s, path, **kwargs) + sio.save(parts[0], path, **kwargs) assert not path.exists() return - sio.save(s, path, **kwargs) - loaded = sio.load(path) - assert cu.read_stamp(loaded.metadata) == blinds[inputs[0]] - assert np.array_equal(_values(loaded), _values(s)) + derived = sio.save(parts[0], path, **kwargs) + assert sio.stamp(sio.load(path)) == inputs[0] + np.testing.assert_array_equal(derived.mean, parts[0].mean) # --------------------------------------------------------------------------- # -# The blind: drawn once, opened only under its commitment +# The blind: drawn once, opened by name # --------------------------------------------------------------------------- # -def test_a_blind_is_drawn_once_and_kept_private(tmp_path): - cats = _catalogues(tmp_path) - blind = bd.init("toy", cats) +def test_a_blind_is_drawn_once_and_opened_by_name(tmp_path): + catalogues = {"paths": {"blinds": str(tmp_path / "blinds")}} + blind = bd.init("toy", catalogues) assert stat.S_IMODE(blind.path.stat().st_mode) == 0o440 - assert stat.S_IMODE(blind.path.parent.stat().st_mode) == 0o700 - with pytest.raises(cu.CustodyError, match="drawn once"): - bd.init("toy", cats) - - -def test_a_blind_opens_only_under_its_commitment(blinds, monkeypatch): - custody = blinds["TOY"] - assert bd.open_blind(custody) == bd.Blind( - "toy", custody.registry / "toy.blind.json" - ) - with pytest.raises(cu.CustodyError, match="launch again"): - bd.open_blind(dataclasses.replace(custody, commitment="0" * 64)) - monkeypatch.setattr(bd, "draw_scheme", lambda: 99) - with pytest.raises(cu.CustodyError, match="draw scheme 2; .* under 99"): - bd.open_blind(custody) - - -# --------------------------------------------------------------------------- # -# The shift: from each data type's theory, refused where it cannot be made -# --------------------------------------------------------------------------- # -def test_a_custom_type_is_shifted_by_its_own_theory(blinds, tmp_path): - """Under a blind a custom data type is shifted when its birth passes a - theory function and refused without one; its sealed rows then pass an - assembly. A standard type the blind leaves alone takes no theory.""" - custody, s = blinds["TOY"], part(cl=False, unruled=True) - with pytest.raises(ValueError, match="no blinding rule"): - sio.save(s, tmp_path / "refused.sacc", custody=custody) - sealed = sio.save( - s, tmp_path / "custom.sacc", custody=custody, theory={CUSTOM: toy_theory} - ) - rows = s.indices(CUSTOM) - assert np.all(_values(sealed)[rows] != _values(s)[rows]) - sio.save(sealed.copy(), tmp_path / "assembled.sacc", derived_from=[sealed]) - with pytest.raises(ValueError, match="no theory"): - sio.seal(part(), custody, theory={sio.CL_BB: toy_theory}) - - -def test_conceal_refuses_a_theory_blind_to_cosmology(blinds): - """A theory that ignores the cosmology is refused rather than stamped as - blinded.""" - - def flat(params, s, rows): - return np.ones(len(rows)) - - blind = bd.open_blind(blinds["TOY"]) - with pytest.raises(bd.BlindingError, match="unmoved"): - bd.conceal(part(cl=False), blind, theory={sio.XI_PLUS: flat}) + assert set(json.loads(blind.path.read_text())) == {"seed", "envelope", "fiducial"} + with pytest.raises(bd.BlindingError, match="exists"): + bd.init("toy", catalogues) + assert bd.open_blind("toy", catalogues) == blind + assert bd.open_blind("none", {}) is bd.NONE + with pytest.raises(bd.BlindingError, match="init gone"): + bd.open_blind("gone", catalogues) @pytest.mark.slow -def test_the_ccl_defaults_shift_with_s8(blinds, monkeypatch): - """The default theories, at a hidden point with S8 above the fiducial, - raise ξ+ and Cℓ_EE.""" - fiducial = theory.fiducial() - monkeypatch.setattr(bd, "_hidden", lambda blind: {**fiducial, "S8": 0.9}) - blind = bd.open_blind(blinds["TOY"]) +def test_the_ccl_default_shifts_with_s8(tmp_path): + """theory.shear, at a hidden point with S8 above the fiducial, raises ξ+ + and Cℓ_EE and leaves Cℓ_BB/EB.""" + record = { + "seed": "s", + "envelope": {"S8": [0.05, 0.06]}, + "fiducial": theory.fiducial(), + } + path = tmp_path / "up.blind.json" + path.write_text(json.dumps(record)) s = part() - delta = bd.conceal(s, blind, theory=DEFAULTS).mean - s.mean + delta = bd.conceal(s, bd.Blind("up", path)).mean - s.mean for data_type in (sio.XI_PLUS, sio.CL_EE): assert np.all(delta[s.indices(data_type)] > 0), data_type + for data_type in (sio.CL_BB, sio.CL_EB): + assert np.all(delta[s.indices(data_type)] == 0), data_type diff --git a/src/sp_validation/tests/test_blinding_bmodes.py b/src/sp_validation/tests/test_blinding_bmodes.py index 49b4379a..203e358b 100644 --- a/src/sp_validation/tests/test_blinding_bmodes.py +++ b/src/sp_validation/tests/test_blinding_bmodes.py @@ -1,13 +1,14 @@ """B-modes under a blind, on synthetic ξ±. -A blind shifts ξ± by one E-mode signal, so it moves COSEBIs B_n and pure-E/B -ξ_B only by each transform's response to a pure E-mode vector. The input is the -default theory's ξ± at the fiducial on the production reporting and integration -grids, noise-free and with one seeded shape-noise draw, sealed under a blind -whose hidden point is each corner of the envelope in turn. The transform's -response to δ must stay under a tenth of a bin's standard deviation; pure-E/B's -additivity, B(x + δ) − B(x) = B(δ), holds noise-free to its quadrature's floor, -measured at ≤ 10⁻³σ and asserted at 10⁻²σ. +A blind shifts ξ± by one E-mode signal, δ = t(hidden) − t(fiducial), so it +moves COSEBIs B_n and pure-E/B ξ_B only by each transform's response to a pure +E-mode vector. The input is the default theory's ξ± at the fiducial on the +production reporting and integration grids, noise-free and (for COSEBIs) with +one seeded shape-noise draw, and δ is taken with the hidden point at each +corner of the envelope in turn. The transform's response to δ must stay under +a tenth of a bin's standard deviation; pure-E/B's additivity, +B(x + δ) − B(x) = B(δ), holds noise-free to its quadrature's floor, measured +at ≤ 10⁻³σ and asserted at 10⁻²σ. """ import numpy as np @@ -15,7 +16,6 @@ from sp_validation import b_modes, theory from sp_validation import blinding as bd -from sp_validation import custody as cu from sp_validation import sacc_io as sio REPORTING = (1.0, 250.0, 20) @@ -51,7 +51,7 @@ def _fiducial_xi(): s = sio.new_sacc({0: (z, np.exp(-(((z - 0.7) / 0.3) ** 2)))}) for theta, grid in ((THETA, "reporting"), (THETA_INT, "integration")): sio.add_xi(s, (0, 0), theta, 0 * theta, 0 * theta, grid=grid, theta_nom=theta) - for dp, value in zip(s.data, theory.predict(s, theory.fiducial(), bd.STANDARD)): + for dp, value in zip(s.data, theory.shear(theory.fiducial(), s)): dp.value = float(value) s.add_covariance(VARIANCE) return s @@ -64,33 +64,24 @@ def _vector(s): @pytest.fixture(scope="module") -def shifts(tmp_path_factory): - """Each input's ξ± and, per envelope corner, its sealed ξ±'s shift.""" - root = tmp_path_factory.mktemp("bmodes") - catalogues = { - "paths": {"blinds": str(root / "blinds")}, - "SYN": {"shear": {"path": str(root / "SYN.fits")}, "blind": "syn"}, - } - bd.init("syn", catalogues) - custody = cu.custody_of(catalogues, "SYN") - clean = _fiducial_xi() - noisy = clean.copy() - noise = np.random.default_rng(3).normal(0.0, np.sqrt(VARIANCE)) - for dp, n in zip(noisy.data, noise): - dp.value += float(n) +def shifts(): + """Each input's ξ± and the blind's shift δ at each envelope corner.""" + s = _fiducial_xi() + clean = _vector(s) + noisy = clean + np.random.default_rng(3).normal(0.0, np.sqrt(VARIANCE)) fiducial = theory.fiducial() corners = [ {**fiducial, "S8": fiducial["S8"] + a, "Omega_m": fiducial["Omega_m"] + b} for a in (-bd.ENVELOPE["S8"], bd.ENVELOPE["S8"]) for b in (-bd.ENVELOPE["Omega_m"], bd.ENVELOPE["Omega_m"]) ] - out = {"noise-free": (_vector(clean), []), "noisy": (_vector(noisy), [])} - with pytest.MonkeyPatch.context() as m: - for corner in corners: - m.setattr(bd, "_hidden", lambda blind, corner=corner: corner) - for name, s in (("noise-free", clean), ("noisy", noisy)): - out[name][1].append(_vector(sio.seal(s, custody)) - out[name][0]) - return out + deltas = [] + for corner in corners: + shifted = s.copy() + for dp, d in zip(shifted.data, theory.shear(corner, s) - s.mean): + dp.value += float(d) + deltas.append(_vector(shifted) - clean) + return {"noise-free": (clean, deltas), "noisy": (noisy, deltas)} def _as_b(delta): @@ -162,30 +153,11 @@ def test_cosebis_b_modes_move_only_by_the_transforms_response(shifts, cosebis, n assert np.max(np.abs(b_n(_as_b(delta))) / sigma) > 10 * CEILING -@pytest.mark.parametrize( - "noise", - [ - "noise-free", - pytest.param( - "noisy", - marks=pytest.mark.xfail( - strict=True, - reason="pure-eb-bmode-numerics: on a noisy 1000-bin ξ± " - "the pure-E/B transform is not additive: an E-mode δ moves ξ_B by " - "0.1-1.2σ in isolated bins across noise draws and corners, unchanged by " - "tightening epsabs/epsrel from 1e-10 to 1e-13, so blinded and " - "unblinded pure-E/B alike carry this error", - ), - ), - ], -) -def test_pure_eb_b_modes_move_only_by_the_transforms_response( - shifts, sigma_pure_b, noise -): - """ΔB = B(x + δ) − B(x) equals B(δ), the kernel's B response to a pure - E-mode δ, to the quadrature's floor, and B(δ) stays under the ceiling; - δ's pure-B counterpart moves B well past it.""" - x, deltas = shifts[noise] +def test_pure_eb_b_modes_move_only_by_the_transforms_response(shifts, sigma_pure_b): + """On noise-free ξ±, ΔB = B(x + δ) − B(x) equals B(δ), the kernel's B + response to a pure E-mode δ, to the quadrature's floor, and B(δ) stays + under the ceiling; δ's pure-B counterpart moves B well past it.""" + x, deltas = shifts["noise-free"] before = _pure_b(x) for delta in deltas: moved = _pure_b(x + delta) - before diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index e90d7d2a..de79e6b4 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -388,10 +388,8 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( ``pure_eb_from_xi`` of those ξ± and edges, and every reporting bin is finite. ``test_b_modes`` pins the transform itself on the same ξ±, so a failure names the step that moved: measurement, wiring or transform. - Its jackknife covariance, the integration part's pushed through the - kernel, matches TreeCorr's jackknife of the modes in each statistic's - total variance, loosely: on this sparse toy ξ± the kernel's ξ− - quadrature is additive to only ~30%. + The covariance comes from the integration-grid covariance file, which + it requires; its Monte Carlo is stubbed here. Finiteness: the Schneider (2022) integrals are near-singular where a reporting bin meets the integration boundary, so the integration grid @@ -413,6 +411,7 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( npatch = 8 nbins = 6 + nbins_int = 600 cv = CosmologyValidation( versions=[version], npatch=npatch, @@ -423,22 +422,26 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( ) cv.treecorr_config.update(bin_slop=0, angle_slop=0) - import treecorr + grids = dict(npatch=npatch, min_sep_int=1.0, max_sep_int=300.0) + with pytest.raises(ValueError, match="cov_path_int"): + cv.calculate_pure_eb(version, nbins_int=nbins_int, **grids) - kernel = b_modes.pure_eb_from_xi - process, correlations = treecorr.GGCorrelation.process, [] - monkeypatch.setattr( - treecorr.GGCorrelation, - "process", - lambda gg, *a, **kw: correlations.append(gg) or process(gg, *a, **kw), - ) + cov_int = np.diag(np.full(2 * nbins_int, 1e-10)) + cov_path = tmp_path / "cov_int.txt" + np.savetxt(cov_path, cov_int) + seen = {} + + def mc(**kwargs): + seen.update(kwargs) + return np.eye(6 * nbins), np.zeros((kwargs["n_samples"], 6 * nbins)) + + monkeypatch.setattr(b_modes, "pure_eb_covariance_mc", mc) results = cv.calculate_pure_eb( - version, - npatch=npatch, - min_sep_int=1.0, - max_sep_int=300.0, - nbins_int=600, + version, nbins_int=nbins_int, cov_path_int=str(cov_path), **grids ) + np.testing.assert_array_equal(seen["cov_int"], cov_int) + np.testing.assert_array_equal(results["cov"], np.eye(6 * nbins)) + assert results["n_eff"] == seen["n_samples"] measured = { "theta_report": results["theta"], @@ -463,36 +466,6 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog( assert np.all(np.isfinite(vec)), f"{key} not finite" np.testing.assert_allclose(vec, modes[key], rtol=1e-10, err_msg=key) - # TreeCorr's jackknife of the modes, from the two measurements. - def modes_of(pair): - gg, gg_int = pair - return b_modes._eb_vector( - kernel( - gg.meanr, - gg.xip, - gg.xim, - gg_int.meanr, - gg_int.xip, - gg_int.xim, - gg.left_edges[0], - gg.right_edges[-1], - ) - ) - - reference = treecorr.estimate_multi_cov( - correlations, "jackknife", func=modes_of, cross_patch_weight="match" - ) - cov = np.asarray(results["cov"]) - assert cov.shape == reference.shape == (6 * nbins, 6 * nbins) - - def block_variances(c): - return np.diag(c).reshape(6, nbins).sum(axis=1) - - np.testing.assert_allclose( - block_variances(cov), block_variances(reference), rtol=0.4 - ) - assert results["n_eff"] == npatch - # --------------------------------------------------------------------------- # # I14: a blinded catalogue's ξ± leaves CosmologyValidation only concealed @@ -510,36 +483,29 @@ def blinded_and_twin(tmp_path, toy_theory): coherent_shear=True, catalogues={"TOY": "toy", "TOY_OPEN": "none"}, ) - catalogues = yaml.safe_load(open(params["catalog_config"])) - blinding.init("toy", catalogues) + blinding.init("toy", yaml.safe_load(open(params["catalog_config"]))) grid = dict(npatch=1, theta_min=5.0, theta_max=60.0, nbins=6, **params) return ( CosmologyValidation(versions=["TOY"], **grid), CosmologyValidation(versions=["TOY_OPEN"], **grid), - grid, ) def test_a_blinded_catalogues_xi_leaves_concealed(blinded_and_twin): - """[signal-leaves-sealed] calculate_2pcf returns, and caches, a blinded - catalogue's ξ± shifted from its public twin's by exactly the blind's shift; - and the two are never held together.""" + """calculate_2pcf returns, and caches, a blinded catalogue's ξ± shifted + from its public twin's by exactly the blind's shift, stamped with its name.""" from sp_validation import blinding - from sp_validation.custody import CustodyError - cv, cv_open, grid = blinded_and_twin + cv, cv_open = blinded_and_twin blinded, twin = cv.calculate_2pcf("TOY"), cv_open.calculate_2pcf("TOY_OPEN") - shift = blinding.conceal(twin, blinding.open_blind(cv.custody("TOY"))).mean - shift = shift - twin.mean + shift = blinding.conceal(twin, cv.blind("TOY")).mean - twin.mean assert np.all(shift != 0) np.testing.assert_allclose( blinded.mean - twin.mean, shift, rtol=1e-8, atol=1e-12 * np.abs(shift).max() ) - assert blinded.metadata["custody"] == cv.custody("TOY").token + assert sacc_io.stamp(blinded) == "toy" and sacc_io.stamp(twin) == "none" assert cv.xi_parts["TOY", "reporting"] is blinded - with pytest.raises(CustodyError, match="shows the blind's shift"): - CosmologyValidation(versions=["TOY", "TOY_OPEN"], **grid) def test_map2_transform_is_treecorrs_calculate_map_sq(): diff --git a/src/sp_validation/tests/test_custody.py b/src/sp_validation/tests/test_custody.py index 1ce93717..2acb1c69 100644 --- a/src/sp_validation/tests/test_custody.py +++ b/src/sp_validation/tests/test_custody.py @@ -1,10 +1,5 @@ -"""Custody is declared on every catalogue entry and resolved in one place. +"""Every catalogue entry declares its blind.""" -Blind records are hand-written here: the host never draws, so a record needs -no real seed. -""" - -import json from pathlib import Path import pytest @@ -18,146 +13,37 @@ def _catalogues(tmp_path, **blinds): """A parsed catalogue config: one entry per ``name=blind``, each reading its own file; ``None`` declares nothing.""" - entries = { - "nz": {"subdir": "/nz"}, - "paths": {"output": "./output", "blinds": str(tmp_path / "blinds")}, - } + entries = {"nz": {"subdir": "/nz"}, "paths": {"output": "./output"}} for name, blind in blinds.items(): - shear = { - "path": f"{name}.fits", - "e1_col_corrected": "e1c", - "e2_col_corrected": "e2c", - } - entries[name] = {"subdir": str(tmp_path), "shear": shear} + entries[name] = {"subdir": str(tmp_path), "shear": {"path": f"{name}.fits"}} if blind is not None: entries[name]["blind"] = blind return entries -def _record(tmp_path, name): - """A blind's record in the registry; returns its commitment.""" - record = {"seed": f"seed-of-{name}", "envelope": {"S8": 0.075}, "draw_scheme": 2} - (tmp_path / "blinds").mkdir(exist_ok=True) - (tmp_path / "blinds" / f"{name}.blind.json").write_text(json.dumps(record)) - return cu.commitment(record) - - def test_every_entry_declares_its_blind(tmp_path): - cats = _catalogues(tmp_path, OPEN="none", MOCK="mock", Y3="y3", NEW=None) - commitment = _record(tmp_path, "y3") - assert cu.custody_of(cats, "OPEN").token == "none" - assert cu.custody_of(cats, "MOCK").token == "mock" - assert cu.custody_of(cats, "Y3").token == f"y3:{commitment}" - with pytest.raises(cu.CustodyError, match="declares no `blind:`"): - cu.custody_of(cats, "NEW") - # _leak_corr and _seed versions take their entry's custody + cats = _catalogues(tmp_path, OPEN="none", Y3="y3", NEW=None) + assert cu.declared(cats, "OPEN") == "none" + assert cu.declared(cats, "Y3") == "y3" + with pytest.raises(ValueError, match="declares no `blind:`"): + cu.declared(cats, "NEW") + # _leak_corr and _seed versions take their entry's blind for version in ("Y3_leak_corr", "Y3_seed00042_leak_corr"): - assert cu.custody_of(cats, version) == cu.custody_of(cats, "Y3") - + assert cu.declared(cats, version) == "y3" -def test_a_missing_blind_fails_closed_naming_the_remedy(tmp_path): - cats = _catalogues(tmp_path, SP_v9="y3") - with pytest.raises(cu.CustodyError, match="get read access") as absent: - cu.custody_of(cats, "SP_v9") - _record(tmp_path, "other") - with pytest.raises(cu.CustodyError, match="blinding init y3") as missing: - cu.custody_of(cats, "SP_v9") - for refusal in (absent, missing): - assert "SP_v9 is blinded under y3" in str(refusal.value) - assert "`blind: none` would unblind it" in str(refusal.value) - -def test_entries_reading_one_file_declare_one_blind(tmp_path, monkeypatch): - """Within the passed config, and against the repository config, which is - authoritative for the files it names.""" +def test_entries_reading_one_file_declare_one_blind(tmp_path): cats = _catalogues(tmp_path, TOY="y3", TWIN="none") cats["TWIN"]["shear"]["path"] = "TOY.fits" - with pytest.raises(cu.CustodyError, match="declare one blind"): + with pytest.raises(ValueError, match="different blinds"): cu.declared(cats, "TWIN") cats["TWIN"]["blind"] = "y3" assert cu.declared(cats, "TWIN") == "y3" - repo = tmp_path / "repo_cat_config.yaml" - repo.write_text(yaml.safe_dump(_catalogues(tmp_path, TOY="y3"))) - monkeypatch.setattr(cu, "REPO_CAT_CONFIG", repo) - personal = _catalogues(tmp_path, MINE="none") - personal["MINE"]["shear"]["path"] = "TOY.fits" - with pytest.raises(cu.CustodyError, match="authoritative"): - cu.declared(personal, "MINE") - assert cu.of_file(tmp_path / "TOY.fits") == {"y3"} - assert cu.of_file(tmp_path / "elsewhere.fits") == set() - - -@pytest.mark.parametrize( - "blinds, allowed", - [ - ({"A": "y3", "B": "none"}, False), - ({"A": "y3", "B": "y4"}, False), - ({"A": "none", "B": "y3"}, False), - ({"A": "y3", "B": "mock"}, True), - ({"A": "y3", "B": "y3"}, True), - ({"A": "none", "B": "mock"}, True), - ], -) -def test_the_mixing_rule(blinds, allowed): - """A blinded catalogue is shown only beside mocks and its own blind.""" - if allowed: - cu.check_mix(blinds) - else: - with pytest.raises(cu.CustodyError, match="shows the blind's shift"): - cu.check_mix(blinds) - -def test_the_registry_stays_out_of_git_worktrees(tmp_path): - (tmp_path / "repo" / ".git").mkdir(parents=True) - inside = {"paths": {"blinds": str(tmp_path / "repo" / "blinds")}} - with pytest.raises(cu.CustodyError, match="inside the git worktree"): - cu.registry(inside) - root = {"paths": {"blinds": str(tmp_path / "repo")}} - assert cu.registry(root) == (tmp_path / "repo").resolve() - with pytest.raises(cu.CustodyError, match="absolute"): - cu.registry({"paths": {"blinds": "blinds"}}) - - -def test_the_commitment_is_the_forks_of_the_whole_record(): - smokescreen = pytest.importorskip("smokescreen") - record = {"seed": "s", "envelope": {"S8": 0.075}, "draw_scheme": 2} - canonical = json.dumps(record, sort_keys=True, separators=(",", ":")) - assert cu.commitment(record) == smokescreen.seed_commitment(canonical) - - -def test_every_catalogue_in_the_repository_resolves(): +def test_every_catalogue_in_the_repository_is_public(): cats = yaml.safe_load((REPO / "cosmo_val" / "cat_config.yaml").read_text()) blinds = { - name: cu.custody_of(cats, name).blind - for name in cats - if name not in cu.NOT_CATALOGUES + name: cu.declared(cats, name) for name in cats if name not in cu.NOT_CATALOGUES } - assert blinds and set(blinds.values()) <= {cu.NONE, cu.MOCK}, blinds - - -def test_a_token_is_the_custody_it_names_and_must_match_the_declaration(tmp_path): - """A job seals under its token's custody, which is the declared one; a - token naming another blind is refused.""" - from sp_validation.cosmo_val import CosmologyValidation - - cats = _catalogues(tmp_path, TOY="toy", TOY_OPEN="none", TOY_MOCK="mock") - _record(tmp_path, "toy") - config = tmp_path / "cat_config.yaml" - config.write_text(yaml.safe_dump(cats)) - - def job(version, token): - return CosmologyValidation( - versions=[version], - catalog_config=str(config), - output_dir=str(tmp_path / "out"), - custody={version: token}, - ) - - for version in ("TOY", "TOY_OPEN", "TOY_MOCK"): - custody = cu.custody_of(cats, version) - assert cu.parse(custody.token, cats) == custody - assert job(version, custody.token).custody(version) == custody - for token in ("none", "mock", "other:" + "0" * 64): - with pytest.raises(cu.CustodyError, match="declares blind: toy"): - job("TOY", token) + assert blinds and set(blinds.values()) == {"none"}, blinds diff --git a/src/sp_validation/tests/test_custody_e2e.py b/src/sp_validation/tests/test_custody_e2e.py deleted file mode 100644 index 04ae3458..00000000 --- a/src/sp_validation/tests/test_custody_e2e.py +++ /dev/null @@ -1,152 +0,0 @@ -"""End to end: one blinded catalogue through its rule scripts, then unblinded. - -A synthetic catalogue ``TOY`` is declared under a blind drawn with ``blinding -init``. The ξ± and assembly rule scripts run as Snakemake runs them (``runpy`` -with a ``snakemake`` object), and no file of the blinded run holds a true ξ± -value. The declaration is then flipped to ``blind: none`` and the chain re-run -into a fresh tree: blinded − true is the blind's shift, and nothing but the -mean moved. -""" - -import re -import runpy -import types -from pathlib import Path - -import numpy as np -import yaml -from _synthetic import write_synthetic_catalogs - -from sp_validation import blinding as bd -from sp_validation import custody as cu -from sp_validation import sacc_io as sio - -SCRIPTS = Path(__file__).resolve().parents[3] / "workflow" / "scripts" -GRID = {"min_sep": 5.0, "max_sep": 60.0, "nbins": 6, "npatch": 1} - - -class Named(list): - """The shape of Snakemake's ``input``/``output``/``params``: a list with names.""" - - def __init__(self, **items): - super().__init__(items.values()) - self._items = items - - def __getitem__(self, key): - return self._items[key] if isinstance(key, str) else super().__getitem__(key) - - def __getattr__(self, name): - try: - return self._items[name] - except KeyError: - raise AttributeError(name) from None - - def get(self, key, default=None): - return self._items.get(key, default) - - -def run_rule(script, **fields): - smk = types.SimpleNamespace(**{k: Named(**v) for k, v in fields.items()}) - runpy.run_path( - str(SCRIPTS / script), init_globals={"snakemake": smk}, run_name="__main__" - ) - - -def run_chain(cat_config, out, cov): - """Rules xi and assemble_sacc for TOY into ``out``.""" - out.mkdir(exist_ok=True) - token = cu.custody_of(yaml.safe_load(cat_config.read_text()), "TOY").token - common = {"cat_config": str(cat_config), "custody": token} - xi = out / "TOY_xi_reporting.sacc" - run_rule( - "run_2pcf.py", - input={}, - output={"sacc": str(xi)}, - params={"ver": "TOY", **GRID, "output_dir": str(out), "grid": "reporting"} - | common, - ) - run_rule( - "assemble_sacc.py", - input={"xi_reporting": str(xi), "xi_cov": str(cov)}, - output={"sacc": str(out / "TOY.sacc")}, - params={"version": "TOY", "expected": ["xi_reporting"], "custody": token}, - ) - - -def plaintext(blobs, values): - """Names of ``blobs`` ({name: bytes}) holding any of ``values``, as float64 - of either byte order at any offset, or as text in exponent notation.""" - values = np.sort(np.asarray(values, float)) - - def near(x, rtol): - keep = np.isfinite(x) & (x != 0) - x, rtol = x[keep], np.broadcast_to(rtol, keep.shape)[keep] - i = np.clip(np.searchsorted(values, x), 1, len(values) - 1) - gap = np.minimum(np.abs(x - values[i - 1]), np.abs(x - values[i])) - return bool(np.any(gap <= rtol * np.abs(x))) - - found = [] - for name, blob in blobs.items(): - for order in "<>": - for offset in range(8): - count = (len(blob) - offset) // 8 - if count > 0 and near( - np.frombuffer(blob, f"{order}f8", count, offset), 1e-9 - ): - found.append(name) - tokens = re.findall(rb"(-?\d\.(\d+)e[-+]\d+)", blob) - digits = np.array([0.51 * 10.0 ** -len(d) for _, d in tokens]) - if tokens and near(np.array([float(t) for t, _ in tokens]), digits): - found.append(name) - return found - - -def test_a_blinded_catalogue_from_birth_to_unblinding( - tmp_path, monkeypatch, toy_theory -): - monkeypatch.syspath_prepend(str(SCRIPTS)) - import cv_runner - - # Rule jobs line-buffer their own streams; under pytest they are captured. - monkeypatch.setattr(cv_runner, "_unbuffer_streams", lambda: None) - params, _ = write_synthetic_catalogs( - tmp_path, n_gal=20000, coherent_shear=True, catalogues={"TOY": "toy"} - ) - cat_config = Path(params["catalog_config"]) - config = yaml.safe_load(cat_config.read_text()) - blind = bd.init("toy", config) - cov = tmp_path / "cov.txt" - np.savetxt(cov, np.diag(np.full(2 * GRID["nbins"], 1e-10))) - - # --- the blinded run ----------------------------------------------------- - blinded_out = tmp_path / "blinded" - run_chain(cat_config, blinded_out, cov) - custody = cu.custody_of(config, "TOY") - for part in blinded_out.glob("*.sacc"): - assert cu.read_stamp(sio.load(part).metadata) == custody - blinded_run = { - str(p): p.read_bytes() for p in blinded_out.rglob("*") if p.is_file() - } - - # --- flip to public, re-measure into a fresh tree ------------------------ - config["TOY"]["blind"] = "none" - cat_config.write_text(yaml.safe_dump(config, sort_keys=False)) - true_out = tmp_path / "true" - run_chain(cat_config, true_out, cov) - - for name in ("TOY_xi_reporting.sacc", "TOY.sacc"): - blinded, true = (sio.load(out / name) for out in (blinded_out, true_out)) - assert cu.read_stamp(true.metadata) == cu.Custody("none") - shift = bd.conceal(true, blind).mean - true.mean - assert np.all(shift != 0) - np.testing.assert_allclose( - blinded.mean - true.mean, shift, rtol=1e-8, atol=1e-12 * np.abs(shift).max() - ) - np.testing.assert_allclose( - blinded.covariance.dense, true.covariance.dense, rtol=1e-10 - ) - - # --- no true ξ± value was written by the blinded run --------------------- - gg = sio.xi_correlation(sio.load(true_out / "TOY_xi_reporting.sacc")) - assert not plaintext(blinded_run, np.concatenate([gg.xip, gg.xim])) - assert not list(tmp_path.rglob("*_xi_*.txt")) diff --git a/src/sp_validation/tests/test_cv_init_params.py b/src/sp_validation/tests/test_cv_init_params.py index 03c691ea..ffc87d8b 100644 --- a/src/sp_validation/tests/test_cv_init_params.py +++ b/src/sp_validation/tests/test_cv_init_params.py @@ -16,9 +16,7 @@ REPO = Path(__file__).resolve().parents[3] -EXEMPT = { - "custody": "per-version custody tokens, passed from the rule's params.custody", -} +EXEMPT = {} def _load_common(): diff --git a/src/sp_validation/tests/test_masks.py b/src/sp_validation/tests/test_masks.py index 76e97748..beeefdda 100644 --- a/src/sp_validation/tests/test_masks.py +++ b/src/sp_validation/tests/test_masks.py @@ -34,12 +34,10 @@ ], ) def test_apply_condition_kinds(kind, value, expected): - npt.assert_array_equal(apply_condition(_ARRAY, kind, value), expected) def test_smaller_equal_alias_matches_less_equal(): - npt.assert_array_equal( apply_condition(_ARRAY, "smaller_equal", 3), apply_condition(_ARRAY, "less_equal", 3), @@ -47,13 +45,11 @@ def test_smaller_equal_alias_matches_less_equal(): def test_unknown_kind_raises(): - with pytest.raises(ValueError): apply_condition(_ARRAY, "not_a_real_kind", 3) def test_mask_apply_matches_apply_condition(): - dat = np.array([(1,), (2,), (3,), (4,), (5,)], dtype=[("col", "i8")]) my_mask = Mask("col", "test_mask", kind="greater_equal", value=3, dat=dat) @@ -64,7 +60,6 @@ def test_mask_apply_matches_apply_condition(): def test_not_equal_2bands_is_two_column_or(): - # Keep an object if EITHER band column differs from the sentinel dat = np.array( [(-99, -99), (-99, 20.0), (21.0, -99), (21.0, 20.0)], @@ -84,13 +79,11 @@ def test_not_equal_2bands_is_two_column_or(): def test_not_equal_2bands_requires_col_name2(): - with pytest.raises(ValueError, match="col_name2"): Mask("mag_z", "zband", kind="not_equal_2bands", value=-99) def test_not_equal_2bands_descr_names_both_columns(): - my_mask = Mask( "mag_z", "zband", kind="not_equal_2bands", value=-99, col_name2="mag_z2" ) diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 5a9128f0..091aeada 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -150,7 +150,7 @@ def _write_synthetic_config(tmp_path): "star": {**psf_cfg}, "psf": psf_cfg, "patch_number": 150, - "blind": "mock", + "blind": "none", }, } config_path = tmp_path / "config.yaml" diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 59857c38..ff36d154 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -9,7 +9,7 @@ import pytest from sp_validation import sacc_io as sio -from sp_validation.custody import Custody +from sp_validation.blinding import NONE # --------------------------------------------------------------------------- # @@ -63,12 +63,11 @@ def _cosebi_block(s, tr): # --------------------------------------------------------------------------- # # 1. Per-writer round-trip (arrays / tags / windows / NZ bitwise) # --------------------------------------------------------------------------- # -MOCK = Custody("mock") def _roundtrip(s, tmp_path, name="rt"): path = tmp_path / f"{name}.sacc" - sio.save(s, str(path), custody=MOCK) + sio.save(s, str(path), blind=NONE) return sio.load(str(path)) @@ -469,7 +468,7 @@ def test_end_to_end_one_file_layout(tmp_path): s, [(xi_c, _spd(len(xi_c), 1)), (co, _spd(len(co), 2)), (xi_f, fine_block)], ) - sio.save(s, str(tmp_path / f"{version}.sacc"), custody=MOCK) + sio.save(s, str(tmp_path / f"{version}.sacc"), blind=NONE) a = sio.load(str(tmp_path / f"{version}.sacc")) @@ -508,7 +507,7 @@ def test_one_file_layout_diagonal_fine_fallback(tmp_path): xi_f = _xi_block(s, tr, grid="integration") variances = np.concatenate([np.arange(1, 51) * 1e-12, np.arange(1, 51) * 2e-12]) sio.assemble_covariance(s, [(xi_c, _spd(len(xi_c), 1)), (xi_f, np.diag(variances))]) - sio.save(s, str(tmp_path / "vDIAG.sacc"), custody=MOCK) + sio.save(s, str(tmp_path / "vDIAG.sacc"), blind=NONE) a = sio.load(str(tmp_path / "vDIAG.sacc")) assert np.array_equal(np.diag(a.covariance.dense[np.ix_(xi_f, xi_f)]), variances) @@ -647,7 +646,7 @@ def test_merge_per_statistic_files(tmp_path): # inputs untouched assert s_xi.metadata["version"] == "vM" # readers work on the merged file after a round-trip - sio.save(merged, str(tmp_path / "vM.sacc"), custody=MOCK) + sio.save(merged, str(tmp_path / "vM.sacc"), blind=NONE) merged_rt = sio.load(str(tmp_path / "vM.sacc")) _, p, _ = sio.get_xi(merged_rt, (0, 0), grid="reporting") assert np.array_equal(p, np.arange(6) * 1e-5) @@ -682,7 +681,7 @@ def test_merge_block_diagonal_covariance_stays_block_diagonal(tmp_path): assert type(s_xi.covariance).__name__ == "BlockDiagonalCovariance" merged = sio.merge([s_xi, s_co]) assert type(merged.covariance).__name__ == "BlockDiagonalCovariance" - sio.save(merged, str(tmp_path / "vBLK.sacc"), custody=MOCK) + sio.save(merged, str(tmp_path / "vBLK.sacc"), blind=NONE) merged_rt = sio.load(str(tmp_path / "vBLK.sacc")) assert type(merged_rt.covariance).__name__ == "BlockDiagonalCovariance" n_xi = len(s_xi.mean) diff --git a/src/sp_validation/tests/test_sacc_io_realdata.py b/src/sp_validation/tests/test_sacc_io_realdata.py index 97f7541f..abe1f230 100644 --- a/src/sp_validation/tests/test_sacc_io_realdata.py +++ b/src/sp_validation/tests/test_sacc_io_realdata.py @@ -29,7 +29,6 @@ from astropy.io import fits from sp_validation import sacc_io -from sp_validation.custody import Custody _DATA = Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") _REAL_FILES = { @@ -124,7 +123,6 @@ def _sacc_from_2pt_fits(hdul): for key in ("t0m", "t2m"): full[np.ix_(idx[key], idx[key])] = np.eye(n) s.add_covariance(full) - s = sacc_io.seal(s, Custody("mock")) tau_sidecar = fits.BinTableHDU.from_columns( fits.ColDefs( diff --git a/src/sp_validation/tests/test_sacc_io_twopoint.py b/src/sp_validation/tests/test_sacc_io_twopoint.py index 0187e8da..78f6809d 100644 --- a/src/sp_validation/tests/test_sacc_io_twopoint.py +++ b/src/sp_validation/tests/test_sacc_io_twopoint.py @@ -37,9 +37,6 @@ from astropy.io import fits from sp_validation import sacc_io -from sp_validation.custody import Custody - -MOCK = Custody("mock") _SCRIPT = ( Path(__file__).resolve().parents[3] @@ -265,7 +262,7 @@ def _sacc(inp, *, cl=False, rho_tau=False): idx = s.indices(dtype, (SOURCE, PSF)) full[np.ix_(idx, idx)] = np.eye(N_ANG) s.add_covariance(full) - return sacc_io.seal(s, MOCK) + return s def _sidecar_hdus(tmp_path, inp): @@ -453,7 +450,6 @@ def test_integration_grid_points_ignored(tmp_path): full[np.ix_(xi_int_all, xi_int_all)] = _spd(2 * N_ANG, 43) s_aug.add_covariance(full) - s_aug = sacc_io.seal(s_aug, MOCK) out_aug = tmp_path / "aug.fits" sacc_io.sacc_to_twopoint_fits( @@ -492,7 +488,6 @@ def test_tomographic_sacc_raises(tmp_path): for pair in [(0, 0), (0, 1), (1, 1)]: sacc_io.add_xi(s, pair, inp["theta"], inp["xip"], inp["xim"], grid="reporting") s.add_covariance(np.eye(len(s.mean))) - s = sacc_io.seal(s, MOCK) with pytest.raises(ValueError, match="single-bin only"): sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) @@ -516,7 +511,6 @@ def test_sacc_without_xi_raises(tmp_path): window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), ) s.add_covariance(np.eye(len(s.mean))) - s = sacc_io.seal(s, MOCK) with pytest.raises(ValueError, match="nothing to convert"): sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) @@ -583,12 +577,3 @@ def _mask_idx(dtype, tracers): np.testing.assert_array_equal( hdul["COVMAT_CELL"].data, encoded[np.ix_(cell_idx, cell_idx)] ) - - -def test_an_unstamped_sacc_is_refused(tmp_path): - """Only a SACC born through the door (stamped) is converted.""" - s = _sacc(_inputs(seed=50)) - del s.metadata["custody"] - with pytest.raises(ValueError, match="custody stamp"): - sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) - assert not (tmp_path / "x.fits").exists() diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index 602e874a..80dc121a 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -15,8 +15,8 @@ import pytest from sp_validation import sacc_io as sio +from sp_validation.blinding import NONE from sp_validation.cosmo_val import sacc_writers as sw -from sp_validation.custody import Custody def _nz(seed=0, n=40): @@ -33,12 +33,9 @@ def _theta(n=6): return np.geomspace(1.0, 100.0, n) -MOCK = Custody("mock") - - def _roundtrip(s, tmp_path, name): p = tmp_path / f"{name}.sacc" - sio.save(s, str(p), custody=MOCK) + sio.save(s, str(p), blind=NONE) return sio.load(str(p)) @@ -147,7 +144,7 @@ def get_bandpower_windows(self): part = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell, cl_all, _Workspace()) path = tmp_path / "pseudo_cl.sacc" - sio.save(part, str(path), custody=MOCK) + sio.save(part, str(path), blind=NONE) data = _paper_pseudo_cl_reader()(path) assert np.array_equal(data["ELL"], ell) @@ -353,7 +350,7 @@ def test_assemble_from_reloaded_parts(tmp_path): parts = _make_parts(nz) reloaded = [] for i, part in enumerate(parts): - sio.save(part, str(tmp_path / f"part{i}.sacc"), custody=MOCK) + sio.save(part, str(tmp_path / f"part{i}.sacc"), blind=NONE) reloaded.append(sio.load(str(tmp_path / f"part{i}.sacc"))) s = sw.assemble_analysis_sacc(reloaded) assert type(s.covariance).__name__ == "BlockDiagonalCovariance" diff --git a/src/sp_validation/tests/test_survey.py b/src/sp_validation/tests/test_survey.py index f42f884c..3d3eac1d 100644 --- a/src/sp_validation/tests/test_survey.py +++ b/src/sp_validation/tests/test_survey.py @@ -17,7 +17,6 @@ class SurveyTestCase(TestCase): def setUp(self): - self._dd = np.array( [(270.283, 1), (270.283, 0), (188.308, 0)], dtype=[("TILE_ID", "f8"), ("FLAGS", "i2")], @@ -33,7 +32,6 @@ def setUp(self): self._patch = ["P5"] def tearDown(self): - self.number_tile = None self.number_exp = None self.number_int = None diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py index 8c7dab63..ad4d723f 100644 --- a/workflow/tests/conftest.py +++ b/workflow/tests/conftest.py @@ -10,15 +10,14 @@ Snakemake is in. The ``candide`` tests need candide itself; CI deselects them. The ``toy`` fixture is a disposable checkout: copies of ``workflow/`` and -``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, one catalogue per -custody state, a hand-written blind record (the host never draws), stand-ins -for the processed CosmoCov covariances (their inputs live on candide), and both -output roots in tmp. +``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, a blinded, a +public and an undeclared catalogue, stand-ins for the processed CosmoCov +covariances (their inputs live on candide), and both output roots in tmp. The +host never opens a blind, so there is no blind record. """ import dataclasses import importlib.util -import json import os import re import shutil @@ -34,8 +33,6 @@ # The toy catalogue and its leakage-corrected variant: blinded under `toy`. VERSIONS = ("SP_v0.1", "SP_v0.1_leak_corr") PUBLIC = "SP_v0.2" -# Declared under the blind `gone`, which the registry does not hold. -UNCOVERED = "SP_v0.4" # Declares no blind. UNDECLARED = "SP_v0.5" @@ -130,8 +127,9 @@ def snakemake(self, *args, config=(), cwd=None, env=None, timeout=300): ) -def _cat_config(data, registry): - """One catalogue per custody state, each reading its own touched file.""" +def _cat_config(data): + """A blinded, a public and an undeclared catalogue, each reading its own + touched file.""" def entry(name, **declaration): catalogue = data / f"{name}.fits" @@ -157,20 +155,11 @@ def entry(name, **declaration): return { VERSIONS[0]: entry(VERSIONS[0], blind="toy"), PUBLIC: entry(PUBLIC, blind="none"), - "SP_v0.3": entry("SP_v0.3", blind="mock"), - UNCOVERED: entry(UNCOVERED, blind="gone"), UNDECLARED: entry(UNDECLARED), - "paths": {"output": "./output", "blinds": str(registry)}, + "paths": {"output": "./output"}, } -def _registry(registry): - """The blind `toy`, as ``blinding init`` writes it but for a stand-in seed.""" - registry.mkdir() - record = {"seed": "toy-seed", "envelope": {"S8": 0.075}, "draw_scheme": 2} - (registry / "toy.blind.json").write_text(json.dumps(record)) - - @pytest.fixture(scope="session") def toy(tmp_path_factory): root = tmp_path_factory.mktemp("toy") @@ -184,9 +173,8 @@ def toy(tmp_path_factory): (root / "data").mkdir() (root / "cosmo_val").mkdir() (root / "cosmo_val" / "cat_config.yaml").write_text( - yaml.safe_dump(_cat_config(root / "data", root / "blinds")) + yaml.safe_dump(_cat_config(root / "data")) ) - _registry(root / "blinds") rundir = root / "papers" / "cosmo_val" config_path = rundir / "config" / "config.yaml" diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 0aad6726..6be9e336 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -8,15 +8,7 @@ import pytest import yaml -from conftest import ( - PUBLIC, - REPO, - UNCOVERED, - UNDECLARED, - VERSIONS, - on_candide, - parse_jobs, -) +from conftest import REPO, UNDECLARED, VERSIONS, on_candide, parse_jobs @pytest.fixture(scope="module") @@ -89,33 +81,16 @@ def test_one_integration_grid(toy, forced, grids): assert {part, covariance} <= by_rule["cv_pure_eb"], by_rule["cv_pure_eb"] -def test_custody_is_the_checkouts_and_no_rule_touches_a_blind(toy, forced, tmp_path): - """A catalogue and its variant share one custody line; even a - forced run schedules nothing that reads or writes the registry; and a - config file declaring the catalogue public changes nothing.""" - output, jobs = forced - line = f"[custody] {VERSIONS[0]} (+ {VERSIONS[1]}): blinded under toy" - assert {x for x in output.splitlines() if x.startswith("[custody]")} == {line} - registry = (toy.root / "blinds").resolve() - assert not [j.rule for j in jobs if "blind" in j.rule] - touched = [ - f - for j in jobs - for f in j.input + j.output - if (toy.rundir / f).resolve().is_relative_to(registry) - ] - assert not touched, touched - - override = tmp_path / "override.yaml" - override.write_text(yaml.safe_dump({VERSIONS[0]: {"blind": "none"}})) - result = toy.snakemake("-n", "assemble_sacc_all", "--configfile", str(override)) - assert result.returncode == 0, result.stdout - assert line in result.stdout +def test_a_launch_prints_each_catalogues_blind(forced): + """A catalogue and its variant share one line.""" + output, _ = forced + line = f"[blind] {VERSIONS[0]} (+ {VERSIONS[1]}): toy" + assert {x for x in output.splitlines() if x.startswith("[blind]")} == {line} -def test_a_custody_flip_reruns_the_catalogues_parts(toy, grids, tmp_path): - """A part's params carry its catalogue's custody token, so declaring - the catalogue public reruns the part and nothing else does.""" +def test_a_blind_flip_reruns_the_catalogues_parts(toy, grids, tmp_path): + """A part's params carry its catalogue's blind, so declaring the catalogue + public reruns the part and nothing else does.""" env = toy.env | {"COSMO_VAL": str(tmp_path / "cosmo_val")} reporting = toy.common.grid_binning(grids["reporting"]) part = tmp_path / "cosmo_val" / f"{VERSIONS[0]}_xi_{reporting}.sacc" @@ -145,23 +120,12 @@ def scheduled(): assert "params have changed" in output.lower(), output -LAUNCH_REFUSALS = { - # A blind the registry does not hold, an entry declaring none, and a - # blinded catalogue overlaid with a public one - "no_blind": ([UNCOVERED], [f"{UNCOVERED} is blinded under gone", "init gone"]), - "undeclared": ([UNDECLARED], ["declares no `blind:`"]), - "mixed": ([VERSIONS[0], PUBLIC], ["shows the blind's shift"]), -} - - -@pytest.mark.parametrize("case", LAUNCH_REFUSALS) -def test_a_launch_the_dag_cannot_honour_stops_with_the_fix(toy, case): - versions, message = LAUNCH_REFUSALS[case] - listed = ", ".join(f'"{v}"' for v in versions) - result = toy.snakemake("-n", "assemble_sacc_all", config=[f"versions=[{listed}]"]) +def test_a_launch_stops_on_a_catalogue_declaring_no_blind(toy): + result = toy.snakemake( + "-n", "assemble_sacc_all", config=[f'versions=["{UNDECLARED}"]'] + ) assert result.returncode != 0, result.stdout - for fragment in message: - assert fragment in result.stdout, result.stdout + assert "declares no `blind:`" in result.stdout, result.stdout assert "rule assemble_sacc" not in result.stdout diff --git a/workflow/tests/test_toy_run.py b/workflow/tests/test_toy_run.py index 76249dd2..3b98a9ff 100644 --- a/workflow/tests/test_toy_run.py +++ b/workflow/tests/test_toy_run.py @@ -7,7 +7,6 @@ under a blind drawn in the image with the default theory. """ -import json import os import shutil import subprocess @@ -17,9 +16,7 @@ import pytest import yaml -from conftest import REPO, _load_module, container, on_candide - -custody = _load_module(REPO / "src" / "sp_validation" / "custody.py", "custody", {}) +from conftest import REPO, container, on_candide VERSIONS = ("SP_v0.1", "SP_v0.1_leak_corr") REPORTING = {"theta_min": 5.0, "theta_max": 60.0, "nbins": 6, "npatch": 4} @@ -45,14 +42,13 @@ def _in_image(image, root, *command): def _stamps(image, root, parts): - """The custody stamp of each SACC part, read in the image.""" + """The blind stamp of each SACC part, read in the image.""" script = ( - "import json, sys\n" - "from sp_validation import custody, sacc_io\n" - "print(json.dumps([custody.read_stamp(sacc_io.load(p).metadata).token" - " for p in sys.argv[1:]]))" + "import sys\n" + "from sp_validation import sacc_io\n" + "print(*(sacc_io.stamp(sacc_io.load(p)) for p in sys.argv[1:]))" ) - return json.loads(_in_image(image, root, "python", "-c", script, *map(str, parts))) + return _in_image(image, root, "python", "-c", script, *map(str, parts)).split() def _toy_checkout(root, image): @@ -100,7 +96,7 @@ def _toy_checkout(root, image): @pytest.mark.candide @on_candide -def test_xi_parts_are_stamped_with_the_blinds_commitment(): +def test_xi_parts_are_stamped_with_the_blind(): image, kind = container.resolve_image() assert kind != "tag", "no local image; run `spv-container pull`" root = Path(tempfile.mkdtemp(prefix="toy_run_", dir=Path.home())) @@ -142,9 +138,7 @@ def test_xi_parts_are_stamped_with_the_blinds_commitment(): ) assert result.returncode == 0, result.stdout - record = json.loads((root / "cosmo_val/blinds/toy.blind.json").read_text()) - token = f"toy:{custody.commitment(record)}" parts = [out / f"{v}_xi_{BINNING}.sacc" for v in VERSIONS] - assert _stamps(image, root, parts) == [token] * len(parts) + assert _stamps(image, root, parts) == ["toy"] * len(parts) shutil.rmtree(root) # kept on failure, for post-mortem From 7dffd7be83ede6a473a178c847186cb97149222f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 04:10:47 +0200 Subject: [PATCH 152/160] Document blinding as blinds, a declared blind per catalogue and a theory Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 7 ++-- papers/bmodes/scripts/pseudo_cl_io.py | 2 - workflow/README.md | 55 +++++++++++++-------------- 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2ee79569..fa111024 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,7 +42,8 @@ is the container (full scientific stack pre-built). For a local dev environment: - `cat.py`: Catalogue handling and manipulation - `cosmo_val.py`: Cosmology validation routines - `cosmology.py`: Cosmological calculations and theory -- `theory.py`: Theory data vectors for SACC rows, one pluggable function per data type +- `theory.py`: Theory data vectors for a SACC, `theory(params, s)`, from which blinding shifts are made +- `blinding.py`: Blinds, and `conceal`, which shifts a SACC by one - `galaxy.py`: Galaxy-specific processing - `io.py`: Input/output utilities - `plots.py`: Plotting functions @@ -81,8 +82,8 @@ Main configuration in `scripts/calibration/params.py` with parameters: - pyccl for cosmological calculations ## Blinded catalogues -Each `cosmo_val/cat_config.yaml` entry declares `blind: none`, `mock` or a blind's name. -- For an entry with `blind: `, signal (ξ±, Cℓ, COSEBIs, M_ap, maps, any cosmological statistic) leaves the function that measures it only sealed: compute it and `sacc_io.save(..., custody=)` it in that function and return the sealed part, as `CosmologyValidation` and the workflow do. Never measure it from the file and keep the raw values. +Each `cosmo_val/cat_config.yaml` entry declares `blind: none` or a blind's name. +- For an entry with `blind: `, signal (ξ±, Cℓ, any cosmological statistic) leaves the function that measures it only concealed: compute it and `sacc_io.save(..., blind=)` it in that function and return the saved part, as `CosmologyValidation` and the workflow do. Never measure it from the file and keep the raw values. - Never set `blind: none` to get a run through. - Never print, paste or commit a `.blind.json`. diff --git a/papers/bmodes/scripts/pseudo_cl_io.py b/papers/bmodes/scripts/pseudo_cl_io.py index 27c7df3a..b9497105 100644 --- a/papers/bmodes/scripts/pseudo_cl_io.py +++ b/papers/bmodes/scripts/pseudo_cl_io.py @@ -7,8 +7,6 @@ def load_pseudo_cl_data(path): """Return a pseudo-Cℓ part's spectra as ``{"ELL", "EE", "EB", "BB"}`` arrays. The bandpower covariance is a separate FITS product (``pseudo_cl_cov``). - Loading goes through ``sacc_io.load``, which refuses a part without a - custody stamp. """ ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(sacc_io.load(str(path)), (0, 0)) missing = [name for name, values in (("BB", bb), ("EB", eb)) if values is None] diff --git a/workflow/README.md b/workflow/README.md index d168e306..b1c86470 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -25,36 +25,35 @@ Runs stay modular, not monolithic: a paper or run composes these rules with Snakemake's `module` directive under its own config and an output `prefix`, so each namespaces cleanly under `results//`. -## Custody: which catalogues are blinded - -Every entry of `cosmo_val/cat_config.yaml` declares `blind: none`, `blind: mock` -or the name of a blind; an entry without one is refused, and `_leak_corr` and -`_seed` versions take their entry's. Entries reading one shear file declare -one blind (the repository config is authoritative), and a blinded catalogue is -shown only beside mocks and its own blind, since any other overlay shows the -shift. A launch prints one `[custody]` line per catalogue. - -Under a blind, ξ± and pseudo-Cℓ_EE are shifted by t(hidden) − t(fiducial) -before they are first written, t being the default PyCCL theory; -`sp_validation.blinding.STANDARD` lists every standard estimator's rule once -(Cℓ_BB/EB unshifted, COSEBIs and pure-E/B inheriting from ξ±, ρ/τ signal-free). -Any other data type is shifted by a theory function its birth passes, -`sacc_io.save(s, path, custody=c, theory={data_type: f})` with -`f(params, s, rows) -> values`, and is refused under a blind without one. -COSEBIs B_n move only by the transform's response to an E-mode shift; pure-E/B -ξ_B is not additive on noisy ξ± and also moves in isolated bins -(`test_blinding_bmodes`). Rule params carry the custody token, so a flip reruns -what it touches. +## Blinding + +Every entry of `cosmo_val/cat_config.yaml` declares `blind: none` or +`blind: `; an entry without one stops the launch, which prints one +`[blind]` line per catalogue. `_leak_corr` and `_seed` versions take their +entry's blind, and entries reading one shear file declare one blind. + +A blind is a secret record, `/.blind.json` (seed, envelope, +fiducial), drawn once with +`spv-container exec python -m sp_validation.blinding init ` and never +committed. On a blinded catalogue, ξ± and Cℓ_EE are shifted by +t(hidden) − t(fiducial) before they are first written: Smokescreen draws the +hidden point (S8 and Ωm) from the seed, and the theory `t` is +`sp_validation.theory.shear`. Cℓ_BB and Cℓ_EB get zero shift; ρ/τ are saved +with `theory.none`. Another statistic passes its own theory, +`theory(params, s) -> array the length of s.mean` (see +`src/sp_validation/theory.py`). A measurement calculates, then saves: one function computes the signal, saves -or seals it under the catalogue's custody, and returns the sealed part, so raw -signal never leaves it (`CosmologyValidation.calculate_2pcf` is the pattern). - -A blind is one record, `/.blind.json`, outside any git -worktree; read access to it is access to the blind. -`spv-container exec python -m sp_validation.blinding init ` draws one, -once; `… show ` prints its public record. Never set `blind: none` to get -a run through; never print, paste or commit a `.blind.json`. +it with `sacc_io.save(s, path, blind=...)` (or `sacc_io.seal`), and returns the +concealed part, so raw signal never leaves it +(`CosmologyValidation.calculate_2pcf` is the pattern). Statistics computed from +parts (COSEBIs, pure-E/B, the assembled `{version}.sacc`) are saved with +`derived_from=parts` and carry their inputs' stamp, `s.metadata["blind"]`. + +Producer rules carry the catalogue's blind in `params.blind`, so unblinding is +flipping the declaration to `blind: none` and rerunning: Snakemake reruns +exactly the parts it touches, and assembly refuses a part whose stamp is not the +catalogue's declared blind. ## Running on the cluster — the candide profile From 1cd0b252e94a4edfa002caaaa2a254b64791b281 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 04:19:28 +0200 Subject: [PATCH 153/160] seal refuses a SACC that already carries a blind stamp, so a loaded part can't be shifted twice Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UzeJqdtGgoeWizre32L5fD --- src/sp_validation/sacc_io.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 6688ce9b..6648816c 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -986,6 +986,11 @@ def stamp(s): def seal(s, blind, theory=None): """What :func:`save` writes for a birth, kept in memory: a copy of ``s`` concealed under ``blind`` with ``theory`` and stamped with its name.""" + if STAMP_KEY in s.metadata: + raise ValueError( + "this SACC already carries a blind stamp, so it is not a birth; " + "save it with derived_from=[its input parts]" + ) if blind.name == "none": out = s.copy() else: From 998593e0109d3b4daaf4dff265c115a74d54e91b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 04:40:37 +0200 Subject: [PATCH 154/160] test: the CPU-affinity test writes its catalogue with the shared synthetic helper Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UzeJqdtGgoeWizre32L5fD --- src/sp_validation/tests/test_cosmo_val.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 9cb3d1a8..09efcc6d 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -348,7 +348,7 @@ def test_treecorr_runs_on_the_cpus_the_process_holds(self, tmp_path): """By default TreeCorr takes the process's CPU affinity, not the node's count.""" import treecorr - params, version = self._write_synthetic_catalogs(tmp_path) + params, version = write_synthetic_catalogs(tmp_path) CosmologyValidation(versions=[version], npatch=1, **params).calculate_2pcf( version ) From 8a840bd0e80ea019248519876896b6e86a5c3f04 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 04:40:38 +0200 Subject: [PATCH 155/160] plot_pure_eb: one covariance, the semi-analytic one; no var_method argument Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UzeJqdtGgoeWizre32L5fD --- src/sp_validation/cosmo_val/pure_eb.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 17743d07..b4e4d63b 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -150,7 +150,6 @@ def plot_pure_eb( max_sep_int=300, nbins_int=1000, npatch=None, - var_method="jackknife", cov_path_int=None, cosmo_cov=None, n_samples=1000, @@ -183,9 +182,6 @@ def plot_pure_eb( (default: 0.08-300 arcmin, 1000 bins) npatch : int, optional Number of patches for jackknife covariance. Uses self.npatch if None. - var_method : str - Variance method ("jackknife" or "semi-analytic"). - Automatically set to "semi-analytic" when cov_path_int is provided. cov_path_int : str, optional Path to integration covariance matrix for semi-analytical calculation cosmo_cov : pyccl.Cosmology, optional @@ -204,8 +200,6 @@ def plot_pure_eb( This function orchestrates the full E/B mode analysis workflow: - Uses instance configuration as defaults for unspecified parameters - - Automatically switches to analytical variance when theoretical - covariance provided - Generates standardized output file naming based on all analysis parameters - Delegates individual plot generation to specialized functions in @@ -216,9 +210,7 @@ def plot_pure_eb( output_dir = output_dir or self.cc["paths"]["output"] npatch = npatch or self.npatch - # Override var_method to analytic when cov_path_int is provided - if cov_path_int is not None: - var_method = "semi-analytic" + var_method = "semi-analytic" # Use treecorr_config defaults for reporting scale binning min_sep = min_sep or self.treecorr_config["min_sep"] From a987d8f527eb39839e5705e989f2dd8bb8d9b274 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 2 Oct 2026 02:06:30 +0200 Subject: [PATCH 156/160] Blinding in one stdlib-loadable module; custody, theory and the banner go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit blinding.py now holds the theory (shear, fiducial, cosmology, and no_signal for statistics without cosmological signal) and blind_of, which reads an already-resolved catalogue entry's `blind:` and refuses a missing one or one shear file read under two blinds. Its scientific imports live inside the functions, so workflow/common.py loads it by path on the host. custody.py and theory.py are deleted. Catalogue and seed resolution are develop's again (_materialize_seed_path, catalogue_entry, get_shear_catalog); CosmologyValidation.blind and common.blind_of call blinding.blind_of on the resolved entry. The per-launch [blind] banner is gone; configure() still refuses a run catalogue that declares no blind before any job is built. Patch centres are written as on develop (PR #367 owns them). Tests keep what guards blinding: ξ±/Cℓ_EE shift and BB/EB do not; pure-E/B under a blind moves E and not B (develop's operator); seal refuses a double stamp; one file under two blinds is refused; a flip reruns the part; an undeclared catalogue stops the launch. _synthetic.py, test_blinding_bmodes.py, test_custody.py and test_toy_run.py go. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 3 +- src/sp_validation/blinding.py | 237 ++++++++++++++- src/sp_validation/cosmo_val/core.py | 74 ++++- .../cosmo_val/psf_systematics.py | 5 +- src/sp_validation/cosmo_val/real_space.py | 20 +- src/sp_validation/custody.py | 111 -------- src/sp_validation/sacc_io.py | 15 +- src/sp_validation/tests/_synthetic.py | 164 ----------- src/sp_validation/tests/conftest.py | 10 - src/sp_validation/tests/test_blinding.py | 269 +++++------------- .../tests/test_blinding_bmodes.py | 168 ----------- src/sp_validation/tests/test_cosmo_val.py | 188 +++++++++--- src/sp_validation/tests/test_custody.py | 49 ---- src/sp_validation/tests/test_masks.py | 7 + src/sp_validation/tests/test_sacc_io.py | 2 - src/sp_validation/tests/test_survey.py | 2 + src/sp_validation/theory.py | 172 ----------- workflow/README.md | 11 +- workflow/common.py | 57 ++-- workflow/rules/twopoint.smk | 2 +- workflow/tests/conftest.py | 13 +- workflow/tests/test_dag.py | 7 - workflow/tests/test_toy_run.py | 144 ---------- 23 files changed, 571 insertions(+), 1159 deletions(-) delete mode 100644 src/sp_validation/custody.py delete mode 100644 src/sp_validation/tests/_synthetic.py delete mode 100644 src/sp_validation/tests/test_blinding_bmodes.py delete mode 100644 src/sp_validation/tests/test_custody.py delete mode 100644 src/sp_validation/theory.py delete mode 100644 workflow/tests/test_toy_run.py diff --git a/CLAUDE.md b/CLAUDE.md index fa111024..7bedf4b4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,8 +42,7 @@ is the container (full scientific stack pre-built). For a local dev environment: - `cat.py`: Catalogue handling and manipulation - `cosmo_val.py`: Cosmology validation routines - `cosmology.py`: Cosmological calculations and theory -- `theory.py`: Theory data vectors for a SACC, `theory(params, s)`, from which blinding shifts are made -- `blinding.py`: Blinds, and `conceal`, which shifts a SACC by one +- `blinding.py`: Blinds, the theory `theory(params, s)` they shift by, and `conceal`, which shifts a SACC by one - `galaxy.py`: Galaxy-specific processing - `io.py`: Input/output utilities - `plots.py`: Plotting functions diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 99717403..09c49718 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -1,27 +1,54 @@ -"""Blinds, and the one call that conceals a data vector under one. +"""Blinds, the theory a blind shifts by, and the call that conceals a SACC. A blind is a named secret, ``/.blind.json`` holding a seed, the envelope the hidden point is drawn in and the fiducial point. It is drawn once, ``python -m sp_validation.blinding init ``, and never -committed (``*.blind.json`` is gitignored). +committed (``*.blind.json`` is gitignored). Each catalogue entry of the +catalogue config declares ``blind: none`` (public) or ``blind: `` +(:func:`blind_of`). :func:`conceal` adds the blind's shift, t(hidden) − t(fiducial), to every row of a SACC; Smokescreen draws the hidden point from the seed and evaluates the -theory t at both points (:mod:`sp_validation.theory`). +theory t at both points. + +A theory is a function ``theory(params, s)``: + +- ``params`` is a dict of cosmological parameters with the keys of + :func:`fiducial` (``S8``, ``Omega_m``, ``Omega_b``, ``h``, ``n_s``, + ``m_nu``, ``w0``, ``wa``, ``logT_AGN``, ``A_IA``). The fiducial and hidden + points differ only in ``S8`` and ``Omega_m``. +- ``s`` is the ``sacc.Sacc`` being concealed: its rows carry a data type, a + tracer pair and a θ or ℓ tag, its tracers the n(z). +- It returns an array shaped like the data, ``len(s.mean)`` values in the + order of ``s.data``: the prediction for each row, zero where the cosmology + has no effect. + +For Cℓ_EE/BB/EB bandpowers, for example:: + + def my_theory(params, s): + out = np.zeros(len(s.mean)) + ee = s.indices(sacc_io.CL_EE) + window = s.get_bandpower_windows(ee) + out[ee] = window.weight.T @ my_cl_ee(params, window.values) + return out # Cℓ_BB and Cℓ_EB stay zero + +:func:`shear` is the default theory; :func:`no_signal` is that of a statistic +with no cosmological signal (ρ/τ). + +The host Snakemake loads this module by path, so at import it needs only the +standard library. """ import argparse import dataclasses +import functools import json +import math import os import re import secrets from pathlib import Path -import numpy as np - -from . import theory as _theory - ENVELOPE = {"S8": 0.075, "Omega_m": 0.1} REPO_CAT_CONFIG = Path(__file__).parents[2] / "cosmo_val" / "cat_config.yaml" _NAME = re.compile(r"[a-z0-9][a-z0-9_.-]*") @@ -46,6 +73,36 @@ def record(self): NONE = Blind("none") +def _shear_file(entry): + shear = entry.get("shear") if isinstance(entry, dict) else None + if not isinstance(shear, dict) or "path" not in shear: + return None + return os.path.normpath(os.path.join(str(entry.get("subdir", "")), shear["path"])) + + +def blind_of(catalogues, entry): + """The ``blind:`` of catalogue-config entry ``entry``. + + Refuses an entry that declares none, and entries reading one shear file + under different blinds. + """ + blind = catalogues[entry].get("blind") + if not isinstance(blind, str): + raise ValueError( + f"{entry} declares no `blind:`; declare `blind: none` (public) or " + "`blind: ` in its catalogue config entry" + ) + path = _shear_file(catalogues[entry]) + readers = { + name: other.get("blind") + for name, other in catalogues.items() + if path and _shear_file(other) == path + } + if len(set(readers.values())) > 1: + raise ValueError(f"{path} is read under different blinds: {readers}") + return blind + + def registry(catalogues): """The directory a catalogue config keeps its blinds in (``paths.blinds``).""" return Path(os.path.expanduser(catalogues["paths"]["blinds"])) @@ -68,13 +125,13 @@ def conceal(s, blind, theory=None): """A copy of SACC ``s`` with ``blind``'s shift added to every row. The shift is t(hidden) − t(fiducial), t being ``theory`` (default - :func:`sp_validation.theory.shear`) evaluated on ``s``. A failure raises - :class:`BlindingError` naming only the exception type, so no message or - traceback shows the hidden point. + :func:`shear`). A failure raises :class:`BlindingError` naming only the + exception type, so no message or traceback shows the hidden point. """ + import numpy as np from smokescreen.datavector import concealing_factor - theory = theory or _theory.shear + theory = theory or shear record, failure = blind.record(), None try: shift = concealing_factor( @@ -108,7 +165,7 @@ def init(name, catalogues): record = { "seed": secrets.token_hex(32), "envelope": ENVELOPE, - "fiducial": _theory.fiducial(), + "fiducial": fiducial(), } path = where / f"{name}.blind.json" try: @@ -121,6 +178,162 @@ def init(name, catalogues): return Blind(name, path) +# --------------------------------------------------------------------------- # +# Theories +# --------------------------------------------------------------------------- # +_COSMOLOGY = ("S8", "Omega_m", "Omega_b", "h", "n_s", "m_nu", "w0", "wa", "logT_AGN") + + +def fiducial(): + """cs_util's Planck 2018 point, with feedback and no IA.""" + from cs_util.cosmo import PLANCK18 as p + + return { + "S8": float(p["sigma_8"] * math.sqrt(p["Omega_m"] / 0.3)), + **{ + k: float(p[k]) + for k in ("Omega_m", "Omega_b", "h", "n_s", "m_nu", "w0", "wa") + }, + "logT_AGN": 7.5, + "A_IA": 0.0, + } + + +def no_signal(params, s): + """Zeros: the theory of a statistic with no cosmological signal (ρ/τ).""" + import numpy as np + + return np.zeros(len(s.mean)) + + +def shear(params, s): + """Cosmic-shear ξ± and Cℓ_EE from pyccl; zeros for Cℓ_BB and Cℓ_EB. + + ξ± is evaluated at each row's stored θ (TreeCorr's ``meanr``) and Cℓ_EE + through each row's bandpower window. Any other data type raises: pass your + own theory. + """ + import numpy as np + + from .sacc_io import CL_BB, CL_EB, CL_EE, XI_MINUS, XI_PLUS + + out = np.zeros(len(s.mean)) + groups = {} + for i, dp in enumerate(s.data): + if dp.data_type in (XI_PLUS, XI_MINUS): + key = (_xi, dp.tracers) + elif dp.data_type == CL_EE: + key = (_cl, dp.tracers, id(dp.tags.get("window"))) + elif dp.data_type in (CL_BB, CL_EB): + continue + else: + raise ValueError( + f"blinding.shear has no prediction for {dp.data_type}; pass your " + "own theory" + ) + groups.setdefault(key, []).append(i) + # One call per tracer pair (and window), so ξ+ and ξ− share their Cℓ. + for (function, *_), rows in groups.items(): + out[rows] = function(params, s, np.asarray(rows)) + return out + + +def cosmology(params): + """The ``pyccl.Cosmology`` at ``params``, built once per point. + + Built through ``cs_util.cosmo.get_cosmo`` on the CAMB HMCode2020-feedback + route the CosmoSIS inference runs, with σ8 = S8/√(Ωm/0.3); ``get_cosmo`` + subtracts CAMB's Ω_ν from ``Omega_m`` for Ω_c. + """ + return _cosmology(tuple(float(params[k]) for k in _COSMOLOGY)) + + +@functools.lru_cache(maxsize=4) +def _cosmology(point): + from cs_util.cosmo import get_cosmo + + p = dict(zip(_COSMOLOGY, point)) + cosmo = get_cosmo( + Omega_m=p["Omega_m"], + Omega_b=p["Omega_b"], + h=p["h"], + sig8=p["S8"] / math.sqrt(p["Omega_m"] / 0.3), + ns=p["n_s"], + w0=p["w0"], + wa=p["wa"], + mnu=p["m_nu"], + # get_cosmo's default "halofit" ignores extra_params. + matter_power_spectrum="camb", + extra_params={ + "camb": { + "halofit_version": "mead2020_feedback", + "HMCode_logT_AGN": p["logT_AGN"], + } + }, + ) + # CCL's Hankel transform extrapolates C_ℓ to ELL_MAX_CORR; its default + # (6·10⁴) rings in ξ− below a few arcmin. + cosmo.cosmo.spline_params.ELL_MAX_CORR = 10_000_000 + cosmo.cosmo.spline_params.N_ELL_CORR = 5_000 + return cosmo + + +@functools.cache +def _ell(): + """The multipoles the ξ± Hankel transform integrates over: every ℓ below 50, + then 200 log-spaced up to 6·10⁴.""" + import numpy as np + + return np.unique(np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)])) + + +def _lensing(cosmo, params, s, name): + import numpy as np + import pyccl as ccl + + tracer = s.tracers[name] + if getattr(tracer, "nz", None) is None: + raise ValueError(f"tracer {name} has no n(z)") + z, n = np.asarray(tracer.z, float), np.asarray(tracer.nz, float) + ia = None if params["A_IA"] == 0 else (z, np.full_like(z, params["A_IA"])) + return ccl.WeakLensingTracer(cosmo, dndz=(z, n), ia_bias=ia) + + +def _shear_cl(params, s, rows, ell): + import pyccl as ccl + + cosmo = cosmology(params) + a, b = (_lensing(cosmo, params, s, t) for t in s.data[rows[0]].tracers) + return ccl.angular_cl(cosmo, a, b, ell) + + +def _xi(params, s, rows): + """ξ± of one shear pair: Limber Cℓ on :func:`_ell`, then CCL's Hankel transform.""" + import numpy as np + import pyccl as ccl + + from .sacc_io import XI_PLUS + + ell = _ell() + cosmo, cl = cosmology(params), _shear_cl(params, s, rows, ell) + theta = np.array([s.data[i].tags["theta"] for i in rows]) / 60.0 + xip, xim = ( + ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta, type=t) + for t in ("GG+", "GG-") + ) + plus = np.array([s.data[i].data_type == XI_PLUS for i in rows]) + return np.where(plus, xip, xim) + + +def _cl(params, s, rows): + """Cℓ_EE of one shear pair through its bandpower window.""" + import numpy as np + + window = s.get_bandpower_windows(rows) + cl = _shear_cl(params, s, rows, np.asarray(window.values, float)) + return np.asarray(window.weight).T @ cl + + def main(argv=None): import yaml diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index dbda9b1b..f66c19b6 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -10,7 +10,6 @@ from shear_psf_leakage import run_object, run_scale from .. import blinding -from .. import custody as _custody from ..b_modes import ( _get_pte_from_scale_cut, covariance_label, @@ -167,6 +166,64 @@ def _split_seed_variant(version): return None, None return base, seed_label + @staticmethod + def _materialize_seed_path( + base_cfg, seed_label, version, base_version, catalog_config + ): + """Render the seed-specific shear path using Python string formatting.""" + shear_cfg = base_cfg["shear"] + template = shear_cfg.get("path_template") + + try: + seed_value = int(seed_label) + except ValueError as error: + raise ValueError( + f"Seed suffix for '{version}' is not numeric; cannot materialize path." + ) from error + + format_context = {"seed": seed_value, "seed_label": seed_label} + + if template: + try: + return template.format(**format_context) + except KeyError as error: + raise KeyError( + f"Missing placeholder '{error.args[0]}' in path_template for " + f"'{base_version}' while materializing '{version}'. Update " + f"{catalog_config}." + ) from error + except ValueError as error: + raise ValueError( + f"Invalid format specification in path_template for '{base_version}' " + f"while materializing '{version}'." + ) from error + + path = shear_cfg.get("path", "") + token_start = path.rfind("seed") + if token_start == -1: + raise ValueError( + f"Cannot materialize '{version}': '{base_version}' lacks a shear " + f"path_template and its shear path '{path}' does not contain a 'seed' " + f"token. Update {catalog_config}." + ) + cursor = token_start + 4 # len("seed") + if cursor < len(path) and not path[cursor].isdigit(): + cursor += 1 + digit_start = cursor + while cursor < len(path) and path[cursor].isdigit(): + cursor += 1 + digit_end = cursor + digits = path[digit_start:digit_end] + if not digits: + raise ValueError( + f"Cannot materialize '{version}': shear path '{path}' for base version " + f"'{base_version}' lacks digits after the seed token. Update " + f"{catalog_config}." + ) + + template = f"{path[:digit_start]}{{seed_label}}{path[digit_end:]}" + return template.format(**format_context) + def __init__( self, versions, @@ -267,8 +324,6 @@ def __init__( self.catalog_config_path = Path(catalog_config) with self.catalog_config_path.open("r") as file: self.cc = cc = yaml.load(file, Loader=yaml.FullLoader) - # The catalogues as declared, before virtual versions are materialised. - self._declared = copy.deepcopy(cc) def resolve_paths_for_version(ver): """Resolve relative paths for a version using its subdir.""" @@ -320,9 +375,14 @@ def ensure_version_exists(ver): ensure_version_exists(seed_base) if ver not in cc: cc[ver] = copy.deepcopy(cc[seed_base]) - cc[ver]["shear"]["path"] = _custody.seed_path( - cc[seed_base]["shear"], seed_label + seed_path = self._materialize_seed_path( + cc[seed_base], + seed_label, + ver, + seed_base, + catalog_config, ) + cc[ver]["shear"]["path"] = seed_path resolve_paths_for_version(ver) processed.add(ver) return @@ -454,9 +514,7 @@ def results_objectwise(self): def blind(self, version): """The blind every SACC this object writes for ``version`` is saved under, as the catalogue config declares it.""" - return blinding.open_blind( - _custody.declared(self._declared, version), self._declared - ) + return blinding.open_blind(blinding.blind_of(self.cc, version), self.cc) def basename(self, version, treecorr_config=None, npatch=None): cfg = treecorr_config or self.treecorr_config diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index 88c4d1f0..d9ac5304 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -16,7 +16,7 @@ from shear_psf_leakage.rho_tau_stat import PSFErrorFit from uncertainties import ufloat -from .. import sacc_io, theory +from .. import blinding, sacc_io from ..rho_tau import ( get_rho_tau_w_cov, get_samples, @@ -78,8 +78,7 @@ def rho_tau_to_sacc_part( tau_cov_th=tau_cov_th, ) out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") - # ρ/τ carry no cosmological signal: the blind leaves them unshifted. - sacc_io.save(s, out_path, blind=self.blind(version), theory=theory.none) + sacc_io.save(s, out_path, blind=self.blind(version), theory=blinding.no_signal) @property def rho_stat_handler(self): diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index a20a9729..d8313f7e 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -7,7 +7,6 @@ """ import os -import uuid import matplotlib.pyplot as plt import matplotlib.ticker as mticker @@ -31,14 +30,9 @@ def calculate_2pcf( ): """ξ± of ``ver`` on one binning, as its SACC part sealed under its blind. - A catalogue's ξ± leaves this object only as a part sealed under the - catalogue's blind (:func:`sp_validation.sacc_io.seal`), concealed - before it is returned, cached or written. The blind is opened first, - so a blind that cannot open fails before TreeCorr runs. - - With patches, the catalogue splits at the output tree's - ``{ver}_patches_npatch={npatch}.dat``, drawn by TreeCorr's k-means and - written there when missing. + The part is concealed (:func:`sp_validation.sacc_io.seal`) before it is + returned, cached or written. Patches split at the output tree's + ``{ver}_patches_npatch={npatch}.dat``, written when missing. Parameters: ver (str): The catalogue version to measure. @@ -78,12 +72,8 @@ def calculate_2pcf( npatch=npatch, patch_centers=patch_file if os.path.exists(patch_file) else None, ) - # Through a temporary file, so a concurrent reader never sees a - # torn one. - if jackknife and not os.path.exists(patch_file): - tmp = f"{patch_file}.{uuid.uuid4().hex}.tmp" - catalogue.write_patch_centers(tmp) - os.replace(tmp, patch_file) + if not os.path.exists(patch_file): + catalogue.write_patch_centers(patch_file) gg.process(catalogue) s = xi_to_sacc( diff --git a/src/sp_validation/custody.py b/src/sp_validation/custody.py deleted file mode 100644 index 12bacfe1..00000000 --- a/src/sp_validation/custody.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Which blind each catalogue is declared under, and where its shear file is. - -Every catalogue entry of the catalogue config declares ``blind: none`` or -``blind: `` (:mod:`sp_validation.blinding`); an entry without one is -refused. ``_leak_corr`` and ``_seed`` versions take their entry's, and -entries reading one shear file declare one blind. - -The host Snakemake loads this module by path, so it imports only the standard -library. -""" - -import os -import re - -NOT_CATALOGUES = ("nz", "paths") -_SEED_SUFFIX = re.compile(r"_seed\d+$") -_LEAK_SUFFIX = "_leak_corr" - - -def _variant_chain(version): - """``version``, then each name that stripping ``_leak_corr``/``_seed`` leaves.""" - chain = [version] - while True: - name = chain[-1] - stripped = ( - name[: -len(_LEAK_SUFFIX)] - if name.endswith(_LEAK_SUFFIX) - else _SEED_SUFFIX.sub("", name) - ) - if stripped in ("", name): - return chain - chain.append(stripped) - - -def entry_name(catalogues, version): - """The config entry describing ``version``: its own, or its variant chain's.""" - for name in _variant_chain(version): - if name in catalogues and name not in NOT_CATALOGUES: - return name - raise ValueError(f"no catalogue {version} in the catalogue config") - - -def seed_path(shear, seed): - """The shear path of seed label ``seed``: ``path_template`` formatted, or the - digits after ``path``'s last ``seed`` replaced.""" - if shear.get("path_template"): - return shear["path_template"].format(seed=int(seed), seed_label=seed) - path = shear.get("path", "") - match = re.match(r"(.*seed\D?)\d+", path) - if match is None: - raise ValueError(f"shear path {path!r} has no seed to put a seed in") - return match.group(1) + seed + path[match.end() :] - - -def _reads(entry): - shear = entry.get("shear") if isinstance(entry, dict) else None - if not isinstance(shear, dict) or "path" not in shear: - return None - return os.path.normpath(os.path.join(str(entry.get("subdir", "")), shear["path"])) - - -def shear_file(catalogues, version): - """The shear catalogue file ``version`` reads, a ``_seed``'s seed rendered.""" - name = entry_name(catalogues, version) - entry, chain = catalogues[name], _variant_chain(version) - seeds = [ - m.group()[5:] - for n in chain[: chain.index(name)] - if (m := _SEED_SUFFIX.search(n)) - ] - if seeds: - entry = { - **entry, - "shear": {**entry["shear"], "path": seed_path(entry["shear"], seeds[0])}, - } - return _reads(entry) - - -def declared(catalogues, version): - """The blind ``version`` is declared under: ``none`` or a blind's name.""" - name = entry_name(catalogues, version) - blind = catalogues[name].get("blind") - if not isinstance(blind, str): - raise ValueError( - f"{name} declares no `blind:`; declare `blind: none` (public) or " - "`blind: ` in its catalogue config entry" - ) - path = _reads(catalogues[name]) - others = { - other: entry.get("blind") - for other, entry in catalogues.items() - if other not in NOT_CATALOGUES and path and _reads(entry) == path - } - if len(set(others.values())) > 1: - raise ValueError( - f"{path} is read by entries declaring different blinds: {others}" - ) - return blind - - -def summary(catalogues, versions): - """One ``[blind]`` line per catalogue entry among ``versions``.""" - by_entry = {} - for version in dict.fromkeys(versions): - by_entry.setdefault(entry_name(catalogues, version), []).append(version) - lines = [] - for name, members in by_entry.items(): - also = [v for v in members if v != name] - also = f" (+ {', '.join(also)})" if also else "" - lines.append(f"[blind] {name}{also}: {declared(catalogues, name)}") - return lines diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index ce874d8e..2c2e7919 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -1006,21 +1006,14 @@ def save(s, path, *, blind=None, theory=None, derived_from=None): - ``save(s, path, blind=b)``: a birth. ``b`` is a :class:`sp_validation.blinding.Blind`; unless it is ``none``, every row - of ``s`` is shifted by t(hidden) − t(fiducial), t being ``theory``. The - output is stamped with ``b``'s name. + of ``s`` is shifted by t(hidden) − t(fiducial), t being ``theory`` + (:mod:`sp_validation.blinding` describes a theory; the default is + :func:`sp_validation.blinding.shear`). The output is stamped with + ``b``'s name. - ``save(s, path, derived_from=parts)``: a derivation (COSEBIs, pure-E/B, an assembly); the parts must share one :func:`stamp`, which ``s`` takes, unshifted. With ``blind=b`` too, that stamp must be ``b``'s name. - ``theory(params, s)`` returns the prediction for every row of ``s``: an - array the length of ``s.mean``, in its order, zero where the cosmology has - no effect. ``params`` is a plain dict with the keys of - :func:`sp_validation.theory.fiducial`; the theory is called at the blind's - fiducial and hidden points, which differ only in ``S8`` and ``Omega_m``. - The default, :func:`sp_validation.theory.shear`, predicts ξ± and Cℓ_EE and - gives zeros for Cℓ_BB and Cℓ_EB; :func:`sp_validation.theory.none` gives - zeros throughout. - A measurement computes its signal and saves (or seals) it in the same function, returning the sealed part, so on a blinded catalogue its raw signal never leaves that function. diff --git a/src/sp_validation/tests/_synthetic.py b/src/sp_validation/tests/_synthetic.py deleted file mode 100644 index 72f63f1f..00000000 --- a/src/sp_validation/tests/_synthetic.py +++ /dev/null @@ -1,164 +0,0 @@ -"""A small deterministic synthetic catalogue, on disk, with its catalogue config. - -The glue tests run the real compute seams on it: a shear catalogue -(RA/Dec/e1/e2/w), a PSF star catalogue with the columns the leakage and ρ/τ -seams read, a cs_util-readable dndz, and a ``cat_config.yaml`` whose blinds -live in ``blinds/`` beside it. Every catalogue entry in the config reads its -own copy of the same galaxies and declares its own blind. - -``toy_shear`` stands in for :func:`sp_validation.theory.shear`, so a blind -shifts without running CAMB. -""" - -import copy - -import numpy as np -import yaml - -from sp_validation import sacc_io - -SIGNAL = (sacc_io.XI_PLUS, sacc_io.XI_MINUS, sacc_io.CL_EE) - - -def toy_shear(params, s): - """A power law in θ or ℓ for ξ± and Cℓ_EE, its amplitude growing with S8 - and Ωm; zeros for every other row.""" - out = np.zeros(len(s.mean)) - for i, dp in enumerate(s.data): - if dp.data_type in SIGNAL: - x = dp.tags.get("theta", dp.tags.get("ell")) - out[i] = params["S8"] ** 2 * params["Omega_m"] ** 0.3 * (x / 10.0) ** -0.8 - return 1e-4 * out - - -def write_synthetic_catalogs( - tmp_path, - n_gal=2000, - n_star=800, - ra_range=(10.0, 14.0), - dec_range=(10.0, 14.0), - seed=1234, - coherent_shear=False, - with_psf=False, - catalogues=None, -): - """Write the catalogue files and their config; return ``(params, version)``. - - Parameters - ---------- - coherent_shear : bool - Inject a smooth position-dependent shear on top of shape noise, so ξ± - is smooth (the pure-E/B integral needs it to be well-posed). - with_psf : bool - Add a ``psf`` block (ρ/τ and pseudo-Cℓ read it via - ``get_params_rho_tau``). - catalogues : dict, optional - ``{version: blind}``, one catalogue entry each, reading its own copy of - the shear catalogue. ``blind`` is the entry's ``blind:`` value, or - ``None`` for an entry that declares none. Defaults to one public - catalogue, ``TestCatalog``. - - Returns - ------- - (dict, str) - ``CosmologyValidation`` keyword arguments (``catalog_config``, - ``output_dir``) and the first catalogue's version. - """ - from astropy.table import Table - - catalogues = catalogues or {"TestCatalog": "none"} - rng = np.random.default_rng(seed) - - cat_dir = tmp_path / "catalog" - nz_dir = tmp_path / "nz" - output_dir = tmp_path / "output" - for directory in (cat_dir, nz_dir, output_dir): - directory.mkdir() - - ra = rng.uniform(*ra_range, n_gal) - dec = rng.uniform(*dec_range, n_gal) - if coherent_shear: - e1 = 0.02 * np.cos(np.radians(ra) * 40) + rng.normal(0, 0.05, n_gal) - e2 = 0.02 * np.sin(np.radians(dec) * 40) + rng.normal(0, 0.05, n_gal) - else: - e1 = rng.normal(0, 0.25, n_gal) - e2 = rng.normal(0, 0.25, n_gal) - w = rng.uniform(0.5, 1.0, n_gal) - shear = Table({"RA": ra, "Dec": dec, "e1": e1, "e2": e2, "w": w}) - - Table( - { - "RA": rng.uniform(*ra_range, n_star), - "Dec": rng.uniform(*dec_range, n_star), - "HSM_G1_PSF": rng.normal(0, 0.03, n_star), - "HSM_G2_PSF": rng.normal(0, 0.03, n_star), - "HSM_G1_STAR": rng.normal(0, 0.03, n_star), - "HSM_G2_STAR": rng.normal(0, 0.03, n_star), - "HSM_T_PSF": rng.uniform(0.4, 0.6, n_star), - "HSM_T_STAR": rng.uniform(0.4, 0.6, n_star), - "HSM_FLAG_PSF": np.zeros(n_star, dtype=int), - "HSM_FLAG_STAR": np.zeros(n_star, dtype=int), - } - ).write(cat_dir / "star.fits", overwrite=True) - - # cs_util.read_dndz's commented-header format: "z" holds the n+1 bin - # edges, "dn_dz" the densities. - z_edges = np.linspace(0.05, 3.0, 31) - dndz = np.exp(-(((z_edges - 0.7) / 0.3) ** 2)) - lines = ["# z dn_dz"] + [f"{zz} {nn}" for zz, nn in zip(z_edges, dndz)] - (nz_dir / "dndz_SP_A.txt").write_text("\n".join(lines) + "\n") - - entry = { - "subdir": str(cat_dir), - "pipeline": "SP", - "colour": "orange", - "ls": "-", - "marker": "o", - "shear": { - "redshift_path": str(nz_dir / "dndz_SP_A.txt"), - "w_col": "w", - "e1_col": "e1", - "e2_col": "e2", - "R": 1.0, - "e1_col_corrected": "e1", - "e2_col_corrected": "e2", - }, - "star": { - "path": "star.fits", - "ra_col": "RA", - "dec_col": "Dec", - "e1_col": "HSM_G1_PSF", - "e2_col": "HSM_G2_PSF", - }, - } - if with_psf: - entry["psf"] = { - "path": "star.fits", - "hdu": 1, - "ra_col": "RA", - "dec_col": "Dec", - "e1_PSF_col": "HSM_G1_PSF", - "e2_PSF_col": "HSM_G2_PSF", - "e1_star_col": "HSM_G1_STAR", - "e2_star_col": "HSM_G2_STAR", - "PSF_size": "HSM_T_PSF", - "star_size": "HSM_T_STAR", - "PSF_flag": "HSM_FLAG_PSF", - "star_flag": "HSM_FLAG_STAR", - } - - config = { - "nz": {"subdir": str(nz_dir), "dndz": {"path": "dndz_{pipeline}_A.txt"}}, - "paths": {"output": str(output_dir), "blinds": str(tmp_path / "blinds")}, - } - for version, blind in catalogues.items(): - config[version] = copy.deepcopy(entry) - config[version]["shear"]["path"] = f"shear_{version}.fits" - shear.write(cat_dir / f"shear_{version}.fits", overwrite=True) - if blind is not None: - config[version]["blind"] = blind - config_path = tmp_path / "cat_config.yaml" - config_path.write_text(yaml.safe_dump(config, sort_keys=False)) - - params = {"catalog_config": str(config_path), "output_dir": str(output_dir)} - return params, next(iter(catalogues)) diff --git a/src/sp_validation/tests/conftest.py b/src/sp_validation/tests/conftest.py index fbfdce9c..004abc83 100644 --- a/src/sp_validation/tests/conftest.py +++ b/src/sp_validation/tests/conftest.py @@ -18,13 +18,3 @@ def pure_eb_xi(): """ with np.load(PURE_EB_XI) as npz: return {k: (v.item() if v.ndim == 0 else v) for k, v in npz.items()} - - -@pytest.fixture -def toy_theory(monkeypatch): - """Blinds shift by the analytic ``_synthetic.toy_shear``.""" - from _synthetic import toy_shear - - from sp_validation import theory - - monkeypatch.setattr(theory, "shear", toy_shear) diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 171f2a59..d148e88b 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -1,215 +1,104 @@ -"""Blinds, concealment and the blind stamp. +"""What a blind shifts, and what the catalogue config may declare. -Blinds here shift by the analytic ``toy_shear``; one slow test runs the CCL -default. +The shifts run the default theory, :func:`sp_validation.blinding.shear`, under +a blind whose hidden point raises S8 above the fiducial. """ import json -import stat import numpy as np import pytest -from _synthetic import toy_shear -from smokescreen.param_shifts import draw_param_shifts +from sp_validation import b_modes from sp_validation import blinding as bd from sp_validation import sacc_io as sio -from sp_validation import theory - -DELTA_SIGMA = "galaxy_shearDensity_xi_t" - - -@pytest.fixture(scope="module") -def catalogues(tmp_path_factory): - return {"paths": {"blinds": str(tmp_path_factory.mktemp("blinds"))}} - - -@pytest.fixture(scope="module") -def toy(catalogues): - return bd.init("toy", catalogues) @pytest.fixture(scope="module") -def other(catalogues): - return bd.init("other", catalogues) +def up(tmp_path_factory): + path = tmp_path_factory.mktemp("blinds") / "up.blind.json" + path.write_text( + json.dumps( + {"seed": "s", "envelope": {"S8": [0.05, 0.06]}, "fiducial": bd.fiducial()} + ) + ) + return bd.Blind("up", path) -def _nz(z0): +def _sacc(): z = np.linspace(0.0, 3.0, 200) - return z, np.exp(-0.5 * ((z - z0) / 0.2) ** 2) + return sio.new_sacc({0: (z, np.exp(-0.5 * ((z - 0.7) / 0.2) ** 2))}) -def part(*, cl=True, rho=False, delta_sigma=False): - """ξ± and pseudo-Cℓ (EE, BB, EB) on pairs (0,0), (0,1), (1,1), and - optionally ρ and a ΔΣ-like custom type.""" - s = sio.new_sacc({0: _nz(0.5), 1: _nz(0.9)}) +@pytest.mark.slow +def test_a_blind_shifts_xi_and_cl_ee_and_leaves_bb_eb(up): + s = _sacc() theta = np.geomspace(2.0, 200.0, 6) + sio.add_xi(s, (0, 0), theta, 0 * theta, 0 * theta, grid="reporting") ell = np.array([30.0, 80.0, 150.0, 280.0, 450.0]) w_ell = np.arange(2, 501).astype(float) - window = np.exp(-0.5 * ((w_ell[:, None] - ell[None, :]) / 40.0) ** 2) - for pair in ((0, 0), (0, 1), (1, 1)): - xip, xim = 1e-4 * (theta / 10) ** -0.6, 0.5e-4 * (theta / 10) ** -0.9 - sio.add_xi(s, pair, theta, xip, xim, grid="reporting") - if cl: - ee = 1e-8 * (ell / 100.0) ** -1.2 - sio.add_pseudo_cl( - s, - pair, - ell, - ee, - 0.01 * ee, - 0.02 * ee, - window_ells=w_ell, - window_weights=window / window.sum(axis=0), - ) - if rho: - sio.add_rho(s, 0, theta, np.arange(1, 7) * 1e-7, np.arange(1, 7) * 2e-7) - if delta_sigma: - for r in (0.5, 2.0, 8.0): # tagged theta, as sacc requires - s.add_data_point(DELTA_SIGMA, ("source_0", "source_1"), 10.0, theta=r) - s.add_covariance(np.abs(np.asarray(s.mean)) ** 2 + 1e-20) - return s - - -def hidden(blind): - """The blind's hidden point, as Smokescreen draws it.""" - record = blind.record() - shift = draw_param_shifts(record["envelope"], record["seed"]) - return {k: v + shift.get(k, 0.0) for k, v in record["fiducial"].items()} - - -# --------------------------------------------------------------------------- # -# Concealing: every row moves by its theory's t(hidden) − t(fiducial) -# --------------------------------------------------------------------------- # -def test_conceal_moves_signal_rows_by_the_theory_shift(toy, other): - """ξ± and Cℓ_EE move by t(hidden) − t(fiducial); Cℓ_BB/EB and the - covariance stay identical; the same blind gives the same shift, another - blind another.""" - s = part() - shift = bd.conceal(s, toy, toy_shear).mean - s.mean - fiducial = toy.record()["fiducial"] - expected = toy_shear(hidden(toy), s) - toy_shear(fiducial, s) - np.testing.assert_allclose(shift, expected, rtol=1e-10, atol=0) - signal = np.isin([dp.data_type for dp in s.data], [sio.XI_PLUS, sio.XI_MINUS]) - signal |= np.array([dp.data_type == sio.CL_EE for dp in s.data]) - assert np.all(shift[signal] != 0) and np.all(shift[~signal] == 0) - concealed = bd.conceal(s, toy, toy_shear) - np.testing.assert_array_equal(concealed.covariance.dense, s.covariance.dense) - np.testing.assert_array_equal(concealed.mean - s.mean, shift) - assert not np.allclose(bd.conceal(s, other, toy_shear).mean - s.mean, shift) - - -def test_a_theory_failure_names_no_value(toy): - """Any failure, including a vector of the wrong length, is a - BlindingError carrying neither the hidden point nor the original message.""" - - def broken(params, s): - raise RuntimeError(f"S8 = {params['S8']}") - - with pytest.raises(bd.BlindingError) as err: - bd.conceal(part(), toy, broken) - assert "S8" not in str(err.value) and "RuntimeError" in str(err.value) - assert err.value.__cause__ is None and err.value.__context__ is None - with pytest.raises(bd.BlindingError, match="values for"): - bd.conceal(part(), toy, lambda params, s: np.zeros(3)) - - -def test_a_custom_theory_blinds_a_custom_type(toy): - """A ΔΣ-like type is shifted by a theory that knows it; the default - theory refuses it; theory.none leaves every value but stamps.""" - - def delta_sigma(params, s): - out = toy_shear(params, s) - for i, dp in enumerate(s.data): - if dp.data_type == DELTA_SIGMA: - out[i] = 10.0 * params["S8"] / dp.tags["theta"] - return out - - s = part(cl=False, delta_sigma=True) - rows = s.indices(DELTA_SIGMA) - sealed = sio.seal(s, toy, delta_sigma) - assert np.all(sealed.mean[rows] != s.mean[rows]) - assert sio.stamp(sealed) == "toy" - with pytest.raises(bd.BlindingError, match="ValueError"): - sio.seal(s, toy) # theory.shear: no prediction for this type - rho = part(cl=False, rho=True) - kept = sio.seal(rho, toy, theory.none) - np.testing.assert_array_equal(kept.mean, rho.mean) - assert sio.stamp(kept) == "toy" - - -def test_none_stamps_without_concealing(tmp_path): - s = part() - saved = sio.save(s, tmp_path / "x.sacc", blind=bd.NONE) - np.testing.assert_array_equal(saved.mean, s.mean) - assert sio.stamp(sio.load(tmp_path / "x.sacc")) == "none" - assert sio.stamp(part()) == "none" # absent reads as none - with pytest.raises(ValueError, match="blind= .* or derived_from="): - sio.save(s, tmp_path / "y.sacc") - - -# --------------------------------------------------------------------------- # -# Derivations inherit their inputs' one stamp -# --------------------------------------------------------------------------- # -@pytest.mark.parametrize( - "inputs, declared, allowed", - [ - (["toy"], None, True), - (["toy", "toy"], "toy", True), # an assembly - (["none"], None, True), - (["toy", "other"], None, False), # mixed stamps - (["toy", "none"], None, False), - (["none"], "toy", False), # a stale public part under a blind - (["other"], "toy", False), - ], -) -def test_a_derivation_carries_its_inputs_one_stamp( - toy, other, tmp_path, inputs, declared, allowed -): - blinds = {"toy": toy, "other": other, "none": bd.NONE} - parts = [sio.seal(part(cl=False), blinds[b], toy_shear) for b in inputs] - path = tmp_path / "d.sacc" - kwargs = dict(derived_from=parts, blind=bd.Blind(declared) if declared else None) - if not allowed: - with pytest.raises(ValueError): - sio.save(parts[0], path, **kwargs) - assert not path.exists() - return - derived = sio.save(parts[0], path, **kwargs) - assert sio.stamp(sio.load(path)) == inputs[0] - np.testing.assert_array_equal(derived.mean, parts[0].mean) - - -# --------------------------------------------------------------------------- # -# The blind: drawn once, opened by name -# --------------------------------------------------------------------------- # -def test_a_blind_is_drawn_once_and_opened_by_name(tmp_path): - catalogues = {"paths": {"blinds": str(tmp_path / "blinds")}} - blind = bd.init("toy", catalogues) - assert stat.S_IMODE(blind.path.stat().st_mode) == 0o440 - assert set(json.loads(blind.path.read_text())) == {"seed", "envelope", "fiducial"} - with pytest.raises(bd.BlindingError, match="exists"): - bd.init("toy", catalogues) - assert bd.open_blind("toy", catalogues) == blind - assert bd.open_blind("none", {}) is bd.NONE - with pytest.raises(bd.BlindingError, match="init gone"): - bd.open_blind("gone", catalogues) + window = np.exp(-0.5 * ((w_ell[:, None] - ell) / 40.0) ** 2) + zeros = 0 * ell + sio.add_pseudo_cl( + s, + (0, 0), + ell, + zeros, + zeros, + zeros, + window_ells=w_ell, + window_weights=window / window.sum(axis=0), + ) + + sealed = sio.seal(s, up) + delta = sealed.mean - s.mean + for data_type in (sio.XI_PLUS, sio.XI_MINUS, sio.CL_EE): + assert np.all(delta[s.indices(data_type)] > 0), data_type + for data_type in (sio.CL_BB, sio.CL_EB): + assert np.all(delta[s.indices(data_type)] == 0), data_type + assert sio.stamp(sealed) == "up" @pytest.mark.slow -def test_the_ccl_default_shifts_with_s8(tmp_path): - """theory.shear, at a hidden point with S8 above the fiducial, raises ξ+ - and Cℓ_EE and leaves Cℓ_BB/EB.""" - record = { - "seed": "s", - "envelope": {"S8": [0.05, 0.06]}, - "fiducial": theory.fiducial(), +def test_pure_eb_under_a_blind_shifts_e_and_not_b(up): + """The pure-E/B operator on the integration-grid ξ± is linear, so a blind + moves the modes by its response to the shift: E by several σ, B by under + a tenth of σ (σ: UNIONS shape noise through the operator).""" + edges_int = np.geomspace(0.08, 300.0, 1001) + theta_int = np.sqrt(edges_int[:-1] * edges_int[1:]) + s = _sacc() + sio.add_xi(s, (0, 0), theta_int, 0 * theta_int, 0 * theta_int, grid="integration") + shifted = sio.xi_correlation(sio.seal(s, up)) + delta = np.r_[shifted.xip, shifted.xim] + + annuli = np.diff(edges_int**2) # ∝ pair counts on a uniform field + operator, _, _ = b_modes.pure_eb_operator( + annuli, edges_int, np.geomspace(1.0, 250.0, 21) + ) + # Var ξ± = σ_e⁴ / (2 N_pairs), SP_v1.4.6.3's area, n_eff and σ_e. + n_eff, area, sigma_e = 4.96, 2894.0 * 3600, 0.378 + variance = np.tile(sigma_e**4 / (n_eff**2 * area * np.pi * annuli), 2) + sigma = np.sqrt(np.einsum("ij,j,ij->i", operator, variance, operator)) + response = operator @ delta / sigma + + n = len(response) // 6 # blocks in _EB_KEYS order: E+, E−, B+, B−, amb + assert np.max(np.abs(response[: 2 * n])) > 1 + assert np.max(np.abs(response[2 * n : 4 * n])) < 0.1 + + +def test_seal_refuses_a_sacc_already_stamped(): + sealed = sio.seal(_sacc(), bd.NONE) + with pytest.raises(ValueError, match="already carries a blind stamp"): + sio.seal(sealed, bd.NONE) + + +def test_entries_reading_one_file_declare_one_blind(): + catalogues = { + "paths": {"blinds": "/blinds"}, + "Y3": {"blind": "y3", "subdir": "/d", "shear": {"path": "y3.fits"}}, + "TWIN": {"blind": "none", "subdir": "/d", "shear": {"path": "twin.fits"}}, } - path = tmp_path / "up.blind.json" - path.write_text(json.dumps(record)) - s = part() - delta = bd.conceal(s, bd.Blind("up", path)).mean - s.mean - for data_type in (sio.XI_PLUS, sio.CL_EE): - assert np.all(delta[s.indices(data_type)] > 0), data_type - for data_type in (sio.CL_BB, sio.CL_EB): - assert np.all(delta[s.indices(data_type)] == 0), data_type + assert bd.blind_of(catalogues, "TWIN") == "none" + catalogues["TWIN"]["shear"]["path"] = "/d/sub/../y3.fits" + with pytest.raises(ValueError, match="different blinds"): + bd.blind_of(catalogues, "TWIN") diff --git a/src/sp_validation/tests/test_blinding_bmodes.py b/src/sp_validation/tests/test_blinding_bmodes.py deleted file mode 100644 index cf41f962..00000000 --- a/src/sp_validation/tests/test_blinding_bmodes.py +++ /dev/null @@ -1,168 +0,0 @@ -"""B-modes under a blind, on synthetic ξ±. - -A blind shifts ξ± by one E-mode signal, δ = t(hidden) − t(fiducial), so it -moves COSEBIs B_n and pure-E/B ξ_B only by each transform's response to a pure -E-mode vector. The input is the default theory's ξ± at the fiducial on the -production reporting and integration grids, noise-free and (for COSEBIs) with -one seeded shape-noise draw, and δ is taken with the hidden point at each -corner of the envelope in turn. Both transforms are linear with fixed -weights, so B(x + δ) − B(x) = B(δ) to round-off, and the transform's response -to δ must stay under a tenth of a bin's standard deviation. -""" - -import numpy as np -import pytest - -from sp_validation import b_modes, theory -from sp_validation import blinding as bd -from sp_validation import sacc_io as sio - -REPORTING = (1.0, 250.0, 20) -INTEGRATION = (0.08, 300.0, 1000) -COSEBIS_CUT, NMODES = (12.0, 83.0), 20 -# SP_v1.4.6.3's shape noise (its cov_th in cosmo_val/cat_config.yaml). -AREA_DEG2, N_EFF, SIGMA_E = 2894.0, 4.96, 0.378 -CEILING = 0.1 - - -def _grid(lo, hi, n): - edges = np.geomspace(lo, hi, n + 1) - return np.sqrt(edges[:-1] * edges[1:]), edges - - -THETA, EDGES = _grid(*REPORTING) -THETA_INT, EDGES_INT = _grid(*INTEGRATION) - - -def _variance(edges): - """Var ξ± = σ_e⁴ / (2 N_pairs) (Schneider et al. 2002) in each bin.""" - n_gal = N_EFF * AREA_DEG2 * 3600 - return SIGMA_E**4 / (n_gal * N_EFF * np.pi * np.diff(edges**2)) - - -VARIANCE = np.concatenate([_variance(e) for e in (EDGES, EDGES, EDGES_INT, EDGES_INT)]) - - -def _fiducial_xi(): - """The default theory's ξ± on both grids, with the shape-noise covariance.""" - z = np.linspace(0.01, 3.0, 300) - s = sio.new_sacc({0: (z, np.exp(-(((z - 0.7) / 0.3) ** 2)))}) - for theta, grid in ((THETA, "reporting"), (THETA_INT, "integration")): - sio.add_xi(s, (0, 0), theta, 0 * theta, 0 * theta, grid=grid, theta_nom=theta) - for dp, value in zip(s.data, theory.shear(theory.fiducial(), s)): - dp.value = float(value) - s.add_covariance(VARIANCE) - return s - - -def _vector(s): - """(ξ+, ξ−) on the reporting grid, then on the integration grid.""" - views = [sio.xi_correlation(s, grid=g) for g in ("reporting", "integration")] - return np.concatenate([np.r_[v.xip, v.xim] for v in views]) - - -@pytest.fixture(scope="module") -def shifts(): - """Each input's ξ± and the blind's shift δ at each envelope corner.""" - s = _fiducial_xi() - clean = _vector(s) - noisy = clean + np.random.default_rng(3).normal(0.0, np.sqrt(VARIANCE)) - fiducial = theory.fiducial() - corners = [ - {**fiducial, "S8": fiducial["S8"] + a, "Omega_m": fiducial["Omega_m"] + b} - for a in (-bd.ENVELOPE["S8"], bd.ENVELOPE["S8"]) - for b in (-bd.ENVELOPE["Omega_m"], bd.ENVELOPE["Omega_m"]) - ] - deltas = [] - for corner in corners: - shifted = s.copy() - for dp, d in zip(shifted.data, theory.shear(corner, s) - s.mean): - dp.value += float(d) - deltas.append(_vector(shifted) - clean) - return {"noise-free": (clean, deltas), "noisy": (noisy, deltas)} - - -def _as_b(delta): - """The pure-B counterpart of an E-mode ξ± vector: (δξ+, −δξ−) per grid.""" - n, n_int = len(THETA), len(THETA_INT) - sign = np.r_[np.ones(n), -np.ones(n), np.ones(n_int), -np.ones(n_int)] - return sign * delta - - -@pytest.fixture(scope="module") -def pure_b(): - """Pure-E/B (ξ+_B, ξ−_B) of an integration-grid ξ±, and their σ. - - The operator :func:`b_modes.pure_eb_operator` builds, averaged into the - reporting bins with pair weights ∝ the bins' annulus areas; σ is the - shape-noise covariance pushed exactly through it. - """ - n, n_int = len(THETA), len(THETA_INT) - operator, _, _ = b_modes.pure_eb_operator(np.diff(EDGES_INT**2), EDGES_INT, EDGES) - b_rows = slice(2 * n, 4 * n) # xip_B, xim_B in _EB_KEYS order - operator_b = operator[b_rows] - rows = np.r_[2 * n : 2 * n + 2 * n_int] - sigma = np.sqrt(np.einsum("ij,j,ij->i", operator_b, VARIANCE[rows], operator_b)) - - def b(x): - return operator_b @ x[rows] - - return b, sigma - - -@pytest.fixture(scope="module") -def cosebis(): - """The fiducial cut's COSEBIs B_n of an integration-grid ξ±, and σ(B_n). - - The transform :func:`b_modes.cosebis_scan_from_xi` runs, built once. - """ - from cosmo_numba.B_modes.cosebis import COSEBIS - - n, n_int = len(THETA), len(THETA_INT) - cut = np.flatnonzero((THETA_INT >= COSEBIS_CUT[0]) & (THETA_INT <= COSEBIS_CUT[1])) - theta = THETA_INT[cut] - transform = COSEBIS( - theta_min=theta.min(), theta_max=theta.max(), N_max=NMODES, precision=120 - ) - rows = np.r_[2 * n + cut, 2 * n + n_int + cut] - covariance = transform.cosebis_covariance_from_xipm_covariance( - theta, np.diag(VARIANCE[rows]) - ) - - def b_n(x): - return transform.cosebis_from_xipm( - theta, x[2 * n + cut], x[2 * n + n_int + cut], parallel=True - )[1] - - return b_n, np.sqrt(np.diag(covariance)[NMODES:]) - - -@pytest.mark.parametrize("noise", ["noise-free", "noisy"]) -def test_cosebis_b_modes_move_only_by_the_transforms_response(shifts, cosebis, noise): - """COSEBIs are linear with fixed weights: ΔB_n = B_n(δ) to round-off, and - B_n(δ), the transform's B response to a pure E-mode δ, stays under the - ceiling; δ's pure-B counterpart moves B_n well past it.""" - b_n, sigma = cosebis - x, deltas = shifts[noise] - for delta in deltas: - before, after = b_n(x), b_n(x + delta) - roundoff = 1e-10 * np.max(np.abs(np.r_[before, after, b_n(delta)])) - np.testing.assert_allclose(after - before, b_n(delta), rtol=0, atol=roundoff) - assert np.max(np.abs(b_n(delta)) / sigma) < CEILING - assert np.max(np.abs(b_n(_as_b(delta))) / sigma) > 10 * CEILING - - -@pytest.mark.parametrize("noise", ["noise-free", "noisy"]) -def test_pure_eb_b_modes_move_only_by_the_transforms_response(shifts, pure_b, noise): - """Pure-E/B is one fixed linear operator on the integration-grid ξ±: - ΔB = B(δ) to round-off, and B(δ), the operator's B response to a pure - E-mode δ, stays under the ceiling; δ's pure-B counterpart moves B well - past it.""" - b, sigma = pure_b - x, deltas = shifts[noise] - for delta in deltas: - before, after = b(x), b(x + delta) - roundoff = 1e-10 * np.max(np.abs(np.r_[before, after, b(delta)])) - np.testing.assert_allclose(after - before, b(delta), rtol=0, atol=roundoff) - assert np.max(np.abs(b(delta)) / sigma) < CEILING - assert np.max(np.abs(b(_as_b(delta))) / sigma) > 10 * CEILING diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index b4b61beb..09393707 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -14,7 +14,6 @@ import numpy as np import pytest import yaml -from _synthetic import write_synthetic_catalogs from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation @@ -276,6 +275,142 @@ def test_v1_4_6_glass_mock_default_seed(self, base_config): # shear_psf_leakage, cosmo_numba), i.e. they run in the container. # ------------------------------------------------------------------ + @staticmethod + def _write_synthetic_catalogs( + tmp_path, + n_gal=2000, + n_star=800, + ra_range=(10.0, 14.0), + dec_range=(10.0, 14.0), + seed=1234, + coherent_shear=False, + with_psf=False, + ): + """Write small deterministic FITS catalogs + dndz, return a config dict. + + Builds a synthetic shear catalog (RA/Dec/e1/e2/w), a PSF star catalog + with the columns the leakage/rho-tau seams read, and a cs_util-readable + dndz file. Returns ``(params, version)`` ready to hand to + ``CosmologyValidation``. + + Parameters + ---------- + coherent_shear : bool + If True, inject a smooth position-dependent shear pattern on top of + shape noise so that xi+/- is smooth (needed for the pure-E/B + integral to be numerically well-posed). + with_psf : bool + If True, add a ``psf`` config block (rho/tau / pseudo-Cl read it via + ``get_params_rho_tau``). + """ + from astropy.table import Table + + rng = np.random.default_rng(seed) + version = "TestCatalog" + + cat_dir = tmp_path / "catalog" + cat_dir.mkdir() + nz_dir = tmp_path / "nz" + nz_dir.mkdir() + output_dir = tmp_path / "output" + output_dir.mkdir() + + ra = rng.uniform(*ra_range, n_gal) + dec = rng.uniform(*dec_range, n_gal) + if coherent_shear: + # Smooth E-mode-like pattern so xi+/- is smooth, plus shape noise. + e1 = 0.02 * np.cos(np.radians(ra) * 40) + rng.normal(0, 0.05, n_gal) + e2 = 0.02 * np.sin(np.radians(dec) * 40) + rng.normal(0, 0.05, n_gal) + else: + e1 = rng.normal(0, 0.25, n_gal) + e2 = rng.normal(0, 0.25, n_gal) + w = rng.uniform(0.5, 1.0, n_gal) + + shear_path = cat_dir / "shear.fits" + Table({"RA": ra, "Dec": dec, "e1": e1, "e2": e2, "w": w}).write( + shear_path, overwrite=True + ) + + star_path = cat_dir / "star.fits" + Table( + { + "RA": rng.uniform(*ra_range, n_star), + "Dec": rng.uniform(*dec_range, n_star), + "HSM_G1_PSF": rng.normal(0, 0.03, n_star), + "HSM_G2_PSF": rng.normal(0, 0.03, n_star), + "HSM_G1_STAR": rng.normal(0, 0.03, n_star), + "HSM_G2_STAR": rng.normal(0, 0.03, n_star), + "HSM_T_PSF": rng.uniform(0.4, 0.6, n_star), + "HSM_T_STAR": rng.uniform(0.4, 0.6, n_star), + "HSM_FLAG_PSF": np.zeros(n_star, dtype=int), + "HSM_FLAG_STAR": np.zeros(n_star, dtype=int), + } + ).write(star_path, overwrite=True) + + # dndz in the commented-header format cs_util.read_dndz expects: + # column "z" holds bin edges (n+1), "dn_dz" the densities. + z_edges = np.linspace(0.05, 3.0, 31) + dndz = np.exp(-(((z_edges - 0.7) / 0.3) ** 2)) + dndz_lines = ["# z dn_dz"] + [f"{zz} {nn}" for zz, nn in zip(z_edges, dndz)] + (nz_dir / "dndz_SP_A.txt").write_text("\n".join(dndz_lines) + "\n") + + shear_cfg = { + "path": "shear.fits", + "redshift_path": str(nz_dir / "dndz_SP_A.txt"), + "w_col": "w", + "e1_col": "e1", + "e2_col": "e2", + "R": 1.0, + "e1_col_corrected": "e1", + "e2_col_corrected": "e2", + } + star_cfg = { + "path": "star.fits", + "ra_col": "RA", + "dec_col": "Dec", + "e1_col": "HSM_G1_PSF", + "e2_col": "HSM_G2_PSF", + } + version_cfg = { + "blind": "none", + "subdir": str(cat_dir), + "pipeline": "SP", + "shear": shear_cfg, + "star": star_cfg, + } + if with_psf: + version_cfg["psf"] = { + "path": "star.fits", + "hdu": 1, + "ra_col": "RA", + "dec_col": "Dec", + "e1_PSF_col": "HSM_G1_PSF", + "e2_PSF_col": "HSM_G2_PSF", + "e1_star_col": "HSM_G1_STAR", + "e2_star_col": "HSM_G2_STAR", + "PSF_size": "HSM_T_PSF", + "star_size": "HSM_T_STAR", + "PSF_flag": "HSM_FLAG_PSF", + "star_flag": "HSM_FLAG_STAR", + } + + config_data = { + "nz": { + "subdir": str(nz_dir), + "dndz": {"path": "dndz_{pipeline}_A.txt"}, + }, + "paths": {"output": str(output_dir)}, + version: version_cfg, + } + config_path = tmp_path / "synthetic_config.yaml" + config_path.write_text(yaml.dump(config_data, sort_keys=False)) + + params = { + "catalog_config": str(config_path), + "output_dir": str(output_dir), + } + return params, version + def test_calculate_2pcf_runs_on_synthetic_catalog(self, tmp_path): """calculate_2pcf wires catalog+config into a ξ± part. @@ -287,7 +422,7 @@ def test_calculate_2pcf_runs_on_synthetic_catalog(self, tmp_path): from sp_validation.b_modes import log_bin_edges pytest.importorskip("treecorr") - params, version = write_synthetic_catalogs(tmp_path) + params, version = self._write_synthetic_catalogs(tmp_path) nbins = 8 cv = CosmologyValidation( @@ -320,7 +455,7 @@ def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): """ import treecorr - params, version = write_synthetic_catalogs( + params, version = self._write_synthetic_catalogs( tmp_path, n_gal=4000, coherent_shear=True ) cv = CosmologyValidation( @@ -348,7 +483,7 @@ def test_treecorr_runs_on_the_cpus_the_process_holds(self, tmp_path): """By default TreeCorr takes the process's CPU affinity, not the node's count.""" import treecorr - params, version = write_synthetic_catalogs(tmp_path) + params, version = self._write_synthetic_catalogs(tmp_path) CosmologyValidation(versions=[version], npatch=1, **params).calculate_2pcf( version ) @@ -365,7 +500,7 @@ def test_calculate_scale_dependent_leakage_runs_on_synthetic_catalog( tightened to allclose vs. a committed reference later). """ pytest.importorskip("treecorr") - params, version = write_synthetic_catalogs(tmp_path) + params, version = self._write_synthetic_catalogs(tmp_path) nbins = 8 cv = CosmologyValidation( @@ -407,7 +542,7 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path, pure_eb_xi) from sp_validation import b_modes # Coherent shear -> smooth xi+/-, so the pure-E/B integral is well-posed. - params, version = write_synthetic_catalogs( + params, version = self._write_synthetic_catalogs( tmp_path, n_gal=4000, coherent_shear=True ) @@ -464,47 +599,6 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path, pure_eb_xi) ) -# --------------------------------------------------------------------------- # -# I14: a blinded catalogue's ξ± leaves CosmologyValidation only concealed -# --------------------------------------------------------------------------- # -@pytest.fixture -def blinded_and_twin(tmp_path, toy_theory): - """TOY, blinded under `toy`, and TOY_OPEN: the same galaxies, public.""" - import yaml - - from sp_validation import blinding - - params, _ = write_synthetic_catalogs( - tmp_path, - n_gal=4000, - coherent_shear=True, - catalogues={"TOY": "toy", "TOY_OPEN": "none"}, - ) - blinding.init("toy", yaml.safe_load(open(params["catalog_config"]))) - grid = dict(npatch=1, theta_min=5.0, theta_max=60.0, nbins=6, **params) - return ( - CosmologyValidation(versions=["TOY"], **grid), - CosmologyValidation(versions=["TOY_OPEN"], **grid), - ) - - -def test_a_blinded_catalogues_xi_leaves_concealed(blinded_and_twin): - """calculate_2pcf returns, and caches, a blinded catalogue's ξ± shifted - from its public twin's by exactly the blind's shift, stamped with its name.""" - from sp_validation import blinding - - cv, cv_open = blinded_and_twin - blinded, twin = cv.calculate_2pcf("TOY"), cv_open.calculate_2pcf("TOY_OPEN") - shift = blinding.conceal(twin, cv.blind("TOY")).mean - twin.mean - - assert np.all(shift != 0) - np.testing.assert_allclose( - blinded.mean - twin.mean, shift, rtol=1e-8, atol=1e-12 * np.abs(shift).max() - ) - assert sacc_io.stamp(blinded) == "toy" and sacc_io.stamp(twin) == "none" - assert cv.xi_parts["TOY", "reporting"] is blinded - - def test_map2_transform_is_treecorrs_calculate_map_sq(): """⟨M_ap²⟩, ⟨M_ײ⟩ from the ξ± transform equal TreeCorr's own sum.""" import treecorr diff --git a/src/sp_validation/tests/test_custody.py b/src/sp_validation/tests/test_custody.py deleted file mode 100644 index 2acb1c69..00000000 --- a/src/sp_validation/tests/test_custody.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Every catalogue entry declares its blind.""" - -from pathlib import Path - -import pytest -import yaml - -from sp_validation import custody as cu - -REPO = Path(__file__).resolve().parents[3] - - -def _catalogues(tmp_path, **blinds): - """A parsed catalogue config: one entry per ``name=blind``, each reading - its own file; ``None`` declares nothing.""" - entries = {"nz": {"subdir": "/nz"}, "paths": {"output": "./output"}} - for name, blind in blinds.items(): - entries[name] = {"subdir": str(tmp_path), "shear": {"path": f"{name}.fits"}} - if blind is not None: - entries[name]["blind"] = blind - return entries - - -def test_every_entry_declares_its_blind(tmp_path): - cats = _catalogues(tmp_path, OPEN="none", Y3="y3", NEW=None) - assert cu.declared(cats, "OPEN") == "none" - assert cu.declared(cats, "Y3") == "y3" - with pytest.raises(ValueError, match="declares no `blind:`"): - cu.declared(cats, "NEW") - # _leak_corr and _seed versions take their entry's blind - for version in ("Y3_leak_corr", "Y3_seed00042_leak_corr"): - assert cu.declared(cats, version) == "y3" - - -def test_entries_reading_one_file_declare_one_blind(tmp_path): - cats = _catalogues(tmp_path, TOY="y3", TWIN="none") - cats["TWIN"]["shear"]["path"] = "TOY.fits" - with pytest.raises(ValueError, match="different blinds"): - cu.declared(cats, "TWIN") - cats["TWIN"]["blind"] = "y3" - assert cu.declared(cats, "TWIN") == "y3" - - -def test_every_catalogue_in_the_repository_is_public(): - cats = yaml.safe_load((REPO / "cosmo_val" / "cat_config.yaml").read_text()) - blinds = { - name: cu.declared(cats, name) for name in cats if name not in cu.NOT_CATALOGUES - } - assert blinds and set(blinds.values()) == {"none"}, blinds diff --git a/src/sp_validation/tests/test_masks.py b/src/sp_validation/tests/test_masks.py index beeefdda..76e97748 100644 --- a/src/sp_validation/tests/test_masks.py +++ b/src/sp_validation/tests/test_masks.py @@ -34,10 +34,12 @@ ], ) def test_apply_condition_kinds(kind, value, expected): + npt.assert_array_equal(apply_condition(_ARRAY, kind, value), expected) def test_smaller_equal_alias_matches_less_equal(): + npt.assert_array_equal( apply_condition(_ARRAY, "smaller_equal", 3), apply_condition(_ARRAY, "less_equal", 3), @@ -45,11 +47,13 @@ def test_smaller_equal_alias_matches_less_equal(): def test_unknown_kind_raises(): + with pytest.raises(ValueError): apply_condition(_ARRAY, "not_a_real_kind", 3) def test_mask_apply_matches_apply_condition(): + dat = np.array([(1,), (2,), (3,), (4,), (5,)], dtype=[("col", "i8")]) my_mask = Mask("col", "test_mask", kind="greater_equal", value=3, dat=dat) @@ -60,6 +64,7 @@ def test_mask_apply_matches_apply_condition(): def test_not_equal_2bands_is_two_column_or(): + # Keep an object if EITHER band column differs from the sentinel dat = np.array( [(-99, -99), (-99, 20.0), (21.0, -99), (21.0, 20.0)], @@ -79,11 +84,13 @@ def test_not_equal_2bands_is_two_column_or(): def test_not_equal_2bands_requires_col_name2(): + with pytest.raises(ValueError, match="col_name2"): Mask("mag_z", "zband", kind="not_equal_2bands", value=-99) def test_not_equal_2bands_descr_names_both_columns(): + my_mask = Mask( "mag_z", "zband", kind="not_equal_2bands", value=-99, col_name2="mag_z2" ) diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 7782fe6a..d2279969 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -63,8 +63,6 @@ def _cosebi_block(s, tr): # --------------------------------------------------------------------------- # # 1. Per-writer round-trip (arrays / tags / windows / NZ bitwise) # --------------------------------------------------------------------------- # - - def _roundtrip(s, tmp_path, name="rt"): path = tmp_path / f"{name}.sacc" sio.save(s, str(path), blind=NONE) diff --git a/src/sp_validation/tests/test_survey.py b/src/sp_validation/tests/test_survey.py index 3d3eac1d..f42f884c 100644 --- a/src/sp_validation/tests/test_survey.py +++ b/src/sp_validation/tests/test_survey.py @@ -17,6 +17,7 @@ class SurveyTestCase(TestCase): def setUp(self): + self._dd = np.array( [(270.283, 1), (270.283, 0), (188.308, 0)], dtype=[("TILE_ID", "f8"), ("FLAGS", "i2")], @@ -32,6 +33,7 @@ def setUp(self): self._patch = ["P5"] def tearDown(self): + self.number_tile = None self.number_exp = None self.number_int = None diff --git a/src/sp_validation/theory.py b/src/sp_validation/theory.py deleted file mode 100644 index 0b610eb1..00000000 --- a/src/sp_validation/theory.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Theory data vectors: a prediction for every row of a SACC. - -A theory is a function ``theory(params, s) -> np.ndarray``: - -- ``params`` is a plain dict of cosmological parameters by name, with exactly - the keys of :func:`fiducial`: ``S8``, ``Omega_m``, ``Omega_b``, ``h``, - ``n_s``, ``m_nu``, ``w0``, ``wa``, ``logT_AGN`` and ``A_IA``. Blinding calls - the theory twice, at a blind's fiducial point and at the hidden point drawn - from it; the two differ only in ``S8`` and ``Omega_m``. -- ``s`` is the ``sacc.Sacc`` being blinded. Its rows, ``s.data``, each carry a - data type, a tracer pair and a θ or ℓ tag (Cℓ rows also a bandpower window, - ``s.get_bandpower_windows(rows)``); its tracers, ``s.tracers``, carry the - n(z). -- It returns an array the length of ``s.mean``, in the same order: the - prediction for each row, zero where the cosmology has no effect. - -For a SACC of Cℓ_EE/BB/EB bandpowers, a theory could read:: - - def my_theory(params, s): - out = np.zeros(len(s.mean)) - ee = s.indices(sacc_io.CL_EE) - window = s.get_bandpower_windows(ee) - cl = my_cl_ee(params, window.values) # Cℓ_EE on the window's ℓ - out[ee] = window.weight.T @ cl - return out # Cℓ_BB and Cℓ_EB stay zero - -:func:`shear` is the default, :func:`none` the theory of statistics with no -cosmological signal. -""" - -import functools - -import numpy as np - -from .sacc_io import CL_BB, CL_EB, CL_EE, XI_MINUS, XI_PLUS - -# Multipoles the ξ± Hankel transform integrates over: every ℓ below 50, -# then 200 log-spaced up to 6·10⁴. -ELL = np.unique(np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)])) -_COSMOLOGY = ("S8", "Omega_m", "Omega_b", "h", "n_s", "m_nu", "w0", "wa", "logT_AGN") - - -def fiducial(): - """cs_util's Planck 2018 point, with feedback and no IA.""" - from cs_util.cosmo import PLANCK18 as p - - return { - "S8": float(p["sigma_8"] * np.sqrt(p["Omega_m"] / 0.3)), - **{ - k: float(p[k]) - for k in ("Omega_m", "Omega_b", "h", "n_s", "m_nu", "w0", "wa") - }, - "logT_AGN": 7.5, - "A_IA": 0.0, - } - - -def none(params, s): - """Zeros: the theory of a statistic with no cosmological signal (ρ/τ).""" - return np.zeros(len(s.mean)) - - -def shear(params, s): - """Cosmic-shear ξ± and Cℓ_EE from pyccl; zeros for Cℓ_BB and Cℓ_EB. - - ξ± is evaluated at each row's stored θ (TreeCorr's ``meanr``) and Cℓ_EE - through each row's bandpower window. Any other data type raises: pass your - own theory. pyccl and cs_util are imported only when this runs. - """ - out = np.zeros(len(s.mean)) - groups = {} - for i, dp in enumerate(s.data): - if dp.data_type in (XI_PLUS, XI_MINUS): - key = (_xi, dp.tracers) - elif dp.data_type == CL_EE: - key = (_cl, dp.tracers, id(dp.tags.get("window"))) - elif dp.data_type in (CL_BB, CL_EB): - continue - else: - raise ValueError( - f"theory.shear has no prediction for {dp.data_type}; pass your " - "own theory" - ) - groups.setdefault(key, []).append(i) - # One call per tracer pair (and window), so ξ+ and ξ− share their Cℓ. - for (function, *_), rows in groups.items(): - out[rows] = function(params, s, np.asarray(rows)) - return out - - -def cosmology(params): - """The ``pyccl.Cosmology`` at ``params``, built once per point. - - Built through ``cs_util.cosmo.get_cosmo`` on the CAMB HMCode2020-feedback - route the CosmoSIS inference runs, with σ8 = S8/√(Ωm/0.3); ``get_cosmo`` - subtracts CAMB's Ω_ν from ``Omega_m`` for Ω_c. - """ - return _cosmology(tuple(float(params[k]) for k in _COSMOLOGY)) - - -@functools.lru_cache(maxsize=4) -def _cosmology(point): - from cs_util.cosmo import get_cosmo - - p = dict(zip(_COSMOLOGY, point)) - cosmo = get_cosmo( - Omega_m=p["Omega_m"], - Omega_b=p["Omega_b"], - h=p["h"], - sig8=p["S8"] / np.sqrt(p["Omega_m"] / 0.3), - ns=p["n_s"], - w0=p["w0"], - wa=p["wa"], - mnu=p["m_nu"], - # get_cosmo's default "halofit" ignores extra_params. - matter_power_spectrum="camb", - extra_params={ - "camb": { - "halofit_version": "mead2020_feedback", - "HMCode_logT_AGN": p["logT_AGN"], - } - }, - ) - # CCL's Hankel transform extrapolates C_ℓ to ELL_MAX_CORR; its default - # (6·10⁴) rings in ξ− below a few arcmin. - cosmo.cosmo.spline_params.ELL_MAX_CORR = 10_000_000 - cosmo.cosmo.spline_params.N_ELL_CORR = 5_000 - return cosmo - - -def _tag(s, rows, name): - return np.array([s.data[i].tags[name] for i in rows]) - - -def _lensing(cosmo, params, s, name): - import pyccl as ccl - - tracer = s.tracers[name] - if getattr(tracer, "nz", None) is None: - raise ValueError(f"tracer {name} has no n(z)") - z, n = np.asarray(tracer.z, float), np.asarray(tracer.nz, float) - ia = None if params["A_IA"] == 0 else (z, np.full_like(z, params["A_IA"])) - return ccl.WeakLensingTracer(cosmo, dndz=(z, n), ia_bias=ia) - - -def _shear_cl(params, s, rows, ell): - import pyccl as ccl - - cosmo = cosmology(params) - a, b = (_lensing(cosmo, params, s, t) for t in s.data[rows[0]].tracers) - return ccl.angular_cl(cosmo, a, b, ell) - - -def _xi(params, s, rows): - """ξ± of one shear pair: Limber Cℓ on :data:`ELL`, then CCL's Hankel transform.""" - import pyccl as ccl - - cosmo, cl = cosmology(params), _shear_cl(params, s, rows, ELL) - theta = _tag(s, rows, "theta") / 60.0 - xip, xim = ( - ccl.correlation(cosmo, ell=ELL, C_ell=cl, theta=theta, type=t) - for t in ("GG+", "GG-") - ) - plus = np.array([s.data[i].data_type == XI_PLUS for i in rows]) - return np.where(plus, xip, xim) - - -def _cl(params, s, rows): - """Cℓ_EE of one shear pair through its bandpower window.""" - window = s.get_bandpower_windows(rows) - cl = _shear_cl(params, s, rows, np.asarray(window.values, float)) - return np.asarray(window.weight).T @ cl diff --git a/workflow/README.md b/workflow/README.md index 351fe1f8..a3cf07ad 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -28,9 +28,8 @@ each namespaces cleanly under `results//`. ## Blinding Every entry of `cosmo_val/cat_config.yaml` declares `blind: none` or -`blind: `; an entry without one stops the launch, which prints one -`[blind]` line per catalogue. `_leak_corr` and `_seed` versions take their -entry's blind, and entries reading one shear file declare one blind. +`blind: `; an entry without one stops the launch. `_leak_corr` versions +take their entry's blind, and entries reading one shear file declare one blind. A blind is a secret record, `/.blind.json` (seed, envelope, fiducial), drawn once with @@ -38,10 +37,10 @@ fiducial), drawn once with committed. On a blinded catalogue, ξ± and Cℓ_EE are shifted by t(hidden) − t(fiducial) before they are first written: Smokescreen draws the hidden point (S8 and Ωm) from the seed, and the theory `t` is -`sp_validation.theory.shear`. Cℓ_BB and Cℓ_EB get zero shift; ρ/τ are saved -with `theory.none`. Another statistic passes its own theory, +`sp_validation.blinding.shear`. Cℓ_BB and Cℓ_EB get zero shift; ρ/τ are saved +with `blinding.no_signal`. Another statistic passes its own theory, `theory(params, s) -> array the length of s.mean` (see -`src/sp_validation/theory.py`). +`src/sp_validation/blinding.py`). A measurement calculates, then saves: one function computes the signal, saves it with `sacc_io.save(s, path, blind=...)` (or `sacc_io.seal`), and returns the diff --git a/workflow/common.py b/workflow/common.py index 1a76eb1f..fb1dda03 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -35,13 +35,10 @@ def _plain(path): def _load_checkout_module(name): - """Load the stdlib-only module ``sp_validation/.py`` from this checkout. + """Load ``sp_validation/.py``, a stdlib-only module, from this checkout. - Loaded by file path rather than imported: snakemake runs on the host, where - sp_validation is usually not installed, and importing the package would drag - in ``__init__`` -> ``version`` -> a metadata warning on every launch. Taking - it from *this checkout's* src/ also means the workflow and the package can - never disagree about what the module says. + By path, since the host Snakemake has no sp_validation installed; from + *this checkout's* src/, so the workflow and the package never disagree. """ alias = f"_spv_{name}" module = importlib.util.module_from_spec( @@ -60,8 +57,8 @@ def _load_checkout_module(name): image_revision = _container.image_revision resolve_image = _container.resolve_image -# Each catalogue's declared blind and shear file, resolved as container jobs do. -_custody = _load_checkout_module("custody") +# Each catalogue's declared blind (``blinding.blind_of``). +_blinding = _load_checkout_module("blinding") # Every job inherits this launch's environment (the slurm executor submits with # --export=ALL), and the Snakemake each job step starts keeps its source cache @@ -206,28 +203,18 @@ def configure(workflow_config): DEFAULT_MASK_SUFFIX = ( "_masked" if workflow_config["covariance"].get("default_masked", False) else "" ) - announce_blinds(workflow_config) with open(COSMOLOGY_PARAMS) as f: PLANCK18 = json.load(f) - - -def announce_blinds(workflow_config): - """Print each run catalogue's blind, or stop the launch if one declares none.""" - from snakemake.exceptions import WorkflowError - - versions = workflow_config.get("versions", []) - fiducial = [FIDUCIAL.get(k) for k in ("version", "mock_version")] - try: - lines = _custody.summary(CATALOG_CONFIG, [*versions, *filter(None, fiducial)]) - except ValueError as err: - raise WorkflowError(str(err)) from None - print("\n".join(lines), file=sys.stderr) + # A run catalogue that declares no blind stops the launch before any job. + fiducial = (FIDUCIAL.get(k) for k in ("version", "mock_version")) + for version in [*workflow_config.get("versions", []), *filter(None, fiducial)]: + blind_of(version) def blind_of(version): """``version``'s declared blind: a producer's ``params.blind``, so flipping it reruns exactly the jobs it touches.""" - return _custody.declared(CATALOG_CONFIG, version) + return _blinding.blind_of(CATALOG_CONFIG, catalogue_name(version)) def fiducial_binning_suffix(fiducial=None): @@ -295,10 +282,17 @@ def base_version(version): return re.sub(r"_ecut\d+", "", re.sub(r"_leak_corr$", "", version)) +def catalogue_name(version): + """The name of the catalogue-config entry describing ``version`` (its own, + or the one its ``_leak_corr`` variant derives from).""" + if version in CATALOG_CONFIG: + return version + return re.sub(r"_leak_corr$", "", version) + + def catalogue_entry(version): - """The catalogue-config entry describing ``version`` (its own, or the one - its ``_leak_corr``/``_seed`` variant chain reaches).""" - return CATALOG_CONFIG[_custody.entry_name(CATALOG_CONFIG, version)] + """The catalogue-config entry describing ``version``.""" + return CATALOG_CONFIG[catalogue_name(version)] def redshift_path(version): @@ -383,9 +377,14 @@ def pseudo_cl_tag(config): return f"{fiducial['binning']}_nbins={fiducial['nbins']}" -def shear_catalog(version): - """The shear catalogue file of ``version``, from its catalogue entry.""" - return _custody.shear_file(CATALOG_CONFIG, version) +def get_shear_catalog(wildcards): + """Resolve shear catalog path from config for a given version.""" + cat_config = CATALOG_CONFIG[wildcards.version.replace("_leak_corr", "")] + shear_path = cat_config["shear"]["path"] + if shear_path.startswith("/"): + return shear_path + subdir = cat_config.get("subdir", "") + return str(Path(subdir) / shear_path) # --------------------------------------------------------------------------- diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index d7ba5ff9..300eb3c4 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -20,7 +20,7 @@ rule xi: binds the wildcards and the grid label resolves from them. """ input: - catalog=lambda w: shear_catalog(w.version), + catalog=get_shear_catalog, output: sacc=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 24 diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py index ad4d723f..64ffee14 100644 --- a/workflow/tests/conftest.py +++ b/workflow/tests/conftest.py @@ -10,10 +10,10 @@ Snakemake is in. The ``candide`` tests need candide itself; CI deselects them. The ``toy`` fixture is a disposable checkout: copies of ``workflow/`` and -``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, a blinded, a -public and an undeclared catalogue, stand-ins for the processed CosmoCov -covariances (their inputs live on candide), and both output roots in tmp. The -host never opens a blind, so there is no blind record. +``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, a blinded and an +undeclared catalogue, stand-ins for the processed CosmoCov covariances (their +inputs live on candide), and both output roots in tmp. The host never opens a +blind, so there is no blind record. """ import dataclasses @@ -32,7 +32,6 @@ # The toy catalogue and its leakage-corrected variant: blinded under `toy`. VERSIONS = ("SP_v0.1", "SP_v0.1_leak_corr") -PUBLIC = "SP_v0.2" # Declares no blind. UNDECLARED = "SP_v0.5" @@ -128,8 +127,7 @@ def snakemake(self, *args, config=(), cwd=None, env=None, timeout=300): def _cat_config(data): - """A blinded, a public and an undeclared catalogue, each reading its own - touched file.""" + """A blinded and an undeclared catalogue, each reading its own touched file.""" def entry(name, **declaration): catalogue = data / f"{name}.fits" @@ -154,7 +152,6 @@ def entry(name, **declaration): return { VERSIONS[0]: entry(VERSIONS[0], blind="toy"), - PUBLIC: entry(PUBLIC, blind="none"), UNDECLARED: entry(UNDECLARED), "paths": {"output": "./output"}, } diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index 88f85364..0183ca53 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -81,13 +81,6 @@ def test_one_integration_grid(toy, forced, grids): assert by_rule["cv_pure_eb"] == {part, covariance}, by_rule["cv_pure_eb"] -def test_a_launch_prints_each_catalogues_blind(forced): - """A catalogue and its variant share one line.""" - output, _ = forced - line = f"[blind] {VERSIONS[0]} (+ {VERSIONS[1]}): toy" - assert {x for x in output.splitlines() if x.startswith("[blind]")} == {line} - - def test_a_blind_flip_reruns_the_catalogues_parts(toy, grids, tmp_path): """A part's params carry its catalogue's blind, so declaring the catalogue public reruns the part and nothing else does.""" diff --git a/workflow/tests/test_toy_run.py b/workflow/tests/test_toy_run.py deleted file mode 100644 index 3b98a9ff..00000000 --- a/workflow/tests/test_toy_run.py +++ /dev/null @@ -1,144 +0,0 @@ -"""Rule xi, run for real through apptainer, under a throwaway blind. - -The launch is the README's with the machine-independent default profile: the -host Snakemake, the image `spv-container` manages, jobs on this node. A toy -checkout under your home directory (which the profile binds) carries copies of -workflow/, papers/cosmo_val/ and src/, and one synthetic catalogue declared -under a blind drawn in the image with the default theory. -""" - -import os -import shutil -import subprocess -import sys -import tempfile -from pathlib import Path - -import pytest -import yaml -from conftest import REPO, container, on_candide - -VERSIONS = ("SP_v0.1", "SP_v0.1_leak_corr") -REPORTING = {"theta_min": 5.0, "theta_max": 60.0, "nbins": 6, "npatch": 4} -BINNING = "minsep=5.0_maxsep=60.0_nbins=6_npatch=4" - - -def _in_image(image, root, *command): - """``command``'s stdout, run in ``image`` on the toy checkout's src/.""" - return subprocess.run( - [ - "apptainer", - "exec", - "--cleanenv", - "--env", - f"PYTHONPATH={root / 'src'}", - str(image), - *command, - ], - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout - - -def _stamps(image, root, parts): - """The blind stamp of each SACC part, read in the image.""" - script = ( - "import sys\n" - "from sp_validation import sacc_io\n" - "print(*(sacc_io.stamp(sacc_io.load(p)) for p in sys.argv[1:]))" - ) - return _in_image(image, root, "python", "-c", script, *map(str, parts)).split() - - -def _toy_checkout(root, image): - """A checkout with one synthetic catalogue, SP_v0.1, under the blind `toy`.""" - skip = shutil.ignore_patterns(".snakemake", "__pycache__", "tests") - shutil.copytree(REPO / "workflow", root / "workflow", ignore=skip) - shutil.copytree( - REPO / "papers" / "cosmo_val", root / "papers" / "cosmo_val", ignore=skip - ) - shutil.copytree( - REPO / "src", root / "src", ignore=shutil.ignore_patterns("__pycache__") - ) - (root / "cosmo_val").mkdir() - _in_image( - image, - root, - "python", - "-c", - "import sys\n" - f"sys.path.insert(0, {str(root / 'src/sp_validation/tests')!r})\n" - "from pathlib import Path\n" - "from _synthetic import write_synthetic_catalogs\n" - f"write_synthetic_catalogs(Path({str(root / 'cosmo_val')!r})," - f" catalogues={{{VERSIONS[0]!r}: 'toy'}})", - ) - _in_image( - image, - root, - "python", - "-c", - "import sys, yaml\n" - "from sp_validation import blinding\n" - "blinding.init('toy', yaml.safe_load(open(sys.argv[1])))", - str(root / "cosmo_val" / "cat_config.yaml"), - ) - - config_path = root / "papers" / "cosmo_val" / "config" / "config.yaml" - config = yaml.safe_load(config_path.read_text()) - config["versions"] = list(VERSIONS) - config["fiducial"]["version"] = VERSIONS[1] - config["fiducial"]["mock_version"] = VERSIONS[0] - config["cosmo_val"].update(REPORTING) - config_path.write_text(yaml.safe_dump(config, sort_keys=False)) - - -@pytest.mark.candide -@on_candide -def test_xi_parts_are_stamped_with_the_blind(): - image, kind = container.resolve_image() - assert kind != "tag", "no local image; run `spv-container pull`" - root = Path(tempfile.mkdtemp(prefix="toy_run_", dir=Path.home())) - _toy_checkout(root, image) - out = root / "out" - env = { - k: v - for k, v in os.environ.items() - if k not in ("SNAKEMAKE_PROFILE", "APPTAINERENV_PYTHONPATH") - } - env.update( - COSMO_VAL=str(out), - COSMO_INFERENCE=str(root / "inference"), - XDG_CACHE_HOME=str(root / "cache"), - PYTHONNOUSERSITE="1", - PYTHONUNBUFFERED="1", - ) - result = subprocess.run( - [ - sys.executable, - "-m", - "snakemake", - "--profile", - str(root / "workflow" / "profiles" / "default"), - "--cores", - "4", - "--config", - f"container={image}", - "--", - str(out / "snakemake_sentinels" / "plot_2pcf.done"), - ], - cwd=root / "papers" / "cosmo_val", - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=1800, - check=False, - ) - assert result.returncode == 0, result.stdout - - parts = [out / f"{v}_xi_{BINNING}.sacc" for v in VERSIONS] - assert _stamps(image, root, parts) == ["toy"] * len(parts) - - shutil.rmtree(root) # kept on failure, for post-mortem From 06176c42a80e9fe5d4d87c68ae6c6545273ebddf Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 2 Oct 2026 02:18:22 +0200 Subject: [PATCH 157/160] Blinds are checked against the catalogue config as written; seeded versions resolve on the host CosmologyValidation resolved the selected entry's paths before the one-file-one-blind check, so a twin entry under another blind passed. The workflow's catalogue_name strips _seed as well as _leak_corr. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0186WiB28hK7DK2pYaTBLL6j --- src/sp_validation/cosmo_val/core.py | 15 ++++++++++++++- src/sp_validation/tests/test_cosmo_val.py | 18 ++++++++++++++++++ workflow/common.py | 4 ++-- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index f66c19b6..f95ed3a0 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -1,6 +1,7 @@ # %% import copy import os +import re from pathlib import Path import colorama @@ -324,6 +325,9 @@ def __init__( self.catalog_config_path = Path(catalog_config) with self.catalog_config_path.open("r") as file: self.cc = cc = yaml.load(file, Loader=yaml.FullLoader) + # The config as written, before virtual versions and resolved paths: + # what each catalogue's blind is checked against. + self._config_as_read = copy.deepcopy(cc) def resolve_paths_for_version(ver): """Resolve relative paths for a version using its subdir.""" @@ -514,7 +518,16 @@ def results_objectwise(self): def blind(self, version): """The blind every SACC this object writes for ``version`` is saved under, as the catalogue config declares it.""" - return blinding.open_blind(blinding.blind_of(self.cc, version), self.cc) + entry = version + while entry not in self._config_as_read: + base = re.sub(r"_leak_corr$", "", entry) + if base == entry: + base = self._split_seed_variant(entry)[0] or entry + if base == entry: + raise ValueError(f"no catalogue {version} in the catalogue config") + entry = base + catalogues = self._config_as_read + return blinding.open_blind(blinding.blind_of(catalogues, entry), catalogues) def basename(self, version, treecorr_config=None, npatch=None): cfg = treecorr_config or self.treecorr_config diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 09393707..51dff8fb 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -195,6 +195,24 @@ def test_seed_variant_updates_shear_path(self, tmp_path): assert seed_version in cv.cc assert cv.cc[seed_version]["shear"]["path"].endswith("shear_seed_007.fits") + def test_blind_is_checked_against_the_config_as_written(self, tmp_path): + """A twin entry reading the same shear file under another blind is + refused, though the selected entry's paths were resolved; a seeded + leak-corrected variant takes its base entry's blind.""" + params, base_version = self._make_seed_config( + tmp_path, shear_filename="shear_seed_1234.fits" + ) + config = yaml.safe_load(open(params["catalog_config"])) + cv = CosmologyValidation(versions=[f"{base_version}_seed7_leak_corr"], **params) + assert cv.blind(f"{base_version}_seed7_leak_corr").name == "none" + + config["Twin"] = {**config[base_version], "blind": "y3"} + with open(params["catalog_config"], "w") as f: + yaml.dump(config, f, sort_keys=False) + cv = CosmologyValidation(versions=[base_version], **params) + with pytest.raises(ValueError, match="different blinds"): + cv.blind(base_version) + def test_seed_leak_corr_materializes_seed_first(self, tmp_path): """_seed_leak_corr should clone the seed variant before leak fixes.""" params, base_version = self._make_seed_config( diff --git a/workflow/common.py b/workflow/common.py index fb1dda03..bf327fac 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -284,10 +284,10 @@ def base_version(version): def catalogue_name(version): """The name of the catalogue-config entry describing ``version`` (its own, - or the one its ``_leak_corr`` variant derives from).""" + or the one its ``_leak_corr`` / ``_seed`` variant derives from).""" if version in CATALOG_CONFIG: return version - return re.sub(r"_leak_corr$", "", version) + return re.sub(r"_seed\d+$", "", re.sub(r"_leak_corr$", "", version)) def catalogue_entry(version): From 914a28f8e6192849b80fe47a5a33fca2bac8751a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 5 Oct 2026 21:44:20 +0200 Subject: [PATCH 158/160] Declare blinds where develop added catalogues, and read parts as stamped develop brought catalogue entries and tests written before every entry had to declare its blind: the GLASS validation mock and two regression-test configs declare `blind: none`. Its pseudo-Cl tests read the part without the removed allow_unblinded flag and pass the version to _load_pseudo_cl, which reuses a part only under the blind declared now. The column-schema scan takes the blinding theory's S8 and A_IA parameter keys for what they are, not catalogue columns. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015izKkaSS9NyrSQBd5S6tqR --- cosmo_val/cat_config.yaml | 1 + .../tests/regression/test_tomo_drops_min_top_pin.py | 6 +++++- .../regression/test_tomo_plot_cosebis_ignores_tomography.py | 1 + src/sp_validation/tests/test_column_schema.py | 2 ++ src/sp_validation/tests/test_pseudo_cl.py | 4 ++-- 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index 09ce339c..41eb3281 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -1246,6 +1246,7 @@ SP_v1.6.6: patch_number: 150 GLASS_mock_validation: + blind: none subdir: /n09data/guerrini/glass_mock_test/results/ pipeline: SP colour: violet diff --git a/src/sp_validation/tests/regression/test_tomo_drops_min_top_pin.py b/src/sp_validation/tests/regression/test_tomo_drops_min_top_pin.py index 09cea030..dc356c16 100644 --- a/src/sp_validation/tests/regression/test_tomo_drops_min_top_pin.py +++ b/src/sp_validation/tests/regression/test_tomo_drops_min_top_pin.py @@ -24,7 +24,11 @@ def _cosmology_validation(tmp_path): cfg = { "paths": {"output": str(tmp_path / "out")}, "nz": {"subdir": str(tmp_path), "dndz": {"path": "nz.txt"}}, - "synthetic": {"subdir": str(tmp_path), "shear": {"path": "cat.fits"}}, + "synthetic": { + "blind": "none", + "subdir": str(tmp_path), + "shear": {"path": "cat.fits"}, + }, } cfg_path = tmp_path / "cat_config.yaml" cfg_path.write_text(yaml.safe_dump(cfg)) diff --git a/src/sp_validation/tests/regression/test_tomo_plot_cosebis_ignores_tomography.py b/src/sp_validation/tests/regression/test_tomo_plot_cosebis_ignores_tomography.py index 63de87b7..645a0bb6 100644 --- a/src/sp_validation/tests/regression/test_tomo_plot_cosebis_ignores_tomography.py +++ b/src/sp_validation/tests/regression/test_tomo_plot_cosebis_ignores_tomography.py @@ -47,6 +47,7 @@ def _make_cv(tmp_path): out.mkdir() cfg = { VER: { + "blind": "none", "subdir": str(tmp_path), "shear": { "path": str(cat), diff --git a/src/sp_validation/tests/test_column_schema.py b/src/sp_validation/tests/test_column_schema.py index d4ad4cfd..da073b80 100644 --- a/src/sp_validation/tests/test_column_schema.py +++ b/src/sp_validation/tests/test_column_schema.py @@ -64,6 +64,8 @@ # NaMaster field/workspace dicts keyed by bin; cosmology parameter dicts "W{}", "H0", + "A_IA", + "S8", # GLASS mock catalogues, which are not ShapePipe products "TOM_BIN_ID", } diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 94109a55..5e4724f0 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -698,7 +698,7 @@ def capture_workspace( s = sacc_io.load(out_path) ell, ee, bb, eb, window = sacc_io.get_pseudo_cl(s, SACC_BIN) assert window is not None # the shared BandpowerWindow rides the part - readback = cv._load_pseudo_cl(out_path, ("all", "all")) + readback = cv._load_pseudo_cl(ver, out_path, ("all", "all")) npt.assert_array_equal(readback["BE"], eb) assert not np.shares_memory(readback["BE"], readback["EB"]) assert saved["nside"] is None @@ -797,7 +797,7 @@ def capture_workspace( out_path = cv._output_path(f"pseudo_cl_map_{ver}.sacc") cv.calculate_pseudo_cl_map(ver, NSIDE, out_path, "all", "all") - s = sacc_io.load(out_path, allow_unblinded=True) + s = sacc_io.load(out_path) _ell, _ee, _bb, _eb, window = sacc_io.get_pseudo_cl(s, SACC_BIN) assert saved["nside"] == NSIDE window_ells, unwindowed_weights = bandpower_window_from_workspace( From 8718fec7c223f6d0cc4a0114768a1ec1cb093c09 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 5 Oct 2026 21:44:47 +0200 Subject: [PATCH 159/160] Name calculate_2pcf_version where the docs point at the sealing pattern Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015izKkaSS9NyrSQBd5S6tqR --- src/sp_validation/cosmo_val/core.py | 2 +- workflow/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index a4ba8ab8..8159a1a0 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -471,7 +471,7 @@ def ensure_version_exists(ver): # B-mode results storage for summarize_bmodes() self._pure_eb_results = {} self._cosebis_results = {} - # The sealed ξ± parts calculate_2pcf returned, by (version, grid). + # The sealed ξ± parts calculate_2pcf_version made, by (version, grid). self.xi_parts = {} def _output_path(self, *parts): diff --git a/workflow/README.md b/workflow/README.md index 847fe5c6..8af65cab 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -45,7 +45,7 @@ with `blinding.no_signal`. Another statistic passes its own theory, A measurement calculates, then saves: one function computes the signal, saves it with `sacc_io.save(s, path, blind=...)` (or `sacc_io.seal`), and returns the concealed part, so raw signal never leaves it -(`CosmologyValidation.calculate_2pcf` is the pattern). Statistics computed from +(`CosmologyValidation.calculate_2pcf_version` is the pattern). Statistics computed from parts (COSEBIs, pure-E/B, the assembled `{version}.sacc`) are saved with `derived_from=parts` and carry their inputs' stamp, `s.metadata["blind"]`. From a3d1e028f3c6489eae9c678ca889d66acae560c5 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 5 Oct 2026 22:31:38 +0200 Subject: [PATCH 160/160] generate_pseudo_cl reads the (all, all) pair of its pseudo-Cl results cv.pseudo_cls is keyed by bin pair since the merge of develop's tomographic API, so the unkeyed lookup raised KeyError after the part was written and Snakemake deleted it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015izKkaSS9NyrSQBd5S6tqR --- workflow/scripts/generate_pseudo_cl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index 2bdcccd9..1decf091 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -127,7 +127,7 @@ def generate_pseudo_cl( # Pseudo-Cls only (no covariance), born directly at the final out_path. cv.calculate_pseudo_cl(compute_tomography=False, out_path=out_path) - ell = cv.pseudo_cls[version]["pseudo_cl"]["ELL"] + ell = cv.pseudo_cls[version]["tomo_bin_all_tomo_bin_all"]["pseudo_cl"]["ELL"] print(f"Generated pseudo-Cl with {len(ell)} ell bins") print(f"ell range: [{ell.min():.1f}, {ell.max():.1f}]") return out_path