diff --git a/.gitignore b/.gitignore index e9d09406..4af80f09 100644 --- a/.gitignore +++ b/.gitignore @@ -173,6 +173,8 @@ cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_0*.ini cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_v0*.ini cosmo_inference/cosmosis_config/glass_mocks_v* +config/glass_mock/test_data/results + # Regenerable catalog-paper plots: the paper TeX lives in the separate docs/ # repo, so these are script/notebook outputs, not LaTeX-tracked figures. papers/catalog/plots/*.pdf @@ -195,6 +197,8 @@ papers/cosmo_val/logs/ # Ignore scratch notebooks scratch/*/*.ipynb +scratch/guerrini/work_notebooks +scratch/guerrini/launch_scripts # Snakemake run state .snakemake/ diff --git a/config/cosmo_val/cosmo_gaussian_sims.yaml b/config/cosmo_val/cosmo_gaussian_sims.yaml new file mode 100644 index 00000000..da63e08d --- /dev/null +++ b/config/cosmo_val/cosmo_gaussian_sims.yaml @@ -0,0 +1,14 @@ +# --- Cosmological parameters (CAMB) --- +h: 0.6766 +Omega_m: 0.30966 +Omega_b: 0.04897 +ns: 0.9665 +sig8: 0.8102 +mnu: 0.06 +extra_params: + camb: + nonlinear: True + halofit_version: "mead2020_feedback" + HMCode_logT_AGN: 7.8 + kmax: 20.0 + kmax_extrapolate: 500 \ No newline at end of file diff --git a/config/glass_mock/config_glass_mock.yaml b/config/glass_mock/config_glass_mock.yaml new file mode 100644 index 00000000..366a0e5a --- /dev/null +++ b/config/glass_mock/config_glass_mock.yaml @@ -0,0 +1,41 @@ +# Config file to generate a GLASS mock. +# Example config file, uncomment and modify the inplace value if needed. + +# To run this configuration run the script `make_glass_sim.py` +# Options: +# -s, --seed: Random seed +# -N, --number: Mock Number for labelling +# -t, --test: Run in test mode +# -cb, --camb: get Camb C_ell +# -c, --config: Path to the configuration file to generate the simulation (Required) + +# --- Cosmological parameters (CAMB) --- +#h: 0.6766 +#Om: 0.30966 +#Ob: 0.04897 +#ns: 0.9665 +#sigma8: 0.8102 +#mnu: 0.06 +#log_T_AGN: 7.8 +#As_init: 2.1e-9 #seed As before sigma8 rescaling +#kmax: 20.0 + +# --- Resolution --- +nside: 1024 +dx: 120.0 +zmax: 3.0 + +# --- galaxy population (downstream of map generation) --- +nbins: 6 +n_arcmin2: [1, 1.02, 1.05, 0.85, 0.9, 1.2] +sigma_e: 0.2684 +bias: 1.2 +phz_sigma_0: 0.03 +#ia_bias: null + +# --- Runtime options --- +limber: True +mask_path: /n09data/guerrini/glass_mock_v1.4.6_rerun/mask_nside4096.fits +nz_path: /home/guerrini/sp_validation_cosmostat/config/glass_mock/test_data/redshift_distribution_tomo.txt +output_path: /n09data/guerrini/glass_mock_test/ +output_prefix: tomo_test_2 \ No newline at end of file diff --git a/config/glass_mock/config_glass_mock_test.yaml b/config/glass_mock/config_glass_mock_test.yaml new file mode 100644 index 00000000..702e0568 --- /dev/null +++ b/config/glass_mock/config_glass_mock_test.yaml @@ -0,0 +1,42 @@ +# Config file to generate a GLASS mock. +# Example config file, uncomment and modify the inplace value if needed. + +# To run this configuration run the script `make_glass_sim.py` +# Options: +# -s, --seed: Random seed +# -N, --number: Mock Number for labelling +# -t, --test: Run in test mode +# -cb, --camb: get Camb C_ell +# -v, --validation: Run some validation checks +# -c, --config: Path to the configuration file to generate the simulation (Required) + +# --- Cosmological parameters (CAMB) --- +#h: 0.7 +#Om: 0.30966 +#Ob: 0.04897 +#ns: 0.9665 +#sigma8: 0.8102 +#mnu: 0.06 +#log_T_AGN: 7.8 +#As_init: 2.1e-9 #seed As before sigma8 rescaling +#kmax: 20.0 + +# --- Resolution --- +nside: 32 +dx: 200.0 +zmax: 3 + +# --- galaxy population (downstream of map generation) --- +nbins: 6 +n_arcmin2: [4.0, 3.8, 4.2, 3.95, 4.05, 4.0] # number density in arcmin^-2. Input can be a float or a list (a number density for each bin) +sigma_e: 0.2684 # intrinsic ellipticity dispersion. Input can be a float or a list (a shape noise for each bin) +bias: 1.2 # galaxy bias. Input can be a float or a list (a galaxy bias for each bin) +phz_sigma_0: 0.03 +#ia_bias: null # Intrinsic alignment amplitude. Warning: this feature has not been robustly tested. + +# --- Runtime options --- +limber: True +mask_path: /n09data/guerrini/glass_mock_v1.4.6_rerun/mask_nside4096.fits +nz_path: /home/guerrini/sp_validation_cosmostat/config/glass_mock/test_data/redshift_distribution_tomo.txt +output_path: /n09data/guerrini/glass_mock_test/ +output_prefix: unions \ No newline at end of file diff --git a/config/glass_mock/test_data/mask.fits b/config/glass_mock/test_data/mask.fits new file mode 100644 index 00000000..4f86bf25 Binary files /dev/null and b/config/glass_mock/test_data/mask.fits differ diff --git a/config/glass_mock/test_data/redshift_distribution_non_tomo.txt b/config/glass_mock/test_data/redshift_distribution_non_tomo.txt new file mode 100644 index 00000000..15c4a47b --- /dev/null +++ b/config/glass_mock/test_data/redshift_distribution_non_tomo.txt @@ -0,0 +1,501 @@ +0.000000000000000000e+00 0.000000000000000000e+00 +1.000000000000000021e-02 1.539937506863479477e-03 +2.000000000000000042e-02 4.354621114841233237e-03 +2.999999999999999889e-02 7.996950397128197580e-03 +4.000000000000000083e-02 1.230564841035703891e-02 +5.000000000000000278e-02 1.718606196257605606e-02 +5.999999999999999778e-02 2.257303471016786772e-02 +7.000000000000000666e-02 2.841755418221013352e-02 +8.000000000000000167e-02 3.468055475441410296e-02 +8.999999999999999667e-02 4.132958972425367777e-02 +1.000000000000000056e-01 4.833686282899146008e-02 +1.100000000000000006e-01 5.567798045703171994e-02 +1.199999999999999956e-01 6.333111840482133492e-02 +1.300000000000000044e-01 7.127644240940279874e-02 +1.400000000000000133e-01 7.949569188500757444e-02 +1.499999999999999944e-01 8.797187290406095761e-02 +1.600000000000000033e-01 9.668902677243641774e-02 +1.700000000000000122e-01 1.056320523994966204e-01 +1.799999999999999933e-01 1.147865678773978826e-01 +1.900000000000000022e-01 1.241388012363979498e-01 +2.000000000000000111e-01 1.336755033059782716e-01 +2.099999999999999922e-01 1.433838775930178222e-01 +2.200000000000000011e-01 1.532515234450602115e-01 +2.300000000000000100e-01 1.632663897155546551e-01 +2.399999999999999911e-01 1.734167368240674412e-01 +2.500000000000000000e-01 1.836911055944393523e-01 +2.600000000000000089e-01 1.940782916144088155e-01 +2.700000000000000178e-01 2.045673241292221278e-01 +2.800000000000000266e-01 2.151474486850073475e-01 +2.899999999999999800e-01 2.258081128930605475e-01 +2.999999999999999889e-01 2.365389548062355740e-01 +3.099999999999999978e-01 2.473297934922943364e-01 +3.200000000000000067e-01 2.581706214628362583e-01 +3.300000000000000155e-01 2.690515986750057764e-01 +3.400000000000000244e-01 2.799630478700771419e-01 +3.500000000000000333e-01 2.908954510508430791e-01 +3.599999999999999867e-01 3.018394469304660488e-01 +3.699999999999999956e-01 3.127858292105780147e-01 +3.800000000000000044e-01 3.237255455670945414e-01 +3.900000000000000133e-01 3.346496972393329106e-01 +4.000000000000000222e-01 3.455495391322799814e-01 +4.100000000000000311e-01 3.564164803537959170e-01 +4.199999999999999845e-01 3.672420851185922985e-01 +4.299999999999999933e-01 3.780180739593260730e-01 +4.400000000000000022e-01 3.887363251923902085e-01 +4.500000000000000111e-01 3.993888765921613793e-01 +4.600000000000000200e-01 4.099679272327816371e-01 +4.700000000000000289e-01 4.204658394611290873e-01 +4.799999999999999822e-01 4.308751409686109590e-01 +4.899999999999999911e-01 4.411885269328758574e-01 +5.000000000000000000e-01 4.513988622035659537e-01 +5.100000000000000089e-01 4.614991835089001793e-01 +5.200000000000000178e-01 4.714827016622296085e-01 +5.300000000000000266e-01 4.813428037497889944e-01 +5.400000000000000355e-01 4.910730552827363704e-01 +5.500000000000000444e-01 5.006672022982299186e-01 +5.600000000000000533e-01 5.101191733957912700e-01 +5.700000000000000622e-01 5.194230816965546005e-01 +5.799999999999999600e-01 5.285732267142277951e-01 +5.899999999999999689e-01 5.375640961277033947e-01 +5.999999999999999778e-01 5.463903674462829985e-01 +6.099999999999999867e-01 5.550469095594072755e-01 +6.199999999999999956e-01 5.635287841636549278e-01 +6.300000000000000044e-01 5.718312470605623199e-01 +6.400000000000000133e-01 5.799497493195616693e-01 +6.500000000000000222e-01 5.878799383010221424e-01 +6.600000000000000311e-01 5.956176585350180241e-01 +6.700000000000000400e-01 6.031589524520577505e-01 +6.800000000000000488e-01 6.105000609625610419e-01 +6.900000000000000577e-01 6.176374238824169360e-01 +7.000000000000000666e-01 6.245676802024405783e-01 +7.099999999999999645e-01 6.312876682000376771e-01 +7.199999999999999734e-01 6.377944253918177431e-01 +7.299999999999999822e-01 6.440851883263314370e-01 +7.399999999999999911e-01 6.501573922165090336e-01 +7.500000000000000000e-01 6.560086704117557010e-01 +7.600000000000000089e-01 6.616368537100251190e-01 +7.700000000000000178e-01 6.670399695105416749e-01 +7.800000000000000266e-01 6.722162408081657770e-01 +7.900000000000000355e-01 6.771640850307087955e-01 +8.000000000000000444e-01 6.818821127208092303e-01 +8.100000000000000533e-01 6.863691260642560410e-01 +8.200000000000000622e-01 6.906241172669157491e-01 +8.300000000000000711e-01 6.946462667826814874e-01 +8.399999999999999689e-01 6.984349413950913243e-01 +8.499999999999999778e-01 7.019896921555075497e-01 +8.599999999999999867e-01 7.053102521809501146e-01 +8.699999999999999956e-01 7.083965343148931337e-01 +8.800000000000000044e-01 7.112486286545204317e-01 +8.900000000000000133e-01 7.138667999481170812e-01 +9.000000000000000222e-01 7.162514848664459644e-01 +9.100000000000000311e-01 7.184032891521118236e-01 +9.200000000000000400e-01 7.203229846510679213e-01 +9.300000000000000488e-01 7.220115062305524356e-01 +9.400000000000000577e-01 7.234699485878687275e-01 +9.500000000000000666e-01 7.246995629545379680e-01 +9.599999999999999645e-01 7.257017537004600838e-01 +9.699999999999999734e-01 7.264780748428069090e-01 +9.799999999999999822e-01 7.270302264644648016e-01 +9.899999999999999911e-01 7.273600510469091507e-01 +1.000000000000000000e+00 7.274695297224667012e-01 +1.010000000000000009e+00 7.273607784509685814e-01 +1.020000000000000018e+00 7.270360441258496564e-01 +1.030000000000000027e+00 7.264977006147820271e-01 +1.040000000000000036e+00 7.257482447399601355e-01 +1.050000000000000044e+00 7.247902922031738138e-01 +1.060000000000000053e+00 7.236265734608122724e-01 +1.070000000000000062e+00 7.222599295539492426e-01 +1.080000000000000071e+00 7.206933078986481611e-01 +1.090000000000000080e+00 7.189297580416124100e-01 +1.100000000000000089e+00 7.169724273862850827e-01 +1.110000000000000098e+00 7.148245568944672224e-01 +1.120000000000000107e+00 7.124894767684958330e-01 +1.130000000000000115e+00 7.099706021189617910e-01 +1.140000000000000124e+00 7.072714286229138114e-01 +1.150000000000000133e+00 7.043955281774211263e-01 +1.159999999999999920e+00 7.013465445533126896e-01 +1.169999999999999929e+00 6.981281890538297841e-01 +1.179999999999999938e+00 6.947442361828605195e-01 +1.189999999999999947e+00 6.911985193273350037e-01 +1.199999999999999956e+00 6.874949264582724817e-01 +1.209999999999999964e+00 6.836373958548797036e-01 +1.219999999999999973e+00 6.796299118559959718e-01 +1.229999999999999982e+00 6.754765006430851759e-01 +1.239999999999999991e+00 6.711812260588544410e-01 +1.250000000000000000e+00 6.667481854654786488e-01 +1.260000000000000009e+00 6.621815056462829752e-01 +1.270000000000000018e+00 6.574853387546280015e-01 +1.280000000000000027e+00 6.526638583136016303e-01 +1.290000000000000036e+00 6.477212552700154502e-01 +1.300000000000000044e+00 6.426617341060522071e-01 +1.310000000000000053e+00 6.374895090118015695e-01 +1.320000000000000062e+00 6.322088001217657238e-01 +1.330000000000000071e+00 6.268238298182954304e-01 +1.340000000000000080e+00 6.213388191047718445e-01 +1.350000000000000089e+00 6.157579840512120706e-01 +1.360000000000000098e+00 6.100855323148330900e-01 +1.370000000000000107e+00 6.043256597379692563e-01 +1.380000000000000115e+00 5.984825470255971114e-01 +1.390000000000000124e+00 5.925603565045712839e-01 +1.400000000000000133e+00 5.865632289665424492e-01 +1.409999999999999920e+00 5.804952805963723428e-01 +1.419999999999999929e+00 5.743605999877305912e-01 +1.429999999999999938e+00 5.681632452474034700e-01 +1.439999999999999947e+00 5.619072411897095831e-01 +1.449999999999999956e+00 5.555965766222765723e-01 +1.459999999999999964e+00 5.492352017242854156e-01 +1.469999999999999973e+00 5.428270255181580906e-01 +1.479999999999999982e+00 5.363759134355238212e-01 +1.489999999999999991e+00 5.298856849781603540e-01 +1.500000000000000000e+00 5.233601114744772520e-01 +1.510000000000000009e+00 5.168029139319725296e-01 +1.520000000000000018e+00 5.102177609859670504e-01 +1.530000000000000027e+00 5.036082669447908833e-01 +1.540000000000000036e+00 4.969779899314723526e-01 +1.550000000000000044e+00 4.903304301218576189e-01 +1.560000000000000053e+00 4.836690280789701091e-01 +1.570000000000000062e+00 4.769971631832984893e-01 +1.580000000000000071e+00 4.703181521585940161e-01 +1.590000000000000080e+00 4.636352476926445809e-01 +1.600000000000000089e+00 4.569516371523797416e-01 +1.610000000000000098e+00 4.502704413925717186e-01 +1.620000000000000107e+00 4.435947136572779836e-01 +1.630000000000000115e+00 4.369274385730903543e-01 +1.640000000000000124e+00 4.302715312331513720e-01 +1.650000000000000133e+00 4.236298363708130843e-01 +1.660000000000000142e+00 4.170051276217281999e-01 +1.669999999999999929e+00 4.104001068730749879e-01 +1.679999999999999938e+00 4.038174036985497928e-01 +1.689999999999999947e+00 3.972595748776704960e-01 +1.699999999999999956e+00 3.907291039978806890e-01 +1.709999999999999964e+00 3.842284011378613306e-01 +1.719999999999999973e+00 3.777598026304040957e-01 +1.729999999999999982e+00 3.713255709031357821e-01 +1.739999999999999991e+00 3.649278943953298548e-01 +1.750000000000000000e+00 3.585688875489879690e-01 +1.760000000000000009e+00 3.522505908723272405e-01 +1.770000000000000018e+00 3.459749710737644590e-01 +1.780000000000000027e+00 3.397439212644507434e-01 +1.790000000000000036e+00 3.335592612273660107e-01 +1.800000000000000044e+00 3.274227377509619230e-01 +1.810000000000000053e+00 3.213360250253015638e-01 +1.820000000000000062e+00 3.153007250986233356e-01 +1.830000000000000071e+00 3.093183683922376415e-01 +1.840000000000000080e+00 3.033904142716395969e-01 +1.850000000000000089e+00 2.975182516717121417e-01 +1.860000000000000098e+00 2.917031997738803728e-01 +1.870000000000000107e+00 2.859465087330672062e-01 +1.880000000000000115e+00 2.802493604523003667e-01 +1.890000000000000124e+00 2.746128694028142636e-01 +1.900000000000000133e+00 2.690380834874948612e-01 +1.910000000000000142e+00 2.635259849455179881e-01 +1.919999999999999929e+00 2.580774912960386858e-01 +1.929999999999999938e+00 2.526934563188009686e-01 +1.939999999999999947e+00 2.473746710695480799e-01 +1.949999999999999956e+00 2.421218649281299262e-01 +1.959999999999999964e+00 2.369357066772201093e-01 +1.969999999999999973e+00 2.318168056095795404e-01 +1.979999999999999982e+00 2.267657126618208285e-01 +1.989999999999999991e+00 2.217829215726565273e-01 +2.000000000000000000e+00 2.168688700636401956e-01 +2.010000000000000231e+00 2.120239410404359814e-01 +2.020000000000000018e+00 2.072484638126886325e-01 +2.030000000000000249e+00 2.025427153305902617e-01 +2.040000000000000036e+00 1.979069214362860762e-01 +2.049999999999999822e+00 1.933412581282832887e-01 +2.060000000000000053e+00 1.888458528370793865e-01 +2.069999999999999840e+00 1.844207857102533321e-01 +2.080000000000000071e+00 1.800660909053100611e-01 +2.089999999999999858e+00 1.757817578886092247e-01 +2.100000000000000089e+00 1.715677327387462869e-01 +2.109999999999999876e+00 1.674239194528092944e-01 +2.120000000000000107e+00 1.633501812539643328e-01 +2.129999999999999893e+00 1.593463418988812497e-01 +2.140000000000000124e+00 1.554121869835467795e-01 +2.149999999999999911e+00 1.515474652460686023e-01 +2.160000000000000142e+00 1.477518898651102308e-01 +2.169999999999999929e+00 1.440251397526542965e-01 +2.180000000000000160e+00 1.403668608398331852e-01 +2.189999999999999947e+00 1.367766673546163547e-01 +2.200000000000000178e+00 1.332541430901921786e-01 +2.209999999999999964e+00 1.297988426629307357e-01 +2.220000000000000195e+00 1.264102927588607039e-01 +2.229999999999999982e+00 1.230879933676455457e-01 +2.240000000000000213e+00 1.198314190030877746e-01 +2.250000000000000000e+00 1.166400199092426510e-01 +2.260000000000000231e+00 1.135132232512673367e-01 +2.270000000000000018e+00 1.104504342901816843e-01 +2.280000000000000249e+00 1.074510375407597695e-01 +2.290000000000000036e+00 1.045143979118249022e-01 +2.300000000000000266e+00 1.016398618282587235e-01 +2.310000000000000053e+00 9.882675833408750832e-02 +2.319999999999999840e+00 9.607440017604933213e-02 +2.330000000000000071e+00 9.338208486709335954e-02 +2.339999999999999858e+00 9.074909572930309198e-02 +2.350000000000000089e+00 8.817470291577877983e-02 +2.359999999999999876e+00 8.565816441106151313e-02 +2.370000000000000107e+00 8.319872700971145396e-02 +2.379999999999999893e+00 8.079562727270708788e-02 +2.390000000000000124e+00 7.844809246135860115e-02 +2.399999999999999911e+00 7.615534144847727061e-02 +2.410000000000000142e+00 7.391658560657310417e-02 +2.419999999999999929e+00 7.173102967289454734e-02 +2.430000000000000160e+00 6.959787259115582381e-02 +2.439999999999999947e+00 6.751630832983530894e-02 +2.450000000000000178e+00 6.548552667695832497e-02 +2.459999999999999964e+00 6.350471401131446703e-02 +2.470000000000000195e+00 6.157305405008499377e-02 +2.479999999999999982e+00 5.968972857289397971e-02 +2.490000000000000213e+00 5.785391812231730629e-02 +2.500000000000000000e+00 5.606480268091939417e-02 +2.510000000000000231e+00 5.432156232490855524e-02 +2.520000000000000018e+00 5.262337785453282540e-02 +2.530000000000000249e+00 5.096943140135768574e-02 +2.540000000000000036e+00 4.935890701259598029e-02 +2.550000000000000266e+00 4.779099121267828265e-02 +2.560000000000000053e+00 4.626487354227725585e-02 +2.569999999999999840e+00 4.477974707501646695e-02 +2.580000000000000071e+00 4.333480891211646813e-02 +2.589999999999999858e+00 4.192926065524795604e-02 +2.600000000000000089e+00 4.056230885787701079e-02 +2.609999999999999876e+00 3.923316545540828093e-02 +2.620000000000000107e+00 3.794104817444162586e-02 +2.629999999999999893e+00 3.668518092147692727e-02 +2.640000000000000124e+00 3.546479415140973979e-02 +2.649999999999999911e+00 3.427912521617817315e-02 +2.660000000000000142e+00 3.312741869392702571e-02 +2.669999999999999929e+00 3.200892669907012822e-02 +2.680000000000000160e+00 3.092290917363746394e-02 +2.689999999999999947e+00 2.986863416030518040e-02 +2.700000000000000178e+00 2.884537805751076489e-02 +2.709999999999999964e+00 2.785242585706639018e-02 +2.720000000000000195e+00 2.688907136468463369e-02 +2.729999999999999982e+00 2.595461740384056790e-02 +2.740000000000000213e+00 2.504837600339402184e-02 +2.750000000000000000e+00 2.416966856940315489e-02 +2.760000000000000231e+00 2.331782604155990465e-02 +2.770000000000000018e+00 2.249218903468288774e-02 +2.780000000000000249e+00 2.169210796570213151e-02 +2.790000000000000036e+00 2.091694316657254429e-02 +2.800000000000000266e+00 2.016606498355184157e-02 +2.810000000000000053e+00 1.943885386327943651e-02 +2.819999999999999840e+00 1.873470042608972944e-02 +2.830000000000000071e+00 1.805300552699367136e-02 +2.839999999999999858e+00 1.739318030475876797e-02 +2.850000000000000089e+00 1.675464621951530675e-02 +2.859999999999999876e+00 1.613683507931390829e-02 +2.870000000000000107e+00 1.553918905605487094e-02 +2.879999999999999893e+00 1.496116069120686544e-02 +2.890000000000000124e+00 1.440221289172637954e-02 +2.899999999999999911e+00 1.386181891658643146e-02 +2.910000000000000142e+00 1.333946235431587163e-02 +2.919999999999999929e+00 1.283463709194639958e-02 +2.930000000000000160e+00 1.234684727575733817e-02 +2.939999999999999947e+00 1.187560726420353215e-02 +2.950000000000000178e+00 1.142044157340304719e-02 +2.959999999999999964e+00 1.098088481555707825e-02 +2.970000000000000195e+00 1.055648163066528859e-02 +2.979999999999999982e+00 1.014678661189444622e-02 +2.990000000000000213e+00 9.751364224949137655e-03 +3.000000000000000000e+00 9.369788721787701211e-03 +3.010000000000000231e+00 9.001644049016828408e-03 +3.020000000000000018e+00 8.646523751292257282e-03 +3.030000000000000249e+00 8.304030870043598370e-03 +3.040000000000000036e+00 7.973777837834540930e-03 +3.050000000000000266e+00 7.655386368660446923e-03 +3.060000000000000053e+00 7.348487344478121276e-03 +3.070000000000000284e+00 7.052720698253536757e-03 +3.080000000000000071e+00 6.767735293805762255e-03 +3.089999999999999858e+00 6.493188802716336541e-03 +3.100000000000000089e+00 6.228747578565676894e-03 +3.109999999999999876e+00 5.974086528749391473e-03 +3.120000000000000107e+00 5.728888984119206822e-03 +3.129999999999999893e+00 5.492846566685419957e-03 +3.140000000000000124e+00 5.265659055608580032e-03 +3.149999999999999911e+00 5.047034251701308823e-03 +3.160000000000000142e+00 4.836687840651790471e-03 +3.169999999999999929e+00 4.634343255173532447e-03 +3.180000000000000160e+00 4.439731536277212454e-03 +3.189999999999999947e+00 4.252591193853345712e-03 +3.200000000000000178e+00 4.072668066746101476e-03 +3.209999999999999964e+00 3.899715182491525142e-03 +3.220000000000000195e+00 3.733492616885461263e-03 +3.229999999999999982e+00 3.573767353539470965e-03 +3.240000000000000213e+00 3.420313143575352040e-03 +3.250000000000000000e+00 3.272910365602150953e-03 +3.260000000000000231e+00 3.131345886112087574e-03 +3.270000000000000018e+00 2.995412920425440974e-03 +3.280000000000000249e+00 2.864910894307190828e-03 +3.290000000000000036e+00 2.739645306372137483e-03 +3.300000000000000266e+00 2.619427591388258370e-03 +3.310000000000000053e+00 2.504074984582378965e-03 +3.320000000000000284e+00 2.393410387045433359e-03 +3.330000000000000071e+00 2.287262232329256937e-03 +3.339999999999999858e+00 2.185464354320594375e-03 +3.350000000000000089e+00 2.087855856472531791e-03 +3.359999999999999876e+00 1.994280982468088839e-03 +3.370000000000000107e+00 1.904588988385240433e-03 +3.379999999999999893e+00 1.818634016427714837e-03 +3.390000000000000124e+00 1.736274970280524870e-03 +3.399999999999999911e+00 1.657375392144809457e-03 +3.410000000000000142e+00 1.581803341501400190e-03 +3.419999999999999929e+00 1.509431275648455547e-03 +3.430000000000000160e+00 1.440135932053693713e-03 +3.439999999999999947e+00 1.373798212557982359e-03 +3.450000000000000178e+00 1.310303069462591136e-03 +3.459999999999999964e+00 1.249539393528863824e-03 +3.470000000000000195e+00 1.191399903915032771e-03 +3.479999999999999982e+00 1.135781040071535703e-03 +3.490000000000000213e+00 1.082582855612596300e-03 +3.500000000000000000e+00 1.031708914178621796e-03 +3.510000000000000231e+00 9.830661873007860967e-04 +3.520000000000000018e+00 9.365649542761766066e-04 +3.530000000000000249e+00 8.921187040589807250e-04 +3.540000000000000036e+00 8.496440391705326880e-04 +3.550000000000000266e+00 8.090605816283328536e-04 +3.560000000000000053e+00 7.702908808918079286e-04 +3.570000000000000284e+00 7.332603238201528852e-04 +3.580000000000000071e+00 6.978970466354378647e-04 +3.589999999999999858e+00 6.641318488820351822e-04 +3.600000000000000089e+00 6.318981093715066296e-04 +3.609999999999999876e+00 6.011317041001773337e-04 +3.620000000000000107e+00 5.717709261249223933e-04 +3.629999999999999893e+00 5.437564073810660040e-04 +3.640000000000000124e+00 5.170310424247296808e-04 +3.649999999999999911e+00 4.915399140806066009e-04 +3.660000000000000142e+00 4.672302209747489265e-04 +3.669999999999999929e+00 4.440512069308050593e-04 +3.680000000000000160e+00 4.219540922069490454e-04 +3.689999999999999947e+00 4.008920065497597273e-04 +3.700000000000000178e+00 3.808199240403083684e-04 +3.709999999999999964e+00 3.616945997068966878e-04 +3.720000000000000195e+00 3.434745078780324068e-04 +3.729999999999999982e+00 3.261197822486135807e-04 +3.740000000000000213e+00 3.095921576315514476e-04 +3.750000000000000000e+00 2.938549133666026425e-04 +3.760000000000000231e+00 2.788728183575983708e-04 +3.770000000000000018e+00 2.646120777089062113e-04 +3.780000000000000249e+00 2.510402809315314646e-04 +3.790000000000000036e+00 2.381263516890092305e-04 +3.800000000000000266e+00 2.258404990529570735e-04 +3.810000000000000053e+00 2.141541702379862069e-04 +3.820000000000000284e+00 2.030400047855086668e-04 +3.830000000000000071e+00 1.924717901659137053e-04 +3.839999999999999858e+00 1.824244187684995189e-04 +3.850000000000000089e+00 1.728738462485919904e-04 +3.859999999999999876e+00 1.637970512012828822e-04 +3.870000000000000107e+00 1.551719961313237036e-04 +3.879999999999999893e+00 1.469775896888190277e-04 +3.890000000000000124e+00 1.391936501405120579e-04 +3.899999999999999911e+00 1.318008700466514831e-04 +3.910000000000000142e+00 1.247807821136226263e-04 +3.919999999999999929e+00 1.181157261927979286e-04 +3.930000000000000160e+00 1.117888173962968014e-04 +3.939999999999999947e+00 1.057839153006771295e-04 +3.950000000000000178e+00 1.000855942098637422e-04 +3.959999999999999964e+00 9.467911444898297101e-05 +3.970000000000000195e+00 8.955039466111365304e-05 +3.979999999999999982e+00 8.468598507935508899e-05 +3.990000000000000213e+00 8.007304174698640564e-05 +4.000000000000000000e+00 7.569930165891772917e-05 +4.009999999999999787e+00 7.155305879803737284e-05 +4.020000000000000462e+00 6.762314104049842590e-05 +4.030000000000000249e+00 6.389888790443169124e-05 +4.040000000000000036e+00 6.037012911701270163e-05 +4.049999999999999822e+00 5.702716397528318892e-05 +4.060000000000000497e+00 5.386074147658470357e-05 +4.070000000000000284e+00 5.086204119493443527e-05 +4.080000000000000071e+00 4.802265488015101832e-05 +4.089999999999999858e+00 4.533456875702039282e-05 +4.099999999999999645e+00 4.279014650226993922e-05 +4.110000000000000320e+00 4.038211287760699484e-05 +4.120000000000000107e+00 3.810353799756363775e-05 +4.129999999999999893e+00 3.594782221137101584e-05 +4.139999999999999680e+00 3.390868157858218285e-05 +4.150000000000000355e+00 3.198013391863285836e-05 +4.160000000000000142e+00 3.015648541502528071e-05 +4.169999999999999929e+00 2.843231775528890664e-05 +4.179999999999999716e+00 2.680247578835809533e-05 +4.190000000000000391e+00 2.526205568147361550e-05 +4.200000000000000178e+00 2.380639355918723205e-05 +4.209999999999999964e+00 2.243105460751029207e-05 +4.219999999999999751e+00 2.113182262671278963e-05 +4.230000000000000426e+00 1.990469001672597201e-05 +4.240000000000000213e+00 1.874584817955670109e-05 +4.250000000000000000e+00 1.765167832355950015e-05 +4.259999999999999787e+00 1.661874265485266630e-05 +4.270000000000000462e+00 1.564377594159051401e-05 +4.280000000000000249e+00 1.472367743722979956e-05 +4.290000000000000036e+00 1.385550314934019049e-05 +4.299999999999999822e+00 1.303645844092226011e-05 +4.310000000000000497e+00 1.226389095159203316e-05 +4.320000000000000284e+00 1.153528382638809769e-05 +4.330000000000000071e+00 1.084824924034087195e-05 +4.339999999999999858e+00 1.020052220732327752e-05 +4.350000000000000533e+00 9.589954662070093405e-06 +4.360000000000000320e+00 9.014509804616692749e-06 +4.370000000000000107e+00 8.472256696760831896e-06 +4.379999999999999893e+00 7.961365100498308370e-06 +4.389999999999999680e+00 7.480100548719846339e-06 +4.400000000000000355e+00 7.026819638787182111e-06 +4.410000000000000142e+00 6.599965539928366694e-06 +4.419999999999999929e+00 6.198063705704921943e-06 +4.429999999999999716e+00 5.819717783110912374e-06 +4.440000000000000391e+00 5.463605710160176731e-06 +4.450000000000000178e+00 5.128475994109442647e-06 +4.459999999999999964e+00 4.813144162745563934e-06 +4.469999999999999751e+00 4.516489381441409406e-06 +4.480000000000000426e+00 4.237451228949907836e-06 +4.490000000000000213e+00 3.975026625166211130e-06 +4.500000000000000000e+00 3.728266904338776927e-06 +4.509999999999999787e+00 3.496275027455002836e-06 +4.520000000000000462e+00 3.278202927764108358e-06 +4.530000000000000249e+00 3.073248983629502589e-06 +4.540000000000000036e+00 2.880655613125952564e-06 +4.549999999999999822e+00 2.699706985013199309e-06 +4.560000000000000497e+00 2.529726840926145037e-06 +4.570000000000000284e+00 2.370076423825027983e-06 +4.580000000000000071e+00 2.220152507944157265e-06 +4.589999999999999858e+00 2.079385525668439710e-06 +4.600000000000000533e+00 1.947237786949184512e-06 +4.610000000000000320e+00 1.823201787048896874e-06 +4.620000000000000107e+00 1.706798598575101179e-06 +4.629999999999999893e+00 1.597576343929440794e-06 +4.639999999999999680e+00 1.495108744457528071e-06 +4.650000000000000355e+00 1.398993742739143427e-06 +4.660000000000000142e+00 1.308852194607379469e-06 +4.669999999999999929e+00 1.224326627628362285e-06 +4.679999999999999716e+00 1.145080062911691148e-06 +4.690000000000000391e+00 1.070794897254885976e-06 +4.700000000000000178e+00 1.001171842753453140e-06 +4.709999999999999964e+00 9.359289211317424164e-07 +4.719999999999999751e+00 8.748005101689583192e-07 +4.730000000000000426e+00 8.175364397089691711e-07 +4.740000000000000213e+00 7.639011348529320970e-07 +4.750000000000000000e+00 7.136728040394321014e-07 +4.759999999999999787e+00 6.666426698190985282e-07 +4.770000000000000462e+00 6.226142402281157827e-07 +4.780000000000000249e+00 5.814026187594347311e-07 +4.790000000000000036e+00 5.428338510206137025e-07 +4.799999999999999822e+00 5.067443062540728789e-07 +4.810000000000000497e+00 4.729800919787301626e-07 +4.820000000000000284e+00 4.413965000918793889e-07 +4.830000000000000071e+00 4.118574828467606342e-07 +4.839999999999999858e+00 3.842351571948298727e-07 +4.850000000000000533e+00 3.584093360520481808e-07 +4.860000000000000320e+00 3.342670851160997905e-07 +4.870000000000000107e+00 3.117023039260469944e-07 +4.879999999999999893e+00 2.906153299179266810e-07 +4.889999999999999680e+00 2.709125642890161304e-07 +4.900000000000000355e+00 2.525061185403120229e-07 +4.910000000000000142e+00 2.353134806210512852e-07 +4.919999999999999929e+00 2.192571996510261051e-07 +4.929999999999999716e+00 2.042645882462201667e-07 +4.940000000000000391e+00 1.902674415206736977e-07 +4.950000000000000178e+00 1.772017718829930070e-07 +4.959999999999999964e+00 1.650075587892168629e-07 +4.969999999999999751e+00 1.536285126552568616e-07 +4.980000000000000426e+00 1.430118521716543266e-07 +4.990000000000000213e+00 1.331080943011983516e-07 +5.000000000000000000e+00 1.238708562759898487e-07 diff --git a/config/glass_mock/test_data/redshift_distribution_tomo.txt b/config/glass_mock/test_data/redshift_distribution_tomo.txt new file mode 100644 index 00000000..92fd76f6 --- /dev/null +++ b/config/glass_mock/test_data/redshift_distribution_tomo.txt @@ -0,0 +1,501 @@ +0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +1.000000000000000021e-02 1.539937506863479477e-03 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +2.000000000000000042e-02 4.354621114841233237e-03 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +2.999999999999999889e-02 7.996950397128197580e-03 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +4.000000000000000083e-02 1.230564841035703891e-02 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +5.000000000000000278e-02 1.718606196257605606e-02 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +5.999999999999999778e-02 2.257303471016786772e-02 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +7.000000000000000666e-02 2.841755418221013352e-02 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +8.000000000000000167e-02 3.468055475441410296e-02 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +8.999999999999999667e-02 4.132958972425367777e-02 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +1.000000000000000056e-01 4.833686282899146008e-02 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +1.100000000000000006e-01 5.567798045703171994e-02 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +1.199999999999999956e-01 6.333111840482133492e-02 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +1.300000000000000044e-01 7.127644240940279874e-02 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +1.400000000000000133e-01 7.949569188500757444e-02 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +1.499999999999999944e-01 8.797187290406095761e-02 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +1.600000000000000033e-01 9.668902677243640387e-02 1.614882619331313001e-17 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +1.700000000000000122e-01 1.056320523994964816e-01 1.351137338877444904e-16 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +1.799999999999999933e-01 1.147865678773969250e-01 9.505027516849669190e-16 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +1.900000000000000022e-01 1.241388012363918020e-01 6.151176652093682308e-15 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +2.000000000000000111e-01 1.336755033059418007e-01 3.646504877060481157e-14 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +2.099999999999999922e-01 1.433838775928187315e-01 1.991003829296580357e-13 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +2.200000000000000011e-01 1.532515234440552376e-01 1.004988630970547006e-12 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +2.300000000000000100e-01 1.632663897108496964e-01 4.704946108657897756e-12 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +2.399999999999999911e-01 1.734167368035731960e-01 2.049424340658864430e-11 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +2.500000000000000000e-01 1.836911055111285218e-01 8.331084045868062068e-11 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +2.600000000000000089e-01 1.940782912974445551e-01 3.169642814338128490e-10 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +2.700000000000000178e-01 2.045673229974847573e-01 1.131737369315146494e-09 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +2.800000000000000266e-01 2.151474448827670471e-01 3.802240322614322598e-09 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +2.899999999999999800e-01 2.258081008434757453e-01 1.204958480370055000e-08 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +2.999999999999999889e-01 2.365389187006463334e-01 3.610558922273093705e-08 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +3.099999999999999978e-01 2.473296909667828114e-01 1.025255115058832154e-07 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +3.200000000000000067e-01 2.581703449681812534e-01 2.764946549727501638e-07 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +3.300000000000000155e-01 2.690508890313944024e-01 7.096436114082356453e-07 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +3.400000000000000244e-01 2.799613110462015886e-01 1.736823875468063563e-06 4.662322231933824855e-17 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +3.500000000000000333e-01 2.908913897960314077e-01 4.061254811421941901e-06 2.583670898901443416e-16 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +3.599999999999999867e-01 3.018303572909819543e-01 9.089639482710861433e-06 1.373947407418390805e-15 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +3.699999999999999956e-01 3.127663228662022066e-01 1.950634436891043497e-05 6.893151510737389432e-15 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +3.800000000000000044e-01 3.236853407172435304e-01 4.020484981860280042e-05 3.241856199299520783e-14 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +3.900000000000000133e-01 3.345699792701520847e-01 7.971796903761345344e-05 1.432642055538383969e-13 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +4.000000000000000222e-01 3.453972446681233266e-01 1.522944635599302987e-04 5.966898959928456991e-13 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +4.100000000000000311e-01 3.561357356872097979e-01 2.807446642384667278e-04 2.347659114530768397e-12 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +4.199999999999999845e-01 3.667419771453656052e-01 5.001079644842026418e-04 8.742488218771877294e-12 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +4.299999999999999933e-01 3.771560018219017318e-01 8.620721065502708535e-04 3.087410164958152419e-11 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +4.400000000000000022e-01 3.872964251093485299e-01 1.439899979450073690e-03 1.035915583661646512e-10 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +4.500000000000000111e-01 3.970554653825547065e-01 2.333410878777936875e-03 3.308287653551427720e-10 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +4.600000000000000200e-01 4.062945736079904835e-01 3.673352617441839798e-03 1.007349317623349535e-09 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +4.700000000000000289e-01 4.148415023345984332e-01 5.624334197152194674e-03 2.929378468558717274e-09 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +4.799999999999999822e-01 4.224897163924213883e-01 8.385416427587607385e-03 8.148601966637974086e-09 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +4.899999999999999911e-01 4.290009818473208281e-01 1.218752337000364276e-02 2.171555134073154624e-08 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 +5.000000000000000000e-01 4.341117422412649196e-01 1.728706443813065111e-02 5.552417039207953005e-08 5.011534101145543459e-17 0.000000000000000000e+00 0.000000000000000000e+00 +5.100000000000000089e-01 4.375435079817874828e-01 2.395553912012464079e-02 1.364069878953243777e-07 2.049468077523805556e-16 0.000000000000000000e+00 0.000000000000000000e+00 +5.200000000000000178e-01 4.390169859943412467e-01 3.246539324048713232e-02 3.224274003695541761e-07 8.636940693148735105e-16 0.000000000000000000e+00 0.000000000000000000e+00 +5.300000000000000266e-01 4.382691325285715256e-01 4.307293697093857177e-02 7.342502750767433476e-07 3.767504937405573431e-15 0.000000000000000000e+00 0.000000000000000000e+00 +5.400000000000000355e-01 4.350718106991963707e-01 5.599963160420910746e-02 1.612979315532863648e-06 1.537465727950917325e-14 0.000000000000000000e+00 0.000000000000000000e+00 +5.500000000000000444e-01 4.292503654041376904e-01 7.141341456206214133e-02 3.422331970336960685e-06 5.975411748414992075e-14 0.000000000000000000e+00 0.000000000000000000e+00 +5.600000000000000533e-01 4.207002636678696672e-01 8.941188803754447634e-02 7.021690155691773514e-06 2.214129888757403200e-13 0.000000000000000000e+00 0.000000000000000000e+00 +5.700000000000000622e-01 4.094000272965073739e-01 1.100091071956577626e-01 1.394720360637749612e-05 7.830387799077097932e-13 0.000000000000000000e+00 0.000000000000000000e+00 +5.799999999999999600e-01 3.954190047130465846e-01 1.331273721792688158e-01 2.684981926551858141e-05 2.646886166283657862e-12 0.000000000000000000e+00 0.000000000000000000e+00 +5.899999999999999689e-01 3.789190465915061590e-01 1.585948995274572282e-01 5.015000017562216392e-05 8.564369808719667552e-12 0.000000000000000000e+00 0.000000000000000000e+00 +5.999999999999999778e-01 3.601497897086659572e-01 1.861496016761532424e-01 9.097603490253185112e-05 2.656128200553507587e-11 0.000000000000000000e+00 0.000000000000000000e+00 +6.099999999999999867e-01 3.394379220485379545e-01 2.154485351236863355e-01 1.604523081202781538e-04 7.906276363770337246e-11 0.000000000000000000e+00 0.000000000000000000e+00 +6.199999999999999956e-01 3.171714061995755674e-01 2.460819864629658904e-01 2.753912749512043212e-04 2.261623034735574779e-10 0.000000000000000000e+00 0.000000000000000000e+00 +6.300000000000000044e-01 2.937800998367049998e-01 2.775907325507723233e-01 4.604140505979135815e-04 6.224870767506286974e-10 0.000000000000000000e+00 0.000000000000000000e+00 +6.400000000000000133e-01 2.697144812642079392e-01 3.094847895376878766e-01 7.504768671437603636e-04 1.650522107997988509e-09 0.000000000000000000e+00 0.000000000000000000e+00 +6.500000000000000222e-01 2.454242469633750034e-01 3.412619728228676319e-01 1.193714293979784543e-03 4.220799766222615784e-09 0.000000000000000000e+00 0.000000000000000000e+00 +6.600000000000000311e-01 2.213384108026694164e-01 3.724248082580806640e-01 1.854429052641198795e-03 1.042162672717165301e-08 0.000000000000000000e+00 0.000000000000000000e+00 +6.700000000000000400e-01 1.978482402854722932e-01 4.024947164274449429e-01 2.815970867028899302e-03 2.487211160489419456e-08 3.348204782605323907e-17 0.000000000000000000e+00 +6.800000000000000488e-01 1.752939689160500913e-01 4.310228690679417940e-01 4.183165543133970064e-03 5.743543506937962249e-08 6.777912242156971001e-17 0.000000000000000000e+00 +6.900000000000000577e-01 1.539557856234219924e-01 4.575976142393253232e-01 6.083895557180169812e-03 1.284624891195774969e-07 3.085718799889921458e-16 0.000000000000000000e+00 +7.000000000000000666e-01 1.340491780397383981e-01 4.818488131742932890e-01 8.669410423207461053e-03 2.785652001697901804e-07 1.248136954195448217e-15 0.000000000000000000e+00 +7.099999999999999645e-01 1.157243407682299330e-01 5.034497692372071898e-01 1.211297199850875674e-02 5.861960871976178571e-07 4.520612173364571743e-15 0.000000000000000000e+00 +7.199999999999999734e-01 9.906908180325493807e-02 5.221176213873008898e-01 1.660652401890975668e-02 1.198182336259976755e-06 1.586130685546551663e-14 0.000000000000000000e+00 +7.299999999999999822e-01 8.411448308133587237e-02 5.376131084736023302e-01 2.235521580753587964e-02 2.380963803790221850e-06 5.355935762190425332e-14 0.000000000000000000e+00 +7.399999999999999911e-01 7.084249342384477122e-02 5.497404978324285096e-01 2.956979726537413769e-02 4.603694687655575934e-06 1.739585492590330683e-13 0.000000000000000000e+00 +7.500000000000000000e-01 5.919464147614116045e-02 5.583482432664836903e-01 3.845711711715178033e-02 8.668551435002927783e-06 5.440519998929155855e-13 0.000000000000000000e+00 +7.600000000000000089e-01 4.908113325887875078e-02 5.633306385784786485e-01 4.920917355332636578e-02 1.590831770127603740e-05 1.640098818002154181e-12 0.000000000000000000e+00 +7.700000000000000178e-01 4.038972126963992493e-02 5.646304145716388723e-01 6.199135756487455379e-02 2.847609961672602441e-05 4.771559347140737470e-12 0.000000000000000000e+00 +7.800000000000000266e-01 3.299387779642509039e-02 5.622419392805708682e-01 7.693066766536615919e-02 4.975605239389383909e-05 1.340984640895080407e-11 0.000000000000000000e+00 +7.900000000000000355e-01 2.675995522130884818e-02 5.562144640312042254e-01 9.410473960353642198e-02 8.492613821850217477e-05 3.644076367341346881e-11 0.000000000000000000e+00 +8.000000000000000444e-01 2.155315592808926956e-02 5.466547389784724986e-01 1.135325149284951984e-01 1.417027899099369457e-04 9.584228925483725884e-11 0.000000000000000000e+00 +8.100000000000000533e-01 1.724225413625518621e-02 5.337283120040172690e-01 1.351672682569173556e-01 2.312914228756922049e-04 2.441905895621031500e-10 0.000000000000000000e+00 +8.200000000000000622e-01 1.370310631446661383e-02 5.176589189564108073e-01 1.588925367334704242e-01 3.695546593339714457e-04 6.032339399339098914e-10 0.000000000000000000e+00 +8.300000000000000711e-01 1.082105440739568325e-02 4.987255519052841701e-01 1.845212680151044615e-01 5.783910087925287023e-04 1.446104618044234327e-09 0.000000000000000000e+00 +8.399999999999999689e-01 8.492368483615756955e-03 4.772570262102269756e-01 2.117982528241139084e-01 8.872905102245751520e-04 3.366910160836069286e-09 0.000000000000000000e+00 +8.499999999999999778e-01 6.624896028738718780e-03 4.536241228125140745e-01 2.404056641564599750e-01 1.335001538209432690e-03 7.619585293097197538e-09 0.000000000000000000e+00 +8.599999999999999867e-01 5.138088405274474427e-03 4.282296256229109477e-01 2.699713143923895275e-01 1.971206986286263250e-03 1.677408889534854048e-08 0.000000000000000000e+00 +8.699999999999999956e-01 3.962565662767755965e-03 4.014967764109840798e-01 3.000790842252494817e-01 2.858072067107956474e-03 3.594878393675884972e-08 0.000000000000000000e+00 +8.800000000000000044e-01 3.039363303081291188e-03 3.738568104795802483e-01 3.302808850321978240e-01 4.071494783317450121e-03 7.505634360454145076e-08 0.000000000000000000e+00 +8.900000000000000133e-01 2.318982557617298070e-03 3.457363046782563343e-01 3.601094908753040835e-01 5.701869059379470125e-03 1.527775598766901995e-07 0.000000000000000000e+00 +9.000000000000000222e-01 1.760342204033970084e-03 3.175450639159804189e-01 3.890916106036985123e-01 7.854164751648755674e-03 3.033910842837618412e-07 0.000000000000000000e+00 +9.100000000000000311e-01 1.329707117851893893e-03 2.896652009065096012e-01 4.167606505186060573e-01 1.064714242860430662e-02 5.881805399220220047e-07 3.987939362915592498e-17 +9.200000000000000400e-01 9.996480514146239768e-04 2.624419405635543545e-01 4.426687258288088955e-01 1.421155625060968597e-02 1.113956680258438677e-06 1.599438325452657675e-16 +9.300000000000000488e-01 7.480693616013980992e-04 2.361765232463523267e-01 4.663975960230017925e-01 1.868725529415628597e-02 2.062305440089852803e-06 4.809562789568694136e-16 +9.400000000000000577e-01 5.573268799382214865e-04 2.111214090595887882e-01 4.875683089034889561e-01 2.421916922979048745e-02 3.734515060670443504e-06 1.526104689638597213e-15 +9.500000000000000666e-01 4.134467924763931949e-04 1.874778164993770035e-01 5.058494283747770881e-01 3.095225256719253207e-02 6.618720710238419233e-06 4.706782123258309453e-15 +9.599999999999999645e-01 3.054480002419670843e-04 1.653954777146330324e-01 5.209637857912046144e-01 3.902555464459427775e-02 1.148754977200705858e-05 1.409958892945934097e-14 +9.699999999999999734e-01 2.247645639621065523e-04 1.449743701667425277e-01 5.326937331039534040e-01 4.856567071110100275e-02 1.953629700688028575e-05 4.097287642725277212e-14 +9.799999999999999822e-01 1.647610477458523871e-04 1.262680966095084123e-01 5.408848924670576208e-01 5.967990317940288780e-02 3.257316063442919438e-05 1.156264860980133771e-13 +9.899999999999999911e-01 1.203314172321944019e-04 1.092885338187701977e-01 5.454483989827731971e-01 7.244951283445183465e-02 5.327399336475601528e-05 3.169966378707302841e-13 +1.000000000000000000e+00 8.757120797261259363e-05 9.401135343377249598e-02 5.463616299897278150e-01 8.692346113349824765e-02 8.551395665096365236e-05 8.445631829787452869e-13 +1.010000000000000009e+00 6.351260558210720388e-05 8.038203093412973899e-02 5.436674144371487216e-01 1.031130335113659702e-01 1.347869605537307329e-04 2.188292442234903755e-12 +1.020000000000000018e+00 4.591258022744503978e-05 6.832199474986185017e-02 5.374717262206131219e-01 1.209876901816309969e-01 2.087203879984701250e-04 5.517788499267541666e-12 +1.030000000000000027e+00 3.308506691439644051e-05 5.773461963661813412e-02 5.279398896906133398e-01 1.404724192159305385e-01 3.176869911565202687e-04 1.354907999599729549e-11 +1.040000000000000036e+00 2.376920947954347198e-05 4.851082969953900148e-02 5.152913643308603131e-01 1.614467736341936299e-01 4.755078334668812450e-04 3.242073798140482060e-11 +1.050000000000000044e+00 1.702676875958501860e-05 4.053414051274251140e-02 4.997932261228877904e-01 1.837456689052915371e-01 7.002298178478492518e-04 7.564456142563883764e-11 +1.060000000000000053e+00 1.216285246587368811e-05 3.368503150020914910e-02 4.817525201847516758e-01 2.071619096456787934e-01 1.014949105503201027e-03 1.722036485676948522e-10 +1.070000000000000062e+00 8.665105188461925210e-06 2.784459530287804185e-02 4.615077162574321989e-01 2.314503141294012023e-01 1.448638376333029982e-03 3.827163644781810072e-10 +1.080000000000000071e+00 6.157374138353131114e-06 2.289745779829197372e-02 4.394195485514988397e-01 2.563332207838229237e-01 2.036922560021247729e-03 8.308747909735498940e-10 +1.090000000000000080e+00 4.364628584036829025e-06 1.873399954403896436e-02 4.158615581335344813e-01 2.815070981489081747e-01 2.822735823495265729e-03 1.763051501130930938e-09 +1.100000000000000089e+00 3.086558380169819119e-06 1.525193649955695376e-02 3.912106747804782181e-01 3.066499420314597324e-01 3.856783857886315284e-03 3.658523730880318455e-09 +1.110000000000000098e+00 2.177816382631798547e-06 1.235733554530114506e-02 3.658381735243608768e-01 3.314291310907604426e-01 5.197731489308821465e-03 7.428353216209023155e-09 +1.120000000000000107e+00 1.533311608890686663e-06 9.965149701945600139e-03 3.401013187109006575e-01 3.555094225198551228e-01 6.912037758511026773e-03 1.476567456409793133e-08 +1.130000000000000115e+00 1.077317174939318544e-06 7.999360496998732598e-03 3.143359672256667015e-01 3.785607984628082123e-01 9.073369868121048537e-03 2.874819218854707111e-08 +1.140000000000000124e+00 7.554420044525390091e-07 6.392812182772026133e-03 2.888503462911807262e-01 4.002659156940988616e-01 1.176154416230164659e-02 5.485055610683904830e-08 +1.150000000000000133e+00 5.287401273633696096e-07 5.086815988022990419e-03 2.639201548017506838e-01 4.203269598298132181e-01 1.506096621093475953e-02 1.026067721427023607e-07 +1.159999999999999920e+00 3.694069600603872627e-07 4.030593650930917524e-03 2.397850660033362113e-01 4.384717552720807654e-01 1.905857194120165812e-02 1.882788031126817605e-07 +1.169999999999999929e+00 2.576478256172417289e-07 3.180619291396995056e-03 2.166466388580143010e-01 4.544590285607395530e-01 2.384130565034060789e-02 3.390455127107025517e-07 +1.179999999999999938e+00 1.794082257805426176e-07 2.499908159783769887e-03 1.946675804952582300e-01 4.680827626086341287e-01 2.949320607678780973e-02 5.994341707921660711e-07 +1.189999999999999947e+00 1.247347553751315127e-07 1.957290636282447288e-03 1.739722466174895843e-01 4.791756108173245710e-01 3.609220554043009871e-02 1.040981052976354072e-06 +1.199999999999999956e+00 8.659556137888371261e-08 1.526700523074696380e-03 1.546482232702307003e-01 4.876113632164771339e-01 4.370677642250402062e-02 1.776430424561485100e-06 +1.209999999999999964e+00 6.003431641129191042e-08 1.186498456821617666e-03 1.367488034117998463e-01 4.933064726950170398e-01 5.239258109886594378e-02 2.980158058868660208e-06 +1.219999999999999973e+00 4.156531527405939585e-08 9.188443069215798805e-04 1.202961554213368978e-01 4.962206600834439540e-01 6.218929355632565187e-02 4.916922652659156695e-06 +1.229999999999999982e+00 2.874226537364183563e-08 7.091267350689117992e-04 1.052849772272313883e-01 4.963566246648489222e-01 7.311776179357656891e-02 7.981480093998342935e-06 +1.239999999999999991e+00 1.985176979539427896e-08 5.454536114729735975e-04 9.168643748116768466e-02 4.937588939219547668e-01 8.517766919146980975e-02 1.275200101938270604e-05 +1.250000000000000000e+00 1.369603324259006201e-08 4.182035971692599718e-04 7.945222197468297665e-02 4.885118548082269530e-01 9.834583083402068104e-02 2.006055534552998338e-05 +1.260000000000000009e+00 9.439214314960034364e-09 3.196367654093523682e-04 6.851852685803538900e-02 4.807370197047131088e-01 1.125752287815075109e-01 3.108409740326714259e-05 +1.270000000000000018e+00 6.499046789844780571e-09 2.435604924737936846e-04 5.880986771019022091e-02 4.705895938211596130e-01 1.277948509127032062e-01 4.745931905434854099e-05 +1.280000000000000027e+00 4.470568134183769465e-09 1.850458460820321276e-04 5.024260282436234654e-02 4.582544266593162052e-01 1.439103541662635211e-01 7.142434700938892834e-05 +1.290000000000000036e+00 3.072562195846065087e-09 1.401891957253903313e-04 4.272809822220174913e-02 4.439414471605867551e-01 1.608055281600602460e-01 1.059894588790527889e-04 +1.300000000000000044e+00 2.110029059468151354e-09 1.059136380165031282e-04 3.617548928941841102e-02 4.278806987493339320e-01 1.783444926099220507e-01 1.551377093322584737e-04 +1.310000000000000053e+00 1.447938509207935114e-09 7.980496463201170300e-05 3.049401836180056641e-02 4.103171049105429069e-01 1.963745247767608171e-01 2.240545501267006835e-04 +1.320000000000000062e+00 9.929082461189637924e-10 5.997721296292581636e-05 2.559494832978258944e-02 3.915051065295243049e-01 2.147293838621967266e-01 3.193831943909459549e-04 +1.330000000000000071e+00 6.804373241412314845e-10 4.496326070311629057e-05 2.139306900207567017e-02 3.717033175897766495e-01 2.332329796163886138e-01 4.494996689140518214e-04 +1.340000000000000080e+00 4.660249166082903513e-10 3.362640219784691930e-05 1.780782552932241164e-02 3.511693450524774884e-01 2.517032233212468939e-01 6.247983335022933580e-04 +1.350000000000000089e+00 3.190020323059575847e-10 2.508933740362713548e-05 1.476410684746647280e-02 3.301549114641024740e-01 2.699559007785266584e-01 8.579753047108228418e-04 +1.360000000000000098e+00 2.182537620060257098e-10 1.867748643680915176e-05 1.219273725074603715e-02 3.089014052730736370e-01 2.878084181409205633e-01 1.164293945402312264e-03 +1.370000000000000107e+00 1.492567491773983311e-10 1.387399592588429562e-05 1.003071635392241250e-02 2.876359647305094525e-01 3.050832908439197744e-01 1.561813664434939146e-03 +1.380000000000000115e+00 1.020304123234632149e-10 1.028421700879462491e-05 8.221252415421138535e-03 2.665681777852352274e-01 3.216112714711567566e-01 2.071561034744768669e-03 +1.390000000000000124e+00 6.972163412658023648e-11 7.607803539589589622e-06 6.713631836315738302e-03 2.458874540373246487e-01 3.372340417499066212e-01 2.717621007763046641e-03 +1.400000000000000133e+00 4.762837548229303302e-11 5.616901495429025980e-06 5.462964162213741412e-03 2.257610972106653402e-01 3.518064245725837647e-01 3.527126071955804382e-03 +1.409999999999999920e+00 3.252686654653270964e-11 4.139177976849373141e-06 4.429837577211066107e-03 2.063330793613646352e-01 3.651981016438504146e-01 4.530122803442502774e-03 +1.419999999999999929e+00 2.220820359598883994e-11 3.044673545841167993e-06 3.579915100585794388e-03 1.877234926355634825e-01 3.772948490350265804e-01 5.759298520800725830e-03 +1.429999999999999938e+00 1.515991523648943325e-11 2.235658889103976142e-06 2.883496809241144730e-03 1.700286320654501859e-01 3.879993250034428631e-01 7.249555695220221210e-03 +1.439999999999999947e+00 1.034683098626099816e-11 1.638840311522065044e-06 2.315068665998265638e-03 1.533216445716258092e-01 3.972314608487446108e-01 9.037428252682576379e-03 +1.449999999999999956e+00 7.060922971419358754e-12 1.199392806151284170e-06 1.852854117469739364e-03 1.376536656025731664e-01 4.049285158375934279e-01 1.116034166477316084e-02 +1.459999999999999964e+00 4.818095350094647536e-12 8.764086843136990109e-07 1.478380653643590918e-03 1.230553559195996793e-01 4.110448613842956900e-01 1.365572735324401508e-02 +1.469999999999999973e+00 3.287475050906292697e-12 6.394379240018318150e-07 1.176070059350544858e-03 1.095387468405975423e-01 4.155515582622911763e-01 1.656001091470735501e-02 +1.479999999999999982e+00 2.243028356765281869e-12 4.658670434431361876e-07 9.328581577592964628e-04 9.709930243004447070e-02 4.184357845621698435e-01 1.990750241626376357e-02 +1.489999999999999991e+00 1.530410370860696685e-12 3.389395278792606130e-07 7.378474379748105241e-04 8.571811109958503427e-02 4.197001625968400651e-01 2.372922490270214929e-02 +1.500000000000000000e+00 1.044227620301937164e-12 2.462652922192765168e-07 5.819940399045956444e-04 7.536412615730746223e-02 4.193620213150490716e-01 2.805172369587968437e-02 +1.510000000000000009e+00 7.125319208727499156e-13 1.787016081457706795e-07 4.578290922701090714e-04 6.599638423629203277e-02 4.174526183660571266e-01 3.289590353503257791e-02 +1.520000000000000018e+00 4.862454061953004069e-13 1.295152431329936622e-07 3.592133083616361297e-04 5.756614145515785014e-02 4.140163340064137443e-01 3.827594270030442586e-02 +1.530000000000000027e+00 3.318641881800761590e-13 9.375680550089078047e-08 2.811229803902905123e-04 5.001887886674720451e-02 4.091098386070023962e-01 4.419833273351361780e-02 +1.540000000000000036e+00 2.265235929570130444e-13 6.779473330279948552e-08 2.194650195892666104e-04 4.329614056732959682e-02 4.028012273977749635e-01 5.066108915181868205e-02 +1.550000000000000044e+00 1.546572594408821060e-13 4.896903398857068421e-08 1.709184117419885595e-04 3.733717920593891682e-02 3.951691107749291110e-01 5.765317276005898356e-02 +1.560000000000000053e+00 1.055972136085097858e-13 3.533460021916003750e-08 1.327993480889430187e-04 3.208039411088316195e-02 3.863016461815973468e-01 6.515415310369479518e-02 +1.570000000000000062e+00 7.212787436823881470e-14 2.547135196869689815e-08 1.029473073417947572e-04 2.746455652191652150e-02 3.762954981625536433e-01 7.313413572006240149e-02 +1.580000000000000071e+00 4.926561120850808183e-14 1.834410667434065353e-08 7.962947006486899347e-05 2.342982428523900443e-02 3.652547163437112610e-01 8.155396371542289469e-02 +1.590000000000000080e+00 3.366389966685696226e-14 1.319936283749637620e-08 6.146101257826301601e-05 1.991855474509214227e-02 3.532895262658639624e-01 9.036569246971370983e-02 +1.600000000000000089e+00 2.300688167185030099e-14 9.489422346087647066e-09 4.733903027706718483e-05 1.687592945220024732e-02 3.405150345524147748e-01 9.951332462804229895e-02 +1.610000000000000098e+00 1.574686925727820699e-14 6.816714197451711120e-09 3.638806161764795288e-05 1.425040778032774305e-02 3.270498570999844645e-01 1.089337816339119580e-01 +1.620000000000000107e+00 1.076088606378924714e-14 4.893014710945852010e-09 2.791541030034892261e-05 1.199402872609029759e-02 3.130146861450396889e-01 1.185580784828221823e-01 +1.630000000000000115e+00 7.373320916393947496e-15 3.509629230139760943e-09 2.137468452168460057e-05 1.006258122764231919e-02 2.985308185410810999e-01 1.283126606102086287e-01 +1.640000000000000124e+00 5.039707156606339832e-15 2.515625287883645411e-09 1.633618184121517184e-05 8.415663457889582463e-03 2.837186728516735146e-01 1.381208562261167228e-01 +1.650000000000000133e+00 3.456878447189422070e-15 1.801967357392948972e-09 1.246294195874585876e-05 7.016650915740045569e-03 2.686963265375161281e-01 1.479043941736273293e-01 +1.660000000000000142e+00 2.361140339769211921e-15 1.289971153793733803e-09 9.491465192571795135e-06 5.832591937290134212e-03 2.535781063595277107e-01 1.575849365697442606e-01 +1.669999999999999929e+00 1.617506550254850546e-15 9.229126500328865171e-10 7.216250793272702641e-06 4.834047646692864360e-03 2.384732650603055559e-01 1.670855769923690337e-01 +1.679999999999999938e+00 1.120818448326204615e-15 6.599389111110727231e-10 5.477446552559663205e-06 3.994891513508192112e-03 2.234847754947852649e-01 1.763322585837637235e-01 +1.689999999999999947e+00 7.718317718684845388e-16 4.716545757365531850e-10 4.151020319190942675e-06 3.292081707036350561e-03 2.087082698534345193e-01 1.852550718252251039e-01 +1.699999999999999956e+00 5.205557371795253563e-16 3.369269462820051816e-10 3.140966303118730650e-06 2.705417442489946919e-03 1.942311467503382849e-01 1.937893985018218634e-01 +1.709999999999999964e+00 3.625923350094396720e-16 2.405762177237260559e-10 2.373145988632483469e-06 2.217288579870357445e-03 1.801318630912998042e-01 2.018768760801259443e-01 +1.719999999999999973e+00 2.516385783948776405e-16 1.717071357389781900e-10 1.790436743033715327e-06 1.812425923627207806e-03 1.664794211831775173e-01 2.094661649151489391e-01 +1.729999999999999982e+00 1.649016793794969713e-16 1.225055812872878406e-10 1.348922463496005526e-06 1.477658020001481991e-03 1.533330548865708964e-01 2.165135089515941580e-01 +1.739999999999999991e+00 1.215454052057018530e-16 8.737113186617575940e-11 1.014911453478866216e-06 1.201678783392800143e-03 1.407421121164291467e-01 2.229830884966831628e-01 +1.750000000000000000e+00 7.961828697422301127e-17 6.229269087776454313e-11 7.626086451560616175e-07 9.748290136706607139e-04 1.287461249800961882e-01 2.288471708842831631e-01 +1.760000000000000009e+00 5.866150746364105615e-17 4.439923292318511293e-11 5.723036423002075206e-07 7.888937929441010430e-04 1.173750535697218461e-01 2.340860711616196688e-01 +1.770000000000000018e+00 3.841093788301158207e-17 3.163695772718413247e-11 4.289640513595782200e-07 6.369168648361575071e-04 1.066496850861715084e-01 2.386879401270684264e-01 +1.780000000000000027e+00 1.885957619321147712e-17 2.253753302325919247e-11 3.211462555028557282e-07 5.130323880938954164e-04 9.658216668239595626e-02 2.426484010251678258e-01 +1.790000000000000036e+00 1.851625859457894989e-17 1.605170754312330251e-11 2.401540797277149262e-07 4.123139021731565281e-04 8.717664822342960362e-02 2.459700589316318187e-01 +1.800000000000000044e+00 1.817561311184611855e-17 1.143011599325978046e-11 1.793904769781704556e-07 3.306399261469550560e-04 7.843001005207206333e-02 2.486619083708357936e-01 +1.810000000000000053e+00 0.000000000000000000e+00 8.137769864236968103e-12 1.338590958179008190e-07 2.645753142235484604e-04 7.033265075436209712e-02 2.507386650894823377e-01 +1.820000000000000062e+00 0.000000000000000000e+00 5.792888185064208087e-12 9.978192334122564621e-08 2.112672920838356422e-04 6.286931072545985655e-02 2.522200472933634474e-01 +1.830000000000000071e+00 0.000000000000000000e+00 4.123163504158291035e-12 7.430659412046685338e-08 1.683549800624304976e-04 5.601990890292645603e-02 2.531300301985314749e-01 +1.840000000000000080e+00 0.000000000000000000e+00 2.934404509437793404e-12 5.528279528550251697e-08 1.338911554219225338e-04 4.976037220137410205e-02 2.534960956291138490e-01 +1.850000000000000089e+00 0.000000000000000000e+00 2.088196987818810075e-12 4.109179192465435618e-08 1.062750019901083088e-04 4.406343978443494347e-02 2.533484957914069469e-01 +1.860000000000000098e+00 0.000000000000000000e+00 1.485930496827888161e-12 3.051669906914862113e-08 8.419462861343407956e-05 3.889942718470752042e-02 2.527195474423744037e-01 +1.870000000000000107e+00 0.000000000000000000e+00 1.057331050114870163e-12 2.264394104089137764e-08 6.657819755548245592e-05 3.423693827886707675e-02 2.516429696116462833e-01 +1.880000000000000115e+00 0.000000000000000000e+00 7.523348095324758579e-13 1.678853196886138342e-08 5.255258132893701560e-05 3.004351610978756720e-02 2.501532749718995974e-01 +1.890000000000000124e+00 0.000000000000000000e+00 5.353109911981471400e-13 1.243751461026292355e-08 4.140855440294176831e-05 2.628622640779162967e-02 2.482852220025697776e-01 +1.900000000000000133e+00 0.000000000000000000e+00 3.809073234244706527e-13 9.207214654667112586e-09 3.257161919934678870e-05 2.293217030971014814e-02 2.460733323509897985e-01 +1.910000000000000142e+00 0.000000000000000000e+00 2.710392715305241659e-13 6.810986302454714229e-09 2.557765954269661335e-05 1.994892514159541178e-02 2.435514753331225191e-01 +1.919999999999999929e+00 0.000000000000000000e+00 1.928876693263395471e-13 5.034911002652640679e-09 2.005270613143748552e-05 1.730491417568628953e-02 2.407525193791170692e-01 +1.929999999999999938e+00 0.000000000000000000e+00 1.372712034894855376e-13 3.719515090093346339e-09 1.569618546753373967e-05 1.496970797228164285e-02 2.377080484413994221e-01 +1.939999999999999947e+00 0.000000000000000000e+00 9.770355550497070407e-14 2.746034365719055889e-09 1.226710458016934130e-05 1.291426126829763044e-02 2.344481399505382035e-01 +1.949999999999999956e+00 0.000000000000000000e+00 6.954095794421101587e-14 2.026109074851652458e-09 9.572697970727701990e-06 1.111109038716246757e-02 2.310011998168181035e-01 +1.959999999999999964e+00 0.000000000000000000e+00 4.950628795425162859e-14 1.494062259318707004e-09 7.459130114271636743e-06 9.534396841737206515e-03 2.273938492112568543e-01 +1.969999999999999973e+00 0.000000000000000000e+00 3.524659622859073820e-14 1.101121192966155883e-09 5.803906544438261252e-06 8.160143214006465237e-03 2.236508573878721873e-01 +1.979999999999999982e+00 0.000000000000000000e+00 2.510052338465078198e-14 8.110980483030642230e-10 4.509699239154103763e-06 6.966087558734339324e-03 2.197951145927241889e-01 +1.989999999999999991e+00 0.000000000000000000e+00 1.787618953549825793e-14 5.971650809132615970e-10 3.499338098546060994e-06 5.931782532330697778e-03 2.158476391050443144e-01 +2.000000000000000000e+00 0.000000000000000000e+00 1.273688180077510876e-14 4.394495211068296458e-10 2.711760254268619611e-06 5.038545232619322418e-03 2.118276126313043406e-01 +2.010000000000000231e+00 0.000000000000000000e+00 9.074433345977802960e-15 3.232418849680934038e-10 2.098743320960148623e-06 4.269403388610665089e-03 2.077524385852534128e-01 +2.020000000000000018e+00 0.000000000000000000e+00 6.465585659239437314e-15 2.376624233325534827e-10 1.622278067577374789e-06 3.609023097719445215e-03 2.036378181992327119e-01 +2.030000000000000249e+00 0.000000000000000000e+00 4.609785513617459268e-15 1.746697314412375089e-10 1.252460916077393156e-06 3.043622803520505942e-03 1.994978398914793449e-01 +2.040000000000000036e+00 0.000000000000000000e+00 3.284826272623138873e-15 1.283242761155044399e-10 9.658077117091639874e-07 2.560877668325015153e-03 1.953450778319217873e-01 +2.049999999999999822e+00 0.000000000000000000e+00 2.339705888585772235e-15 9.424179696584112349e-11 7.439078636058203301e-07 2.149817945345193666e-03 1.911906961808303718e-01 +2.060000000000000053e+00 0.000000000000000000e+00 1.666805060701360910e-15 6.918805073074525467e-11 5.723526929591603475e-07 1.800724414306093413e-03 1.870445560008906216e-01 +2.069999999999999840e+00 0.000000000000000000e+00 1.187539574586914804e-15 5.077857796472893261e-11 4.398840709930952418e-07 1.505023427061821968e-03 1.829153223483407364e-01 +2.080000000000000071e+00 0.000000000000000000e+00 8.496324603286173785e-16 3.725638338540987369e-11 3.377195572034839594e-07 1.255183627141797037e-03 1.788105695213538238e-01 +2.089999999999999858e+00 0.000000000000000000e+00 6.049865602427579646e-16 2.732752590375000800e-11 2.590185875063445521e-07 1.044615966719344155e-03 1.747368828759742199e-01 +2.100000000000000089e+00 0.000000000000000000e+00 4.381004284893701990e-16 2.003953265642662089e-11 1.984611026675992799e-07 8.675782507414362202e-04 1.706999560068622190e-01 +2.109999999999999876e+00 0.000000000000000000e+00 3.066985189116692570e-16 1.469167691858605536e-11 1.519155959001551035e-07 7.190850928289851170e-04 1.667046824296924135e-01 +2.120000000000000107e+00 0.000000000000000000e+00 2.176261587658125932e-16 1.076859572356316944e-11 1.161781079219726925e-07 5.948238709419178356e-04 1.627552411941456667e-01 +2.129999999999999893e+00 0.000000000000000000e+00 1.592189798993372198e-16 7.891476446737450005e-12 8.876738851909710868e-08 4.910770210104920940e-04 1.588551761025905895e-01 +2.140000000000000124e+00 0.000000000000000000e+00 1.121524223927143901e-16 5.781992255128146365e-12 6.776442766957509553e-08 4.046508009077945086e-04 1.550074684124292113e-01 +2.149999999999999911e+00 0.000000000000000000e+00 8.412574261988304456e-17 4.235680665465538987e-12 5.168696472056352329e-08 3.328104916289441047e-04 1.512146030632391669e-01 +2.160000000000000142e+00 0.000000000000000000e+00 5.741314252104399250e-17 3.102450386164660077e-12 3.939151671961303739e-08 2.732218732214534030e-04 1.474786285972695399e-01 +2.169999999999999929e+00 0.000000000000000000e+00 3.997500656956318079e-17 2.272099423400832132e-12 2.999701609687260286e-08 2.238987155308227062e-04 1.438012110378352193e-01 +2.180000000000000160e+00 0.000000000000000000e+00 3.116770415974760792e-17 1.663786591529606892e-12 2.282538529447591215e-08 1.831559538207494378e-04 1.401836820589633104e-01 +2.189999999999999947e+00 0.000000000000000000e+00 2.277789079928866168e-17 1.218207155727556169e-12 1.735536244938146358e-08 1.495681725784303486e-04 1.366270818254572417e-01 +2.200000000000000178e+00 0.000000000000000000e+00 1.479418177854265644e-17 8.918894397921118508e-13 1.318667633675012942e-08 1.219329933526162521e-04 1.331321969092713331e-01 +2.209999999999999964e+00 0.000000000000000000e+00 1.441056636940837695e-17 6.529283516315241764e-13 1.001229077707459566e-08 9.923895066120309329e-05 1.296995936993258214e-01 +2.220000000000000195e+00 0.000000000000000000e+00 7.017180878524692740e-18 4.779682583598309175e-13 7.596930531637991973e-09 8.063744058267207813e-05 1.263296477208695201e-01 +2.229999999999999982e+00 0.000000000000000000e+00 6.832756214583432278e-18 3.498712819677446630e-13 5.760489451858984676e-09 6.541833674581619830e-05 1.230225692700604029e-01 +2.240000000000000213e+00 0.000000000000000000e+00 6.651980022536628490e-18 2.560945788876376436e-13 4.365227426665486635e-09 5.298888558314323672e-05 1.197784257520210921e-01 +2.250000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.874590403134975516e-13 3.305903318390574361e-09 4.285551482304234656e-05 1.165971610883288351e-01 +2.260000000000000231e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.372097122751739994e-13 2.502179923832708894e-09 3.460821452940601739e-05 1.134786125344207919e-01 +2.270000000000000018e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.004356911031548712e-13 1.892784486300859568e-09 2.790717714340951842e-05 1.104225252201533475e-01 +2.280000000000000249e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.351530704690434846e-14 1.431024237201164453e-09 2.247141081932049278e-05 1.074285646988426962e-01 +2.290000000000000036e+00 0.000000000000000000e+00 0.000000000000000000e+00 5.381670415028282227e-14 1.081346184203956386e-09 1.806906800624292621e-05 1.044963277624186559e-01 +2.300000000000000266e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.939346156416903352e-14 8.167008950539859823e-10 1.450925806032794156e-05 1.016253517534581041e-01 +2.310000000000000053e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.883983432968017991e-14 6.165243236779424460e-10 1.163513820918934384e-05 9.881512257932514864e-02 +2.319999999999999840e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.111414100763760632e-14 4.651944504886763807e-10 9.318101116486660250e-06 9.606508160971728716e-02 +2.330000000000000071e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.545793365940586551e-14 3.508520326818904922e-10 7.452899416447807511e-06 9.337463161680942259e-02 +2.339999999999999858e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.131945748820939697e-14 2.645005037302947090e-10 5.953567809983981724e-06 9.074314189698128030e-02 +2.350000000000000089e+00 0.000000000000000000e+00 0.000000000000000000e+00 8.291586680549370729e-15 1.993194796579811860e-10 4.750021615578641666e-06 8.816995269483543507e-02 +2.359999999999999876e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.072113698125966354e-15 1.501416795207547126e-10 3.785227085942417415e-06 8.565437903382781826e-02 +2.370000000000000107e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.447574203944711422e-15 1.130546621776036549e-10 3.012853412633119830e-06 8.319571404323970742e-02 +2.379999999999999893e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.256152314445057105e-15 8.509784987104835391e-11 2.395329251639868255e-06 8.079323185835433907e-02 +2.390000000000000124e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.386399670585543391e-15 6.403222869540918126e-11 1.902237935776897606e-06 7.844619015938820994e-02 +2.399999999999999911e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.750172859952802725e-15 4.816544195615067664e-11 1.508995428454159384e-06 7.615383240488161898e-02 +2.410000000000000142e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.280196765776664886e-15 3.621897812579862607e-11 1.195763653279049312e-06 7.391538980669956627e-02 +2.419999999999999929e+00 0.000000000000000000e+00 0.000000000000000000e+00 9.397218005304305507e-16 2.722745564797629526e-11 9.465592401268478653e-07 7.173008308642601916e-02 +2.430000000000000160e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.876955294791007758e-16 2.046231480086412083e-11 7.485240902358236558e-07 6.959712404660257923e-02 +2.439999999999999947e+00 0.000000000000000000e+00 0.000000000000000000e+00 5.059675803069522445e-16 1.537395235649882171e-11 5.913296031096073117e-07 6.751571698485774042e-02 +2.450000000000000178e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.707880514319666554e-16 1.154797211605313668e-11 4.666910401398046071e-07 6.548505997436984694e-02 +2.459999999999999964e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.714419233202488701e-16 8.672040667114443861e-12 3.679724282153279499e-07 6.350434603021394564e-02 +2.470000000000000195e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.982434846783914486e-16 6.510859497425683577e-12 2.898657250745560432e-07 6.157276417784886269e-02 +2.479999999999999982e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.457916041895565560e-16 4.887226155642315448e-12 2.281307618698030984e-07 5.968950043724473298e-02 +2.490000000000000213e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.059807407408892020e-16 3.667749360184711401e-12 1.793848216311360020e-07 5.785373873382781973e-02 +2.500000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.780554350927864103e-17 2.752041206136252714e-12 1.409326717600859481e-07 5.606466174549552123e-02 +2.510000000000000231e+00 0.000000000000000000e+00 0.000000000000000000e+00 5.729359676538773614e-17 2.064592852155514961e-12 1.106295018367351923e-07 5.432145169334207102e-02 +2.520000000000000018e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.381776429573893385e-17 1.548610346924291867e-12 8.677057519319773001e-08 5.262329108240897807e-02 +2.530000000000000249e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.112308996161187046e-17 1.161411859982025926e-12 6.800252739898963702e-08 5.096936339766884594e-02 +2.540000000000000036e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.191975801428646355e-17 8.709021255748708052e-13 5.325217421360005887e-08 4.935885375955084697e-02 +2.550000000000000266e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.591759764419218979e-17 6.529743434929926783e-13 4.166945754729868309e-08 4.779094954256773997e-02 +2.560000000000000053e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.284108195950185917e-17 4.895225900273460528e-13 3.258178784067368374e-08 4.626484095999987972e-02 +2.569999999999999840e+00 0.000000000000000000e+00 0.000000000000000000e+00 9.943101247914857710e-18 3.669451800036738461e-13 2.545755778150753704e-08 4.477972161709173338e-02 +2.580000000000000071e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.216695393294470206e-18 2.750378836989766458e-13 1.987702485011119497e-08 4.333478903481657180e-02 +2.589999999999999858e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.655083058496595875e-18 2.061317329132877640e-13 1.550910544222397100e-08 4.192924514593637797e-02 +2.600000000000000089e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.503320922597199382e-18 1.544774176078517278e-13 1.209290411957089347e-08 4.056229676481840768e-02 +2.609999999999999876e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.177878180876129440e-18 1.157607589481089058e-13 9.423030381865093257e-09 3.923315603226213560e-02 +2.620000000000000107e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.106151263083867790e-18 8.674183977010909617e-14 7.337940480433001405e-09 3.794104083641439834e-02 +2.629999999999999893e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.036436626078160077e-18 6.499491135791056170e-14 5.710692101569937623e-09 3.668517521071983090e-02 +2.640000000000000124e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.968691551524334159e-18 4.869755421850592770e-14 4.441620975357477385e-09 3.546478970974006883e-02 +2.649999999999999911e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.648760327325065136e-14 3.452546424430209179e-09 3.427912176359525914e-02 +2.660000000000000142e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.733769912132806097e-14 2.682191632828153098e-09 3.312741601170805794e-02 +2.669999999999999929e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.048177727754681947e-14 2.082567857831818741e-09 3.200892461648179033e-02 +2.680000000000000160e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.534610261157525913e-14 1.616122635238265706e-09 3.092290755749948356e-02 +2.689999999999999947e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.149686508592240903e-14 1.253492815999396109e-09 2.986863290680086616e-02 +2.700000000000000178e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 8.614671972962280436e-15 9.717358808334643381e-10 2.884537708576626752e-02 +2.709999999999999964e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.455051934820109232e-15 7.529390734337769394e-10 2.785242510412086481e-02 +2.720000000000000195e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.836164314657621046e-15 5.831266290921146968e-10 2.688907078155316696e-02 +2.729999999999999982e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.623538290012886062e-15 4.514019957649875984e-10 2.595461695243495015e-02 +2.740000000000000213e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.715576559987804673e-15 3.492751169944023655e-10 2.504837565411618949e-02 +2.750000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.035337854910090020e-15 2.701352699961908659e-10 2.416966829926584653e-02 +2.760000000000000231e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.524802455241534417e-15 2.088383228052072319e-10 2.331782583272005646e-02 +2.770000000000000018e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.143687653236109857e-15 1.613838394573600148e-10 2.249218887329790598e-02 +2.780000000000000249e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 8.573575666959108612e-16 1.246626669212186277e-10 2.169210784103860895e-02 +2.790000000000000036e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.421013482646241153e-16 9.625976207138332541e-11 2.091694307031213840e-02 +2.800000000000000266e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.813598377077672525e-16 7.430029327161647599e-11 2.016606490925106770e-02 +2.810000000000000053e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.614895074498864502e-16 5.732960078490365620e-11 1.943885380594947629e-02 +2.819999999999999840e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.703960450424941502e-16 4.421963321993972999e-11 1.873470038186982739e-02 +2.830000000000000071e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.034350533574988660e-16 3.409606368852256177e-11 1.805300549289740389e-02 +2.839999999999999858e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.525514430418198638e-16 2.628156646930657304e-11 1.739318027847704812e-02 +2.850000000000000089e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.143985731144815808e-16 2.025167991601323594e-11 1.675464619926350998e-02 +2.859999999999999876e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 8.599433207823943862e-17 1.560049693150399015e-11 1.613683506371332496e-02 +2.870000000000000107e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.469487052763263171e-17 1.201401342702921619e-11 1.553918904404079197e-02 +2.879999999999999893e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.816965271603458441e-17 9.249424595513707806e-12 1.496116068195739099e-02 +2.890000000000000124e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.597675380538273801e-17 7.119062454373942420e-12 1.440221288460728083e-02 +2.899999999999999911e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.693199341766289624e-17 5.477933603789479502e-12 1.386181891110846963e-02 +2.910000000000000142e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.073368953863476848e-17 4.214062419125637238e-12 1.333946235010178769e-02 +2.919999999999999929e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.496177509279630431e-17 3.241006183757403507e-12 1.283463708870537928e-02 +2.930000000000000160e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.165159100801475770e-17 2.492053936385204407e-12 1.234684727326527259e-02 +2.939999999999999947e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 8.569972200480942165e-18 1.915746088725387656e-12 1.187560726228777640e-02 +2.950000000000000178e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.339618593089206769e-18 1.472398606910043953e-12 1.142044157193064340e-02 +2.959999999999999964e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.876492461195293449e-18 1.131420007945783711e-12 1.098088481442565344e-02 +2.970000000000000195e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.516014689619136218e-18 8.692350115921256453e-13 1.055648162979604988e-02 +2.979999999999999982e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.816299030620873925e-18 6.676808518021171944e-13 1.014678661122676330e-02 +2.990000000000000213e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.165237816808915395e-18 5.127705371577789142e-13 9.751364224436365538e-03 +3.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.560383276220273058e-18 3.937346328552139812e-13 9.369788721393964084e-03 +3.010000000000000231e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 9.993832482698504114e-19 3.022829514125574972e-13 9.001644048714544169e-03 +3.020000000000000018e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 9.599569751652944662e-19 2.320364802305667169e-13 8.646523751060220669e-03 +3.030000000000000249e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.609663134560237002e-19 1.780888036079732782e-13 8.304030869865508188e-03 +3.040000000000000036e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.426335874404852505e-19 1.366648906565995881e-13 7.973777837697875945e-03 +3.050000000000000266e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.249593104444197535e-19 1.048625344859545797e-13 7.655386368555583756e-03 +3.060000000000000053e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.079229923003092215e-19 8.045057254146697992e-14 7.348487344397670873e-03 +3.070000000000000284e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.915046452725743785e-19 6.171483475754226614e-14 7.052720698191821368e-03 +3.080000000000000071e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.733740900516633485e-14 6.767735293758424253e-03 +3.089999999999999858e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.630576073963015200e-14 6.493188802680031381e-03 +3.100000000000000089e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.784203097343751499e-14 6.228747578537834582e-03 +3.109999999999999876e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.134990610049608915e-14 5.974086528728041363e-03 +3.120000000000000107e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.636993654283359404e-14 5.728888984102836236e-03 +3.129999999999999893e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.255087980089360287e-14 5.492846566672869232e-03 +3.140000000000000124e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 9.622023444262508414e-15 5.265659055598958388e-03 +3.149999999999999911e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.376228393573661235e-15 5.047034251693932778e-03 +3.160000000000000142e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 5.654401720407803443e-15 4.836687840646136140e-03 +3.169999999999999929e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.334020965441248550e-15 4.634343255169198240e-03 +3.180000000000000160e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.321961671158865602e-15 4.439731536273890458e-03 +3.189999999999999947e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.546210387916036112e-15 4.252591193850799138e-03 +3.200000000000000178e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.951509551298544049e-15 4.072668066744149912e-03 +3.209999999999999964e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.495644285966709991e-15 3.899715182490029811e-03 +3.220000000000000195e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.146302367693501740e-15 3.733492616884314611e-03 +3.229999999999999982e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 8.784440864424856951e-16 3.573767353538592328e-03 +3.240000000000000213e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.732631345273242546e-16 3.420313143574678534e-03 +3.250000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 5.159797832504702935e-16 3.272910365601634439e-03 +3.260000000000000231e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.954509992190602088e-16 3.131345886111691623e-03 +3.270000000000000018e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.031262881778339936e-16 2.995412920425136964e-03 +3.280000000000000249e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.321903728002282965e-16 2.864910894306957508e-03 +3.290000000000000036e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.779346119587744121e-16 2.739645306371957505e-03 +3.300000000000000266e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.363921798126602872e-16 2.619427591388118291e-03 +3.310000000000000053e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.045310720429986645e-16 2.504074984582268810e-03 +3.320000000000000284e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 8.011516247010664999e-17 2.393410387045344454e-03 +3.330000000000000071e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.145278288723690110e-17 2.287262232329181477e-03 +3.339999999999999858e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.707124520588693767e-17 2.185464354320525420e-03 +3.350000000000000089e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.604467676348906764e-17 2.087855856472461535e-03 +3.359999999999999876e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.767620830385188103e-17 1.994280982468007741e-03 +3.370000000000000107e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.125091140089592956e-17 1.904588988385136784e-03 +3.379999999999999893e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.625366933515682138e-17 1.818634016427572372e-03 +3.390000000000000124e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.243335829660339818e-17 1.736274970280319739e-03 +3.399999999999999911e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 9.568292868192838719e-18 1.657375392144508049e-03 +3.410000000000000142e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.288041134180035406e-18 1.581803341500953282e-03 +3.419999999999999929e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 5.613947943652692736e-18 1.509431275647791799e-03 +3.430000000000000160e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.316954589961256297e-18 1.440135932052709908e-03 +3.439999999999999947e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.279228174557127406e-18 1.373798212556529962e-03 +3.450000000000000178e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.545775114670173031e-18 1.310303069460456776e-03 +3.459999999999999964e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.942174366820658675e-18 1.249539393525743490e-03 +3.470000000000000195e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.521127545592068874e-18 1.191399903910495819e-03 +3.479999999999999982e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.134873235458113793e-18 1.135781040064974545e-03 +3.490000000000000213e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 9.014313092741668182e-19 1.082582855603159621e-03 +3.500000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.872561947392753250e-19 1.031708914165122178e-03 +3.510000000000000231e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.911402221423096914e-19 9.830661872815783710e-04 +3.520000000000000018e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.159183905177672920e-19 9.365649542489919719e-04 +3.530000000000000249e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.971352177835109493e-19 8.921187040207091473e-04 +3.540000000000000036e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.358235937556609512e-19 8.496440391169327684e-04 +3.550000000000000266e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.796475372080792118e-19 8.090605815536522491e-04 +3.560000000000000053e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.282792007437318676e-19 7.702908807882861367e-04 +3.570000000000000284e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.221123741823847783e-19 7.332603236773752768e-04 +3.580000000000000071e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.748213699925597550e-20 6.978970464395008481e-04 +3.589999999999999858e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.373344700157104883e-20 6.641318486144732764e-04 +3.600000000000000089e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.507739151206760329e-20 6.318981090079212742e-04 +3.609999999999999876e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.336951293620867474e-20 6.011317036084937084e-04 +3.620000000000000107e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.173966234975691376e-20 5.717709254631863193e-04 +3.629999999999999893e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.018454416309579469e-20 5.437564064946738074e-04 +3.640000000000000124e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.870098838729405082e-20 5.170310412429644916e-04 +3.649999999999999911e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.915399125123312519e-04 +3.660000000000000142e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.672302189030774246e-04 +3.669999999999999929e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.440512042065474993e-04 +3.680000000000000160e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.219540886405891643e-04 +3.689999999999999947e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.008920019016896041e-04 +3.700000000000000178e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.808199180090044399e-04 +3.709999999999999964e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.616945919146784282e-04 +3.720000000000000195e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.434744978540011599e-04 +3.729999999999999982e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.261197694083200821e-04 +3.740000000000000213e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.095921412528926560e-04 +3.750000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.938548925614003549e-04 +3.760000000000000231e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.788727920382623915e-04 +3.770000000000000018e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.646120445495396285e-04 +3.780000000000000249e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.510402393227918988e-04 +3.790000000000000036e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.381262996860046804e-04 +3.800000000000000266e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.258404343154230280e-04 +3.810000000000000053e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.141540899619723433e-04 +3.820000000000000284e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.030399056257952133e-04 +3.830000000000000071e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.924716681483835924e-04 +3.839999999999999858e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.824242691917173792e-04 +3.850000000000000089e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.728736635738833786e-04 +3.859999999999999876e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.637968289306732496e-04 +3.870000000000000107e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.551717266727870568e-04 +3.879999999999999893e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.469772642084021851e-04 +3.890000000000000124e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.391932584010457258e-04 +3.899999999999999911e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.318004002329337956e-04 +3.910000000000000142e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.247802206441653273e-04 +3.919999999999999929e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.181150575184606466e-04 +3.930000000000000160e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.117880237863992903e-04 +3.939999999999999947e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.057829766174721661e-04 +3.950000000000000178e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.000844876725719612e-04 +3.959999999999999964e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 9.467781438893157803e-05 +3.970000000000000195e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 8.954887226988028496e-05 +3.979999999999999982e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 8.468420815218909300e-05 +3.990000000000000213e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 8.007097442415881782e-05 +4.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.569690416802829188e-05 +4.009999999999999787e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.155028720067656181e-05 +4.020000000000000462e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.761994698701451201e-05 +4.030000000000000249e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.389521840087908475e-05 +4.040000000000000036e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.036592630864834542e-05 +4.049999999999999822e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 5.702236495122367109e-05 +4.060000000000000497e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 5.385527810042758625e-05 +4.070000000000000284e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 5.085583996627522527e-05 +4.080000000000000071e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.801563683198268288e-05 +4.089999999999999858e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.532664939397855914e-05 +4.099999999999999645e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.278123578457674117e-05 +4.110000000000000320e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.037211525536551257e-05 +4.120000000000000107e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.809235249975894268e-05 +4.129999999999999893e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.593534259354158979e-05 +4.139999999999999680e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.389479653263693828e-05 +4.150000000000000355e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.196472734770647897e-05 +4.160000000000000142e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.013943677559264344e-05 +4.169999999999999929e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.841350246800420975e-05 +4.179999999999999716e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.678176571825482426e-05 +4.190000000000000391e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.523931968726843476e-05 +4.200000000000000178e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.378149811048649358e-05 +4.209999999999999964e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.240386446773675930e-05 +4.219999999999999751e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.110220159856760464e-05 +4.230000000000000426e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.987250174599398784e-05 +4.240000000000000213e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.871095701206949778e-05 +4.250000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.761395020916761803e-05 +4.259999999999999787e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.657804609134507862e-05 +4.270000000000000462e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.559998295065271757e-05 +4.280000000000000249e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.467666456376918968e-05 +4.290000000000000036e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.380515247484436792e-05 +4.299999999999999822e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.298265860096709476e-05 +4.310000000000000497e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.220653814719408694e-05 +4.320000000000000284e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.147428281861089934e-05 +4.330000000000000071e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.078351431742450022e-05 +4.339999999999999858e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.013197811361725266e-05 +4.350000000000000533e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 9.517537478213390192e-06 +4.360000000000000320e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 8.938167768725686373e-06 +4.370000000000000107e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 8.391950956853344026e-06 +4.379999999999999893e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.877070388993910244e-06 +4.389999999999999680e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.391805770604876890e-06 +4.400000000000000355e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.934528365906822180e-06 +4.410000000000000142e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.503696404854364192e-06 +4.419999999999999929e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.097850689711229687e-06 +4.429999999999999716e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 5.715610393953746256e-06 +4.440000000000000391e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 5.355669046585743586e-06 +4.450000000000000178e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 5.016790695282744467e-06 +4.459999999999999964e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.697806242087593931e-06 +4.469999999999999751e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.397609945659861551e-06 +4.480000000000000426e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.115156084330508385e-06 +4.490000000000000213e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.849455774438885070e-06 +4.500000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.599573938626780587e-06 +4.509999999999999787e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.364626418939012099e-06 +4.520000000000000462e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.143777229731475377e-06 +4.530000000000000249e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.936235945517863021e-06 +4.540000000000000036e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.741255218998309102e-06 +4.549999999999999822e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.558128424608961101e-06 +4.560000000000000497e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.386187423011975700e-06 +4.570000000000000284e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.224800442015688300e-06 +4.580000000000000071e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.073370069474093740e-06 +4.589999999999999858e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.931331353769111588e-06 +4.600000000000000533e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.798150007526828464e-06 +4.610000000000000320e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.673320710266274216e-06 +4.620000000000000107e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.556365505724561042e-06 +4.629999999999999893e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.446832289650692389e-06 +4.639999999999999680e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.344293383910860287e-06 +4.650000000000000355e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.248344192803720109e-06 +4.660000000000000142e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.158601937545942537e-06 +4.669999999999999929e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.074704464956369815e-06 +4.679999999999999716e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 9.963091264432861814e-07 +4.690000000000000391e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 9.230917234831150279e-07 +4.700000000000000178e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 8.547455158709785526e-07 +4.709999999999999964e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.909802891238337773e-07 +4.719999999999999751e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.315214775254283935e-07 +4.730000000000000426e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.761093394180026812e-07 +4.740000000000000213e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.244981814688013757e-07 +4.750000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 5.764556287684500852e-07 +4.759999999999999787e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 5.317619377533198992e-07 +4.770000000000000462e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.902093490828477776e-07 +4.780000000000000249e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.516014777458562430e-07 +4.790000000000000036e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.157527378149979787e-07 +4.799999999999999822e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.824877994152667324e-07 +4.810000000000000497e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.516410756195147884e-07 +4.820000000000000284e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 3.230562371303328443e-07 +4.830000000000000071e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.965857527523306205e-07 +4.839999999999999858e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.720904538010278337e-07 +4.850000000000000533e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.494391207330565905e-07 +4.860000000000000320e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.285080904167570243e-07 +4.870000000000000107e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2.091808825914865425e-07 +4.879999999999999893e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.913478441877041063e-07 +4.889999999999999680e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.749058102973752587e-07 +4.900000000000000355e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.597577806952962489e-07 +4.910000000000000142e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.458126109160604397e-07 +4.919999999999999929e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.329847169884304582e-07 +4.929999999999999716e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.211937930188092765e-07 +4.940000000000000391e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.103645408980406442e-07 +4.950000000000000178e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.004264114812765874e-07 +4.959999999999999964e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 9.131335665895433652e-08 +4.969999999999999751e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 8.296359179845285449e-08 +4.980000000000000426e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.531936809080894562e-08 +4.990000000000000213e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.832675438539998569e-08 +5.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 6.193542813799492433e-08 diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index 2e1050cb..5dc90cdd 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -1208,3 +1208,48 @@ SP_v1.6.6: e2_col: e2 path: unions_shapepipe_star_2024_v1.6.a.fits patch_number: 150 + +GLASS_mock_validation: + subdir: /n09data/guerrini/glass_mock_test/results/ + pipeline: SP + colour: violet + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: '*' + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /n09data/guerrini/glass_mock_test/mask_nside_1024.fits + psf: + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR + hdu: 1 + path: /n17data/UNIONS/WL/v1.6.x/unions_shapepipe_psf_2024_v1.6.a.fits + ra_col: RA + dec_col: DEC + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR + shear: + R: 1.0 + path: tomo_test_2_glass_sim_00001_1024.fits + redshift_path: /home/guerrini/sp_validation_cosmostat/config/glass_mock/test_data/redshift_distribution_tomo.txt + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + cols: RA,Dec + tomo_bin_col: TOM_BIN_ID + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.6.x/unions_shapepipe_star_2024_v1.6.a.fits + patch_number: 150 diff --git a/cosmo_val/run_cosmo_val.py b/cosmo_val/run_cosmo_val.py index de19ebc3..f7c34fb0 100644 --- a/cosmo_val/run_cosmo_val.py +++ b/cosmo_val/run_cosmo_val.py @@ -134,7 +134,6 @@ # max_sep_int=300, # nbins_int=100, # npatch=256, -# var_method="jackknife", # ) # %% diff --git a/papers/bmodes/rules/figures.smk b/papers/bmodes/rules/figures.smk index 31b65ea0..b9095aaf 100644 --- a/papers/bmodes/rules/figures.smk +++ b/papers/bmodes/rules/figures.smk @@ -84,7 +84,7 @@ def _reporting_cov_path(version): def _xi_integration_path(version): """Path to fine-binned 2PCF integration file. Unpatched: values only, no covariance.""" return ( - f"{COSMO_VAL_OUTPUT}/{version}_xi_minsep={FIDUCIAL['min_sep_int']}" + f"{COSMO_VAL_OUTPUT}/xi_{version}_tomo_bin_all_minsep={FIDUCIAL['min_sep_int']}" f"_maxsep={FIDUCIAL['max_sep_int']}_nbins={FIDUCIAL['nbins_int']}_npatch=1.txt" ) diff --git a/papers/bmodes/scripts/bb_covariance_nz_independence.py b/papers/bmodes/scripts/bb_covariance_nz_independence.py index 7f8cae02..a11f9916 100644 --- a/papers/bmodes/scripts/bb_covariance_nz_independence.py +++ b/papers/bmodes/scripts/bb_covariance_nz_independence.py @@ -511,7 +511,7 @@ def _from_cli(argv=None): } xi_integration_path = os.path.join( a.cosmo_val_dir, - f"{version}_xi_minsep={min_sep_int}_maxsep={max_sep_int}" + f"xi_{version}_tomo_bin_all_minsep={min_sep_int}_maxsep={max_sep_int}" f"_nbins={nbins_int}_npatch={npatch}.txt", ) pseudo_cl_path = os.path.join( diff --git a/papers/bmodes/scripts/cosebis_version_comparison.py b/papers/bmodes/scripts/cosebis_version_comparison.py index 464cca70..51fa9d53 100644 --- a/papers/bmodes/scripts/cosebis_version_comparison.py +++ b/papers/bmodes/scripts/cosebis_version_comparison.py @@ -148,7 +148,7 @@ def _create_stacked_bmode_figure( def _xi_integration(results_dir, ver): - return f"{results_dir}/{ver}_xi_minsep=0.5_maxsep=300.0_nbins=1000_npatch=1.txt" + return f"{results_dir}/xi_{ver}_tomo_bin_all_minsep=0.5_maxsep=300.0_nbins=1000_npatch=1.txt" def _cov_integration(cov_dir, ver): diff --git a/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py b/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py index 5e4e4e3e..2fc52dae 100644 --- a/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py +++ b/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py @@ -889,7 +889,7 @@ def _from_cli(argv=None): default=None, help=( "Explicit path to the fiducial 1000-bin integration xi_pm text file " - "reproduced by lc (e.g. SP_v1.4.6.3_leak_corr_xi_minsep=0.5_maxsep=300.0_" + "reproduced by lc (e.g. xi_SP_v1.4.6.3_leak_corr_tomo_bin_all_minsep=0.5_maxsep=300.0_" "nbins=1000_npatch=1.txt), overriding the --cosmo-val-dir pattern lookup " "for --fiducial-version." ), @@ -946,7 +946,7 @@ def _from_cli(argv=None): if is_fiducial and a.fiducial_xi_path else os.path.join( a.cosmo_val_dir, - f"{ver}_xi_minsep={min_sep_int}_maxsep={max_sep_int}" + f"xi_{ver}_tomo_bin_all_minsep={min_sep_int}_maxsep={max_sep_int}" f"_nbins={nbins_int}_npatch={npatch}.txt", ) ) diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 5c614f82..26ade8e2 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -33,6 +33,8 @@ cosmo_val: cov_estimate_method: th compute_cov_rho: true n_cov: 100 + # simulations behind a cov_estimate_method: sim rho/tau covariance (Hartlap) + n_sim_cov: 300 var_method: jackknife quantile: 0.1587 ylim_xi_sys_ratio: [-0.02, 0.5] @@ -43,7 +45,8 @@ cosmo_val: power: 0.5 n_ell_bins: 32 ell_step: 10 - pol_factor: true + # e2 multiplier (±1) applied before NaMaster; -1 flips the e2 sign + pol_factor: -1 noise_bias_method: analytic cell_seed: 8192 path_onecovariance: "/home/guerrini/OneCovariance/" diff --git a/papers/harmonic/2025_09_11_namaster_covariance.py b/papers/harmonic/2025_09_11_namaster_covariance.py index aea3fc7e..47a77a92 100644 --- a/papers/harmonic/2025_09_11_namaster_covariance.py +++ b/papers/harmonic/2025_09_11_namaster_covariance.py @@ -112,11 +112,14 @@ catalog_config="/home/guerrini/sp_validation/cosmo_val/cat_config.yaml", ) # %% -params = get_params_rho_tau(cv.cc["SP_v1.4.5_leak_corr"], survey="SP_v1.4.5_leak_corr") +params = get_params_rho_tau(cv.cc["SP_v1.4.5_leak_corr"]) # %% print("Collecting maps...") -n_gal, unique_pix, idx, idx_rep = cv.get_n_gal_map(params, nside, cat_gal) +unique_pix, idx, idx_rep = cv.get_pixels(params, nside, cat_gal) +n_gal = cv.get_n_gal_map( + params, nside, cat_gal, unique_pix=unique_pix, idx=idx, idx_rep=idx_rep +) mask = n_gal > 0 print("Computing noise...") diff --git a/pyproject.toml b/pyproject.toml index 2ffcd753..13cbadac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,8 @@ dependencies = [ "jupyterlab", "jupytext>=1.15", "lenspack", + # Sacha's fork of OneCovariance with the bug fix for numpy>2 + "levin @ git+https://github.com/sachaguer/OneCovariance.git@main", "lmfit", "numpy>=2.0", "opencv-python-headless", @@ -133,18 +135,16 @@ docs = [ ] # GLASS mock generation (sp_validation.glass_mock). Kept optional: the core # library and its import guard resolve without GLASS (the production container -# ships it via the ``glass`` extra). Pinned set (verified 2026-06-13, fiber -# glass-cosmology-api-pin): glass 2025.1 is the unique version with the flat API -# the map path uses AND the legacy ``cosmo.dc``/``xm``/``ef`` interface that -# ``cosmology`` 2022.10.9 (its newest release) provides — newer glass calls -# ``comoving_distance``, which no ``cosmology`` release exposes. ``matter_cls`` -# lives in the separate ``glass.ext.camb`` package (absent from glass core -# >=2024.2), so it is pinned explicitly. +# ships it via the ``glass`` extra). glass reads its cosmology through the +# Cosmology API (``comoving_distance``, ``H_over_H0``, ...); ``glass_mock`` +# supplies it from CAMB via ``cosmology.compat.camb``. ``matter_cls`` lives in the +# separate ``glass.ext.camb`` package (absent from glass core >=2024.2), so it is +# pinned explicitly. glass = [ - "glass==2025.3", + "glass==2026.2", "glass.ext.camb==2023.6", - "cosmology==2022.10.9", - # fitsio: make_unions_glass_sim.py writes the mock catalogue as FITS. + "cosmology.compat.camb==0.2.0", + # fitsio: make_glass_sim.py writes the mock catalogue as FITS. "fitsio", ] # Cosmo-inference workflow runners (workflow/scripts/*). Kept optional: the core diff --git a/scripts/cosmo_val/run_cl_gaussian_sims.py b/scripts/cosmo_val/run_cl_gaussian_sims.py new file mode 100644 index 00000000..3e0f2efa --- /dev/null +++ b/scripts/cosmo_val/run_cl_gaussian_sims.py @@ -0,0 +1,728 @@ +""" +Executable script to run Gaussian simulations of shear maps and compute their power spectra using the NaMaster library. +The execution is distributed across multiple MPI processes to speed up the computation. + +To run the script, use the following command: + mpirun -np python run_cl_gaussian_sims.py --config --output --version [options] + +Options: + -c, --config: Path to the configuration file to generate the simulation (required). + -o, --output: Output directory for the simulation results (required, ideally the same as cosmo_val). + -v, --version: Version name for the simulation (required). + -t, --tomo: Whether to run the simulation in tomographic mode (default: False). + -iw, --ignore-warnings: Ignore warnings during the simulation. + -s, --seed: Random seed for the simulations (default: 42). + -n, --n_sims: Number of simulations to run (default: 100). + -ns, --nside: Nside for the HEALPix maps (default: 1024). + -b, --binning: Binning scheme for the power spectra (default: powspace). + -es, --ell_step: Step size for the linear binning (default: 10). + -eb, --ell_bins: Number of bins for the power spectra (default: 32). + -p, --power: Power for the powspace binning (default: 0.5). + -cc, --cosmo-config: Path to the cosmology configuration file (default: None, use Planck18 cosmology by default). + -f, --force: Force overwrite of existing output files (default: False). + -seb, --save-eb: Save the EB power spectrum (default: False). + -sbb, --save-bb: Save the BB power spectrum (default: False). + +Author: Sacha Guerrini +""" + +import argparse +import itertools +import os +import warnings +from pathlib import Path + +import healpy as hp +import numpy as np +import pymaster as nmt +import yaml +from astropy.io import fits +from cs_util.cosmo import get_cosmo +from mpi4py import MPI +from tqdm import tqdm + +import sp_validation.pseudo_cl as spv_pseudo_cl + +pol_to_pol_index_dict = {"EE": 0, "EB": 1, "BE": 2, "BB": 3} + + +def get_parser(): + """Create the argument parser.""" + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="Generate Gaussian simulations of shear maps and compute their power spectra.", + fromfile_prefix_chars="@", + ) + parser.add_argument( + "-c", + "--config", + help="Path to the configuration file to generate the simulation", + type=str, + required=True, + ) + parser.add_argument( + "-o", + "--output", + help="Output directory for the simulation results (Ideally the same than cosmo_val)", + type=str, + required=True, + ) + parser.add_argument( + "-v", "--version", help="Run some validation checks", type=str, required=True + ) + parser.add_argument( + "-t", + "--tomo", + help="Whether to run the simulation in tomographic mode (default: False)", + action="store_true", + ) + parser.add_argument( + "-iw", + "--ignore-warnings", + help="Ignore warnings during the simulation", + action="store_true", + ) + parser.add_argument( + "-s", + "--seed", + help="Random seed for the simulations (default: 42)", + type=int, + default=42, + ) + parser.add_argument( + "-n", + "--n_sims", + help="Number of simulations to run (default: 100)", + type=int, + default=100, + ) + parser.add_argument( + "-ns", + "--nside", + help="Nside for the HEALPix maps (default: 1024)", + type=int, + default=1024, + ) + parser.add_argument( + "-b", + "--binning", + help="Binning scheme for the power spectra (default: powspace)", + type=str, + default="powspace", + ) + parser.add_argument( + "-es", + "--ell_step", + help="Step size for the linear binning (default: 10)", + type=int, + default=10, + ) + parser.add_argument( + "-eb", + "--ell_bins", + help="Number of bins for the power spectra (default: 32)", + type=int, + default=32, + ) + parser.add_argument( + "-p", + "--power", + help="Power for the powspace binning (default: 0.5)", + type=float, + default=0.5, + ) + parser.add_argument( + "-cc", + "--cosmo-config", + help="Path to the cosmology configuration file (default: None, use Planck18 cosmology by default)", + type=str, + default=None, + ) + parser.add_argument( + "-f", + "--force", + help="Force overwrite of existing output files (default: False)", + action="store_true", + ) + parser.add_argument( + "-seb", + "--save-eb", + help="Save the EB covariance matrix (default: False)", + action="store_true", + ) + parser.add_argument( + "-sbb", + "--save-bb", + help="Save the BB covariance matrix (default: False)", + action="store_true", + ) + return parser + + +# --- Helper function for reading configuration and extracting useful data --- +def load_version_config(config, version, tomography): + if not os.path.exists(config): + raise FileNotFoundError(f"Config file {config} does not exist.") + else: + # Load useful information from the config file + with open(config, "r") as file: + cc = yaml.load(file, Loader=yaml.FullLoader) + + # Check that the version exists in the config file + if version not in cc: + raise ValueError(f"Version {version} not found in config file.") + else: + subdir = Path(cc[version]["subdir"]) + cat_path = str(subdir / Path(cc[version]["shear"]["path"])) + + ra_col = cc[version]["shear"]["cols"].split(",")[0] + dec_col = cc[version]["shear"]["cols"].split(",")[1] + e1_col = cc[version]["shear"]["e1_col"] + e2_col = cc[version]["shear"]["e2_col"] + w_col = cc[version]["shear"]["w_col"] + tomo_bin_col = None + if tomography: + tomo_bin_col = cc[version]["shear"]["tomo_bin_col"] + + redshift_path = Path(cc[version]["shear"]["redshift_path"]) + return ( + cat_path, + ra_col, + dec_col, + e1_col, + e2_col, + w_col, + tomo_bin_col, + redshift_path, + ) + + +def load_catalog_slices(cat_path, columns, tomography, tomo_bin_col): + cat_gal = fits.getdata(cat_path) + + ra, dec, e1, e2, w = {}, {}, {}, {}, {} + + ra_col, dec_col, e1_col, e2_col, w_col = columns + + if tomography: + tomo_bin_ids = np.unique(cat_gal[tomo_bin_col]) + else: + tomo_bin_ids = ["all"] + + for bin_id in tomo_bin_ids: + if bin_id == "all": + mask_bin = np.ones(len(cat_gal), dtype=bool) + else: + mask_bin = cat_gal[tomo_bin_col] == bin_id + ra[bin_id] = cat_gal[ra_col][mask_bin] + dec[bin_id] = cat_gal[dec_col][mask_bin] + e1[bin_id] = cat_gal[e1_col][mask_bin] + e2[bin_id] = cat_gal[e2_col][mask_bin] + w[bin_id] = cat_gal[w_col][mask_bin] + + del cat_gal # Free memory + + return ra, dec, e1, e2, w, tomo_bin_ids + + +def build_pre_computed_maps(ra, dec, w, tomo_bin_ids, nside): + n_gal, unique_pix, idx, idx_rep = {}, {}, {}, {} + + for bin_id in tomo_bin_ids: + unique_pix[bin_id], idx[bin_id], idx_rep[bin_id] = spv_pseudo_cl.get_pixels( + ra[bin_id], dec[bin_id], nside + ) + n_gal[bin_id] = spv_pseudo_cl.get_n_gal_map( + nside, + ra[bin_id], + dec[bin_id], + weights=w[bin_id], + unique_pix=unique_pix[bin_id], + idx=idx[bin_id], + idx_rep=idx_rep[bin_id], + ) + + return n_gal, unique_pix, idx, idx_rep + + +def build_fiducial_cl(cosmo_params, redshift_path, lmax, tomography, tomo_bin_ids): + cosmo = get_cosmo(**cosmo_params) + + # --- Load the redshift distribution --- + redshift_distr = np.loadtxt(redshift_path) + z = redshift_distr[:, 0] + dndz = redshift_distr[:, 1:] + + if not tomography and dndz.shape[1] > 1: + # Sum the redshift distributions across all bins to get the total distribution + dndz = np.sum(dndz, axis=1, keepdims=True) + + # Normalize the summed distribution + dndz /= np.trapezoid(dndz[:, 0], z) + + if tomography and dndz.shape[1] != len(tomo_bin_ids): + raise ValueError( + "Mismatch between the number of tomographic bins and the redshift distributions provided." + ) + + fiducial_cl = spv_pseudo_cl.get_fiducial_cl(z, dndz, lmax, cosmo) + + # healpy.synalm expects spectra arrays with an explicit ell=0 slot. + # The theory helper returns ell=1..lmax, so prepend a zero monopole here + # to keep the Gaussian map synthesis numerically stable. + fiducial_cl = { + key: np.concatenate(([0.0], np.asarray(value, dtype=float))) + for key, value in fiducial_cl.items() + } + + if not tomography: + fiducial_cl = {"WallxWall": fiducial_cl["W1xW1"]} + + return cosmo, fiducial_cl + + +def prepare_workspace(n_gal, tomo_bin_ids, nside, b, b_lmax, out_dir, force_run): + """Precomputes the workspace objects for each pair of tomographic bins and saves them to disk.""" + f = {} + for bin_id in tqdm(tomo_bin_ids, desc="Computing fields"): + f[bin_id] = nmt.NmtField( + mask=n_gal[bin_id], + maps=[np.zeros(hp.nside2npix(nside)), np.zeros(hp.nside2npix(nside))], + lmax=b_lmax, + ) + + # Compute wsp object for each pair of tomographic bins + tomo_bin_pairs = list(itertools.combinations_with_replacement(tomo_bin_ids, 2)) + for tomo_bin_a, tomo_bin_b in tqdm(tomo_bin_pairs, desc="Computing workspaces"): + wsp_file = os.path.join(out_dir, f"wsp_{tomo_bin_a}_{tomo_bin_b}.npz") + if os.path.exists(wsp_file) and not force_run: + print( + f"Workspace for bins {tomo_bin_a} and {tomo_bin_b} already exists. Skipping." + ) + continue + wsp = nmt.NmtWorkspace(f[tomo_bin_a], f[tomo_bin_b], b) + wsp.write_to(wsp_file) + + +def get_legacy_seed(base_seed, sim_id): + """Derive a well-mixed, reproducible legacy seed for a given simulation index.""" + ss = np.random.SeedSequence(entropy=base_seed, spawn_key=(sim_id,)) + return int(ss.generate_state(1, dtype=np.uint32)[0]) + + +# --- Function to generate Gaussian simulation, add noise and extract the spectra --- +def get_gaussian_simulation( + nside, + lmax, + fiducial_cl, + n_gal, + unique_pix, + idx, + idx_rep, + ra, + dec, + e1, + e2, + w, + tomo_bin_ids, +): + final_maps = {} + for bin_id in tomo_bin_ids: + noise_map_e1, noise_map_e2 = spv_pseudo_cl.get_noise_realisation( + ra[bin_id], + dec[bin_id], + e1[bin_id], + e2[bin_id], + w[bin_id], + nside, + unique_pix[bin_id], + idx[bin_id], + idx_rep[bin_id], + n_gal_map=n_gal[bin_id], + ) + final_maps[bin_id] = noise_map_e1 + 1j * noise_map_e2 + + # Generate a Gaussian field with the fiducial power spectrum + tomo_bin_pairs = list(itertools.combinations_with_replacement(tomo_bin_ids, 2)) + # Organise the cls in a list to feed the helpy function + cl_list = [ + fiducial_cl[f"W{tomo_bin_a}xW{tomo_bin_b}"] + for tomo_bin_a, tomo_bin_b in tomo_bin_pairs + ] + # Generate alms with the correct cross-bin correlation + alm_shear = hp.synalm(cl_list, lmax=lmax) + + for bin_id in tomo_bin_ids: + if bin_id != "all": + gauss_map = hp.alm2map( + [ + np.zeros_like(alm_shear[bin_id - 1]), + alm_shear[bin_id - 1], + np.zeros_like(alm_shear[bin_id - 1]), + ], + nside=nside, + verbose=False, + ) + else: + gauss_map = hp.alm2map( + [ + np.zeros_like(alm_shear[0]), + alm_shear[0], + np.zeros_like(alm_shear[0]), + ], + nside=nside, + verbose=False, + ) + + mask = n_gal[bin_id] > 0 + final_maps[bin_id][mask] += gauss_map[1][mask] + 1j * gauss_map[2][mask] + + return final_maps + + +def extract_spectra(noisy_gaussian_maps, n_gal, tomo_bin_ids, lmax, out_dir): + # Create the NmtField for each tomo bin + f = {} + for bin_id in tomo_bin_ids: + # Compute the power spectrum + f[bin_id] = nmt.NmtField( + mask=n_gal[bin_id], + maps=[noisy_gaussian_maps[bin_id].real, noisy_gaussian_maps[bin_id].imag], + lmax=lmax, + ) + + # Compute the power spectrum for each bin pair + cl_final = {} + tomo_bin_pairs = list(itertools.combinations_with_replacement(tomo_bin_ids, 2)) + for tomo_bin_a, tomo_bin_b in tomo_bin_pairs: + cl_coupled = nmt.compute_coupled_cell(f[tomo_bin_a], f[tomo_bin_b]) + + wsp = nmt.NmtWorkspace() + wsp_file = os.path.join(out_dir, f"wsp_{tomo_bin_a}_{tomo_bin_b}.npz") + wsp.read_from(wsp_file) + cl_decoupled = wsp.decouple_cell(cl_coupled) + cl_final[f"W{tomo_bin_a}xW{tomo_bin_b}"] = cl_decoupled + + return cl_final + + +# --- single unit of work distributed to the MPI processes --- +def run_one_simulation(sim_id, nside, version, tomography, out_dir, force_run, seed): + """Worker executes this simulation and saves results""" + try: + out_file = f"{out_dir}/cl_sample_{sim_id}_{version}_tomography_{tomography}_seed_{seed}.npz" + if os.path.exists(out_file) and not force_run: + print(f"Rank {rank} skipping {sim_id} (already exists)") + return + + legacy_seed = get_legacy_seed(seed, sim_id) + np.random.seed(legacy_seed) + + print(f"Rank {rank} starting {sim_id}") + setup_file = os.path.join( + out_dir, f"precomputed_setup_{version}_tomography_{tomography}.npz" + ) + data = np.load(setup_file, allow_pickle=True) + n_gal = data["n_gal"].item() + unique_pix = data["unique_pix"].item() + idx = data["idx"].item() + idx_rep = data["idx_rep"].item() + ra_col = data["ra_col"].item() + dec_col = data["dec_col"].item() + e1_col = data["e1_col"].item() + e2_col = data["e2_col"].item() + w = data["w"].item() + fiducial_cl = data["fiducial_cl"].item() + tomo_bin_ids = data["tomo_bin_ids"] + + lmin, lmax, b_lmax = spv_pseudo_cl.pseudo_cl_geometry(nside) + + # --- call the simulation --- + noisy_gaussian_maps = get_gaussian_simulation( + nside, + lmax, + fiducial_cl, + n_gal, + unique_pix, + idx, + idx_rep, + ra_col, + dec_col, + e1_col, + e2_col, + w, + tomo_bin_ids, + ) + + # --- Compute the power spectra --- + cl_final = extract_spectra( + noisy_gaussian_maps, n_gal, tomo_bin_ids, b_lmax, out_dir + ) + + # Save the results + np.savez(out_file, cl_decoupled=cl_final) + + print(f"Rank {rank} finished simulation {sim_id} for version {version}") + + except Exception as e: + print(f"Rank {rank} failed simulation {sim_id} with error {e}") + + +# --- Distribution of the MPI processes in a master-worker scheme --- +def distribute_work(comm, n_sims, worker_fn, worker_args=()): + rank = comm.Get_rank() + size = comm.Get_size() + + work_tag = 1 + done_tag = 2 + stop_tag = 0 + + if rank == 0: + next_sim = 0 + closed_workers = 0 + + for worker in range(1, size): + if next_sim < n_sims: + comm.send(next_sim, dest=worker, tag=work_tag) + next_sim += 1 + else: + comm.send("STOP", dest=worker, tag=stop_tag) + closed_workers += 1 + + while closed_workers < size - 1: + status = MPI.Status() + _ = comm.recv(source=MPI.ANY_SOURCE, tag=done_tag, status=status) + worker = status.Get_source() + + if next_sim < n_sims: + comm.send(next_sim, dest=worker, tag=work_tag) + next_sim += 1 + else: + comm.send("STOP", dest=worker, tag=stop_tag) + closed_workers += 1 + + else: + while True: + sim_id = comm.recv(source=0, tag=MPI.ANY_TAG, status=MPI.Status()) + if sim_id == "STOP": + break + worker_fn(sim_id, *worker_args) + comm.send(sim_id, dest=0, tag=done_tag) + + +# --- Final function to read the computed spectra and extract a covariance matrix --- +def concatenate_spectra(cl_sample, tomo_bin_ids, pol_index): + """Concatenate the spectra from different tomographic bins into a single array.""" + concatenated_cl = [] + tomo_bin_pairs = itertools.combinations_with_replacement(tomo_bin_ids, 2) + for tomo_bin_a, tomo_bin_b in tomo_bin_pairs: + concatenated_cl.append(cl_sample[f"W{tomo_bin_a}xW{tomo_bin_b}"][pol_index]) + return np.concatenate(concatenated_cl) + + +def get_covariance_from_simulated_spectra( + n_sims, version, tomography, tomo_bin_ids, pol, out_dir, seed +): + """Compute the covariance of one polarization from the seed's spectra.""" + cl_samples = [] + pol_index = pol_to_pol_index_dict[pol] + for sim_id in range(n_sims): + out_file = f"{out_dir}/cl_sample_{sim_id}_{version}_tomography_{tomography}_seed_{seed}.npz" + if not os.path.exists(out_file): + raise FileNotFoundError(f"Simulation output file {out_file} not found.") + data = np.load(out_file, allow_pickle=True) + cl_samples.append( + concatenate_spectra(data["cl_decoupled"].item(), tomo_bin_ids, pol_index) + ) + + cl_samples = np.array(cl_samples) + + covariance_matrix = np.cov(cl_samples, rowvar=False) + + return covariance_matrix + + +if __name__ == "__main__": + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + size = comm.Get_size() + + # --- Get the parser and extract arguments --- + parser = get_parser() + args = parser.parse_args() + + out_dir = args.output + version = args.version + out_dir = str(Path(out_dir) / Path(f"gaussian_simulations_{version}")) + + config = args.config + tomography = args.tomo + ignore_warnings = args.ignore_warnings + + if ignore_warnings: + warnings.filterwarnings("ignore") + print("Ignoring warnings during the simulation.") + + nside = args.nside + binning = args.binning + ell_step = args.ell_step + n_ell_bins = args.ell_bins + power = args.power + + force_run = args.force + + save_eb = args.save_eb + save_bb = args.save_bb + seed = args.seed + + if rank == 0: + # Check that the config file exists (same than cosmo_val) + ( + cat_path, + ra_col, + dec_col, + e1_col, + e2_col, + w_col, + tomo_bin_col, + redshift_path, + ) = load_version_config(config, version, tomography) + columns = (ra_col, dec_col, e1_col, e2_col, w_col) + + os.makedirs(out_dir, exist_ok=True) + print(f"Output directory: {out_dir}") + + print("Pre-compute setup for the simulations and workspace...") + + print("Reading catalog...") + ra, dec, e1, e2, w, tomo_bin_ids = load_catalog_slices( + cat_path, columns, tomography, tomo_bin_col + ) + + # Setup the binning + nside = args.nside + lmin, lmax, b_lmax = spv_pseudo_cl.pseudo_cl_geometry(nside) + ells = np.arange(lmin, lmax + 1) + b = spv_pseudo_cl.make_namaster_bin( + lmin, + lmax, + b_lmax, + binning=binning, + ell_step=ell_step, + n_ell_bins=n_ell_bins, + power=power, + ) + ell_eff = b.get_effective_ells() + + print("Getting n_gal map...") + n_gal, unique_pix, idx, idx_rep = build_pre_computed_maps( + ra, dec, w, tomo_bin_ids, nside + ) + + print("Getting fiducial Cl...") + # --- Get the cosmology object to compute the fiducial --- + if args.cosmo_config is None: + cosmo_params = {} + else: + with open(args.cosmo_config, "r") as f: + cosmo_params = yaml.safe_load(f) + + cosmo, fiducial_cl = build_fiducial_cl( + cosmo_params, redshift_path, lmax, tomography, tomo_bin_ids + ) + + print("Saving precomputed setup...") + setup_file = os.path.join( + out_dir, f"precomputed_setup_{version}_tomography_{tomography}.npz" + ) + + np.savez( + setup_file, + n_gal=n_gal, + unique_pix=unique_pix, + idx=idx, + idx_rep=idx_rep, + ra_col=ra, + dec_col=dec, + e1_col=e1, + e2_col=e2, + w=w, + fiducial_cl=fiducial_cl, + tomo_bin_ids=tomo_bin_ids, + ) + + print("Setting up workspace...") + prepare_workspace(n_gal, tomo_bin_ids, nside, b, b_lmax, out_dir, force_run) + + print( + f"Running Gaussian simulations for version {version} with the following parameters:\n" + f" - Number of simulations: {args.n_sims}\n" + f" - Nside: {nside}\n" + f" - Binning: {binning}\n" + f" - Ell step: {ell_step}\n" + f" - Number of ell bins: {n_ell_bins}\n" + f" - Power for powspace: {power}\n" + f" \n" + f" --- Cosmological parameters ---\n" + f" - H0: {cosmo['H0']}\n" + f" - Omega_m: {cosmo['Omega_m']}\n" + f" - Omega_b: {cosmo['Omega_b']}\n" + f" - Omega_c: {cosmo['Omega_c']}\n" + f" - Omega_nu: {cosmo['Omega_nu_mass'] + cosmo['Omega_nu_rel']}\n" + f" - n_s: {cosmo['n_s']}\n" + f" - sigma_8: {cosmo.sigma8()}\n" + f" - w0: {cosmo['w0']}\n" + f" - wa: {cosmo['wa']}\n" + ) + + comm.Barrier() + + n_sims = args.n_sims + distribute_work( + comm, + n_sims, + run_one_simulation, + worker_args=(nside, version, tomography, out_dir, force_run, seed), + ) + + comm.Barrier() + if rank == 0: + print("All simulations completed ✅") + + print(f"Merging into a EE covariance matrix for version {version}...") + outpath_cov = os.path.join( + out_dir, f"covariance_matrix_EE_{version}_tomography_{tomography}.npy" + ) + + covariance_matrix = get_covariance_from_simulated_spectra( + n_sims, version, tomography, tomo_bin_ids, "EE", out_dir, seed + ) + + np.save(outpath_cov, covariance_matrix) + print(f"EE covariance matrix saved to {outpath_cov}") + + if save_eb: + print(f"Merging into a EB covariance matrix for version {version}...") + outpath_cov = os.path.join( + out_dir, f"covariance_matrix_EB_{version}_tomography_{tomography}.npy" + ) + + covariance_matrix = get_covariance_from_simulated_spectra( + n_sims, version, tomography, tomo_bin_ids, "EB", out_dir, seed + ) + + np.save(outpath_cov, covariance_matrix) + print(f"EB covariance matrix saved to {outpath_cov}") + + if save_bb: + print(f"Merging into a BB covariance matrix for version {version}...") + outpath_cov = os.path.join( + out_dir, f"covariance_matrix_BB_{version}_tomography_{tomography}.npy" + ) + + covariance_matrix = get_covariance_from_simulated_spectra( + n_sims, version, tomography, tomo_bin_ids, "BB", out_dir, seed + ) + + np.save(outpath_cov, covariance_matrix) + print(f"BB covariance matrix saved to {outpath_cov}") diff --git a/scripts/glass_mock/make_glass_sim.py b/scripts/glass_mock/make_glass_sim.py new file mode 100644 index 00000000..2cbb5d16 --- /dev/null +++ b/scripts/glass_mock/make_glass_sim.py @@ -0,0 +1,417 @@ +"""Generate a GLASS mock source catalogue. + +Thin CLI wrapper around the generation core in ``sp_validation.glass_mock``: +this script owns argument parsing, the mask downgrade, galaxy sampling, and FITS +catalogue I/O. The reproducibility surface — the fixed cosmology, the +``sigma8``-rescaled CAMB power spectrum, and the lognormal matter / lensing-map +generation — lives in the library and is pinned by +``src/sp_validation/tests/test_glass_mock.py``. + +Run inside an image that has GLASS installed (the production sp_validation +container does not yet ship GLASS). +""" + +import argparse +import os +import time + +import camb +import fitsio + +# GLASS modules +import glass +import healpy as hp +import numpy as np +from tqdm import tqdm + +from sp_validation.glass_mock import ( + Cosmology_from_camb, + GlassMockConfig, + build_camb_params, + build_shells, + camb_sigma8, + downgrade_mask, + ia_convergence, + matter_shell_cls, + validate_number_density, + validate_shape_noise, +) + + +def get_parser(): + """Create the argument parser.""" + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="Creates a weak lensing catalogue simulation using GLASS", + fromfile_prefix_chars="@", + ) + parser.add_argument("-s", "--seed", help="Random seed", type=int, default=42) + parser.add_argument("-N", "--number", help="Mock Number", type=int, default=0) + parser.add_argument("-t", "--test", help="Test run", action="store_true") + parser.add_argument("-cb", "--camb", help="get Camb C_ell", action="store_true") + parser.add_argument( + "-v", "--validation", help="Run some validation checks", action="store_true" + ) + parser.add_argument( + "-c", + "--config", + help="Path to the configuration file to generate the simulation", + type=str, + required=True, + ) + + return parser + + +class Sky: + """Build a UNIONS GLASS mock from a :class:`GlassMockConfig` + CLI args.""" + + def __init__(self): + parser = get_parser() + args = parser.parse_args() + + yaml_config = args.config + + # Test mode shrinks the box; otherwise CLI overrides config fields. + if args.test: + print("[!!!] Running in test mode: ignore the configuration file [!!!]") + if not os.path.exists("config/glass_mock/test_data/"): + raise FileNotFoundError( + "The test must be run from the root of the repository." + ) + if not os.path.exists("config/glass_mock/test_data/mask.fits"): + raise FileNotFoundError( + "Check that you downloaded the test mask from the repository." + ) + if not os.path.exists( + "config/glass_mock/test_data/redshift_distribution.txt" + ): + raise FileNotFoundError( + "Check that you downloaded the test redshift distribution from the repository." + ) + + self.config = GlassMockConfig.from_planck18( + nside=32, + seed=args.seed, + n_arcmin2=0.0824, + dx=120.0, + zmax=0.5, + sigma_e=0.26, + ia_bias=None, + mask_path="config/glass_mock/test_data/mask.fits", + nz_path="config/glass_mock/test_data/redshift_distribution.txt", + output_path="config/glass_mock/test_data", + output_prefix="test", + ) + else: + # Load the configuration from the config file in the parser + self.config = GlassMockConfig.from_yaml( + yaml_config=yaml_config, seed=args.seed + ) + + # Check consistency of the config for the tomographic bins + self.config.check_consistency() + + # Runtime options + self.test = args.test + self.camb = args.camb + self.validation = args.validation + self.limber = self.config.limber + + # Number label and random seed + self.number = args.number + self.n_sim = str(args.number).zfill(5) + self.rng = np.random.default_rng(self.config.seed) + + # Input paths + self.path_mask = self.config.mask_path + self.path_nz = self.config.nz_path + + # Output path + self.path = self.config.output_path + self.prefix = self.config.output_prefix + + print("-" * 66) + print( + f"Creating a weak lensing catalogue GLASS simulation with NSide = {self.config.nside}" + ) + print(f"Mock number: {self.number}") + print(f"Random seed: {self.config.seed}") + if self.test: + print("[!] Running in test mode [!]") + print("-" * 66) + + self.root = f"{self.path}/results" + if not os.path.exists(self.root): + os.makedirs(self.root) + print("A new directory " + str(self.root) + " is created!") + + # Cosmology + CAMB params come from the library core. + self.pars = build_camb_params(self.config) + print("-" * 66) + print("Parameters used for the simulation:") + print(f"H0: {self.pars.H0}") + print(f"omch2: {self.pars.omch2}") + print(f"ombh2: {self.pars.ombh2}") + print(f"n_s: {self.pars.InitPower.ns}") + print(f"As: {self.pars.InitPower.As}") + print(f"sigma_8: {camb_sigma8(self.pars)}") + print(f"Omega_m: {self.pars.omegam}") + print(f"ia_bias: {self.config.ia_bias}") + print("-" * 66) + print(f"number density: {self.config.n_arcmin2}") + print(f"shape noise: {self.config.sigma_e}") + print(f"galaxy bias: {self.config.bias}") + print("-" * 66) + + self.z = None + self.bin_nz = [] + self.ngal_per_bin = [] + self.camb_cls = None + + def read_mask(self): + """Read the survey mask from the specified path.""" + mask = hp.read_map(self.path_mask) + mask_file_name = f"{self.path}/mask_nside_{self.config.nside}.fits" + if not os.path.isfile(mask_file_name): + os.makedirs(self.path, exist_ok=True) + mask = downgrade_mask(mask, self.config.nside) + hp.write_map(mask_file_name, mask, overwrite=True) + else: + mask = hp.read_map(mask_file_name).astype(np.float64) + return mask + + def get_cl_matter_shells(self, shells): + """Get the matter power spectrum for the simulation.""" + cls_path = f"{self.root}/cls_{self.config.nside}_limber_{self.limber}.npy" + try: + cls = np.load(cls_path) + except FileNotFoundError: + print(f"[!] cls file {cls_path} missing; computing matter cls ...") + cls = matter_shell_cls(self.config, self.pars, shells, limber=self.limber) + np.save(cls_path, cls) + print("[!] Done.") + return cls + + def read_redshift_distributions(self, shells): + """Read the redshift distributions from the specified path. + + The convention used here is that the non-tomographic redshift distribution sums to one. + All the per bin distributions will sum to a lower value than 1. + """ + nz = np.loadtxt(self.path_nz) + nz = nz[nz[:, 0] <= self.config.zmax] + self.z = nz[:, 0] + + # Check that the number of nz columns corresponds to what is expected. + dndz_cols = nz[:, 1:] + + # Check that the per bin integrals sum together to one + per_bin_integral = np.trapezoid(dndz_cols, self.z.T, axis=0) + assert np.isclose(per_bin_integral.sum(), 1.0, atol=1e-2), ( + f"Per bin integrals do not sum to one: {per_bin_integral.sum()}. Integrals per bin: {per_bin_integral}" + ) + + for b in range(self.config.nbins): + n_arcmin2 = ( + self.config.n_arcmin2[b] + if isinstance(self.config.n_arcmin2, list) + else self.config.n_arcmin2 + ) + dndz_b = ( + dndz_cols[:, b] + * n_arcmin2 + / per_bin_integral[ + b + ] # Normalize the per bin redshift distribution to have the correct number density + ) + ngal_b = glass.partition(self.z, dndz_b, shells) + self.ngal_per_bin.append(ngal_b) + self.bin_nz.append(dndz_b) + + def galaxies_simulation(self): + """Create a source catalogue from the GLASS mock maps.""" + config = self.config + + # Read + downgrade the survey mask. + mask = self.read_mask() + + # Shells + matter spectra from the library core (cached cls on disk). + shells = build_shells(config, self.pars) + + # Generate the file for the matter shells if not done already + cls = self.get_cl_matter_shells(shells) + + # Apply discretisation to the full set of spectra + cls = glass.discretized_cls(cls, nside=config.nside, lmax=config.lmax, ncorr=3) + + # Generate the matter maps + fields = glass.lognormal_fields(shells) + gls = glass.solve_gaussian_spectra(fields, cls) + matter = glass.generate(fields, gls, config.nside, ncorr=3, rng=self.rng) + convergence = glass.MultiPlaneConvergence(Cosmology_from_camb(self.pars)) + + # n(z) and per-shell galaxy partition. + self.read_redshift_distributions(shells) + + # Name of the output file + out_file = ( + f"{self.root}/{self.prefix}_glass_sim_{self.n_sim}_{config.nside}.fits" + ) + + # Open the FITS file to save the catalogue + fits = fitsio.FITS(out_file, "rw", clobber=True) + fits.write(None) + fits.create_table_hdu( + names=[ + "RA", + "Dec", + "e1", + "e2", + "w", + "n1", + "n2", + "TOM_BIN_ID", + "TRUE_Z", + "PHOTO_Z", + ], + formats=["D", "D", "E", "E", "D", "E", "E", "J", "D", "D"], + extname="SOURCE_CATALOGUE", + ) + cat_dtype = fits["SOURCE_CATALOGUE"].get_rec_dtype()[0] + + ngal_tot = 0 + c = 0 + print("Generating shell ", end="") + for i, delta_i in tqdm(enumerate(matter)): + convergence.add_window(delta_i, shells[i]) + kappa_i = convergence.kappa + if config.ia_bias is not None: + kappa_i = kappa_i + ia_convergence(delta_i, shells[i], config) + gamm1_i, gamm2_i = glass.shear_from_convergence(kappa_i) + + # Sample each tomographic bin against the same matter/convergence + # realisation for this shell + for b in range(config.nbins): + bias_b = ( + config.bias[b] if isinstance(config.bias, list) else config.bias + ) + sigma_e = ( + config.sigma_e[b] + if isinstance(config.sigma_e, list) + else config.sigma_e + ) + + for gal_lon, gal_lat, gal_count in glass.points.positions_from_delta( + self.ngal_per_bin[b][i], delta_i, bias_b, mask, rng=self.rng + ): + ngal_tot += gal_count + gal_z = glass.redshifts(gal_count, shells[i], rng=self.rng) + gal_phz = glass.gaussian_phz( + gal_z, config.phz_sigma_0, rng=self.rng + ) + + gal_ellip = glass.ellipticity_intnorm( + gal_count, + sigma_e, + rng=self.rng, + xp=np, + ) + gal_she = glass.galaxy_shear( + gal_lon, gal_lat, gal_ellip, kappa_i, gamm1_i, gamm2_i + ) + noise_she = glass.galaxies.galaxy_shear( + gal_lon, + gal_lat, + gal_ellip, + np.zeros(np.shape(kappa_i)), + np.zeros(np.shape(gamm1_i)), + np.zeros(np.shape(gamm2_i)), + ) + + catalogue = np.empty(gal_count, dtype=cat_dtype) + gal_lon[gal_lon < 0] += 360 + catalogue["RA"] = gal_lon + catalogue["Dec"] = gal_lat + catalogue["e1"] = gal_she.real + catalogue["e2"] = -gal_she.imag + catalogue["w"] = np.ones_like(gal_lon) + catalogue["n1"] = noise_she.real + catalogue["n2"] = noise_she.imag + catalogue["TOM_BIN_ID"] = b + 1 + catalogue["TRUE_Z"] = gal_z + catalogue["PHOTO_Z"] = gal_phz + fits["SOURCE_CATALOGUE"].append(catalogue) + + c += 1 + + print("[DONE] \n") + print(f"Total number of galaxies sampled: {ngal_tot} using {c} z-shells") + fits.close() + print("Saved simulation to: ", out_file) + + if self.camb: + print("-" * 66) + print("Compute the associated theory power spectra...") + self.get_camb_cls() + print("-" * 66) + + if self.validation: + print("-" * 66) + self.run_validation_checks(out_file, mask) + print("-" * 66) + + def get_camb_cls(self, sav=True): + """Lensing C_ell from CAMB source windows for the mock n(z).""" + if self.bin_nz is None or self.z is None: + print("ERROR: run galaxies_simulation() first to populate n(z)") + return None + lmax = self.config.lmax + sources = [ + camb.sources.SplinedSourceWindow( + z=self.z, W=self.bin_nz[i], source_type="lensing" + ) + for i in range(self.config.nbins) + ] + self.pars.SourceWindows = sources + + results = camb.get_results(self.pars) + cl_camb = results.get_source_cls_dict(lmax=lmax, raw_cl=True) + + dic = {"ell": np.arange(lmax + 1) + 1} + for key in cl_camb: + if "P" not in key: + a, b = key.replace("W", "").replace("x", "-").split("-") + dic[f"{int(a) - 1}-{int(b) - 1}"] = cl_camb[key] + if sav: + out = f"{self.root}/{self.prefix}_camb_cls_{self.n_sim}_{self.config.nside}.fits" + fits = fitsio.FITS(out, "rw", clobber=True) + fits.write(dic) + print("Saved CAMB power spectra to: ", out) + fits.close() + self.camb_cls = dic + return dic + + def run_validation_checks(self, out_file, mask): + print("Running validation checks...") + + cat_glass = fitsio.FITS(out_file)["SOURCE_CATALOGUE"].read() + + # First estimate the non-tomographic and tomographic shape noise. + validate_shape_noise(cat_glass) + + # Next, validate the number density. + validate_number_density(cat_glass, mask) + + print("Validation checks completed.") + + +if __name__ == "__main__": + print("Starting the simulation") + start_time = time.time() + new_sky = Sky() + print("--- init %s seconds ---" % (time.time() - start_time)) + + start_time = time.time() + new_sky.galaxies_simulation() + print("--- simulation %s seconds ---" % (time.time() - start_time)) diff --git a/scripts/glass_mock/make_unions_glass_sim.py b/scripts/glass_mock/make_unions_glass_sim.py deleted file mode 100644 index bc97abae..00000000 --- a/scripts/glass_mock/make_unions_glass_sim.py +++ /dev/null @@ -1,334 +0,0 @@ -"""Generate a UNIONS GLASS mock source catalogue. - -Thin CLI wrapper around the generation core in ``sp_validation.glass_mock``: -this script owns argument parsing, the mask downgrade, galaxy sampling, and FITS -catalogue I/O. The reproducibility surface — the fixed cosmology, the -``sigma8``-rescaled CAMB power spectrum, and the lognormal matter / lensing-map -generation — lives in the library and is pinned by -``src/sp_validation/tests/test_glass_mock.py``. - -Run inside an image that has GLASS installed (the production sp_validation -container does not yet ship GLASS). -""" - -import argparse -import os -import time - -import camb -import fitsio - -# GLASS modules -import glass -import healpy as hp -import numpy as np -from tqdm import tqdm - -from sp_validation.glass_mock import ( - GlassMockConfig, - build_camb_params, - build_shells, - camb_sigma8, - downgrade_mask, - ia_convergence, - matter_shell_cls, -) - - -def get_parser(): - """Create the argument parser.""" - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description="Creates a UNIONS simulation using GLASS", - fromfile_prefix_chars="@", - ) - parser.add_argument("-s", "--seed", help="Random seed", type=int, default=42) - parser.add_argument("-N", "--number", help="Mock Number", type=int, default=0) - parser.add_argument( - "-n", - "--nside", - help="Nside for the simulation. Nside=Lmax", - type=int, - default=32, - ) - parser.add_argument( - "-ne", - "--neff", - help="Effective number of galaxies per arcmin^2", - type=float, - default=6.0905, - ) - parser.add_argument( - "-p", - "--path", - help="Output path to save the mocks", - type=str, - default="./", - ) - parser.add_argument( - "-c", - "--cls", - help="Pre-compute and saves the matter shell cls", - action="store_true", - ) - parser.add_argument("-t", "--test", help="Test run", action="store_true") - parser.add_argument( - "-sg", - "--sigmae", - help="Set sigma of intrinsic ellipticity", - type=float, - default=0.2684, - ) - parser.add_argument("-cb", "--camb", help="get Camb C_ell", action="store_true") - parser.add_argument( - "-nz", - "--pathnz", - help="Path to the n(z) file", - type=str, - default="/n17data/sguerrini/UNIONS/WL/nz/v1.4.6.3/nz_SP_v1.4.6.3_A.txt", - ) - parser.add_argument( - "-m", - "--mask", - help="Path to the mask", - type=str, - default="/n09data/guerrini/glass_mock_v1.4.6_rerun/mask_nside4096.fits", - ) - parser.add_argument( - "-ia", - "--ia_bias", - help="Intrinsic alignment bias for CCL", - type=float, - default=None, - ) - return parser - - -class Sky: - """Build a UNIONS GLASS mock from a :class:`GlassMockConfig` + CLI args.""" - - def __init__(self): - parser = get_parser() - args = parser.parse_args() - - # Test mode shrinks the box; otherwise CLI overrides config fields. - if args.test: - print("[!!!] Running in test mode [!!!]") - self.config = GlassMockConfig.from_planck18( - nside=32, - seed=args.seed, - n_arcmin2=0.0824, - dx=120.0, - zmax=0.5, - sigma_e=args.sigmae, - ia_bias=args.ia_bias, - ) - else: - self.config = GlassMockConfig.from_planck18( - nside=args.nside, - seed=args.seed, - n_arcmin2=args.neff, - sigma_e=args.sigmae, - ia_bias=args.ia_bias, - ) - - self.test = args.test - self.camb = args.camb - self.pre_cls = args.cls - self.number = args.number - self.n_sim = str(args.number).zfill(5) - self.path = args.path - self.path_mask = args.mask - self.path_nz = args.pathnz - self.rng = np.random.default_rng(self.config.seed) - - print("-" * 66) - print(f"Creating a UNIONS GLASS simulation with NSide = {self.config.nside}") - print(f"Mock number: {self.number}") - print(f"Random seed: {self.config.seed}") - print(f"Test mode or not: {self.test}") - print("-" * 66) - - self.root = f"{self.path}/results" - if not os.path.exists(self.root): - os.makedirs(self.root) - print("A new directory " + str(self.root) + " is created!") - - # Cosmology + CAMB params come from the library core. - self.pars = build_camb_params(self.config) - print("-" * 66) - print("Parameters used for the simulation:") - print(f"H0: {self.pars.H0}") - print(f"omch2: {self.pars.omch2}") - print(f"ombh2: {self.pars.ombh2}") - print(f"n_s: {self.pars.InitPower.ns}") - print(f"As: {self.pars.InitPower.As}") - print(f"sigma_8: {camb_sigma8(self.pars)}") - print(f"Omega_m: {self.pars.omegam}") - print(f"ia_bias: {self.config.ia_bias}") - print("-" * 66) - - self.z = None - self.bin_nz = None - self.camb_cls = None - - def galaxies_simulation(self): - """Create a UNIONS source catalogue from the GLASS mock maps.""" - config = self.config - - # Read + downgrade the survey mask. - unions_mask = hp.read_map(self.path_mask) - mask_file_name = f"{self.path}/mask_nside{config.nside}.fits" - if not os.path.isfile(mask_file_name): - os.makedirs(self.path, exist_ok=True) - unions_mask = downgrade_mask(unions_mask, config.nside) - hp.write_map(mask_file_name, unions_mask, overwrite=True) - else: - unions_mask = hp.read_map(mask_file_name).astype(np.float64) - - # Shells + matter spectra from the library core (cached cls on disk). - shells = build_shells(config, self.pars) - cls_path = f"{self.root}/cls_{config.nside}.npy" - try: - cls = np.load(cls_path) - except FileNotFoundError: - print(f"[!] cls file {cls_path} missing; computing matter cls ...") - cls = matter_shell_cls(config, self.pars, shells) - np.save(cls_path, cls) - print("[!] Done.") - - fields = glass.lognormal_fields(shells) - gls = glass.solve_gaussian_spectra(fields, cls) - matter = glass.generate(fields, gls, config.nside, ncorr=3, rng=self.rng) - convergence = glass.MultiPlaneConvergence(_cosmology(self.pars)) - - # n(z) and per-shell galaxy partition. - nz = np.loadtxt(self.path_nz) - nz = nz[nz[:, 0] <= config.zmax] - self.z = nz[:, 0] - dndz = nz[:, 1] * config.n_arcmin2 - ngal = glass.partition(self.z, dndz, shells) - zbins = glass.equal_dens_zbins(self.z, dndz, nbins=config.nbins) - - out_file = f"{self.root}/unions_glass_sim_{self.n_sim}_{config.nside}.fits" - fits = fitsio.FITS(out_file, "rw", clobber=True) - fits.write(None) - fits.create_table_hdu( - names=[ - "RA", - "Dec", - "e1", - "e2", - "w", - "n1", - "n2", - "TOM_BIN_ID", - "TRUE_Z", - "PHOTO_Z", - ], - formats=["D", "D", "E", "E", "D", "E", "E", "J", "D", "D"], - extname="SOURCE_CATALOGUE", - ) - cat_dtype = fits["SOURCE_CATALOGUE"].get_rec_dtype()[0] - - ngal_tot = 0 - c = 0 - print("Generating shell ", end="") - for i, delta_i in tqdm(enumerate(matter)): - convergence.add_window(delta_i, shells[i]) - kappa_i = convergence.kappa - if config.ia_bias is not None: - kappa_i = kappa_i + ia_convergence(delta_i, shells[i], config) - gamm1_i, gamm2_i = glass.shear_from_convergence(kappa_i) - - for gal_lon, gal_lat, gal_count in glass.points.positions_from_delta( - ngal[i], delta_i, config.bias, unions_mask, rng=self.rng - ): - ngal_tot += gal_count - gal_z = glass.redshifts(gal_count, shells[i], rng=self.rng) - gal_phz = glass.gaussian_phz(gal_z, config.phz_sigma_0, rng=self.rng) - tomo_id = np.digitize(gal_phz, np.unique(zbins)) - 1 - gal_ellip = glass.ellipticity_intnorm( - gal_count, config.sigma_e, rng=self.rng - ) - gal_she = glass.galaxy_shear( - gal_lon, gal_lat, gal_ellip, kappa_i, gamm1_i, gamm2_i - ) - noise_she = glass.galaxies.galaxy_shear( - gal_lon, - gal_lat, - gal_ellip, - np.zeros(np.shape(kappa_i)), - np.zeros(np.shape(gamm1_i)), - np.zeros(np.shape(gamm2_i)), - ) - - catalogue = np.empty(gal_count, dtype=cat_dtype) - catalogue["RA"] = gal_lon - catalogue["Dec"] = gal_lat - catalogue["e1"] = gal_she.real - catalogue["e2"] = -gal_she.imag - catalogue["w"] = np.ones_like(gal_lon) - catalogue["n1"] = noise_she.real - catalogue["n2"] = noise_she.imag - catalogue["TOM_BIN_ID"] = tomo_id - catalogue["TRUE_Z"] = gal_z - catalogue["PHOTO_Z"] = gal_phz - fits["SOURCE_CATALOGUE"].append(catalogue) - c += 1 - - print("[DONE] \n") - print(f"Total number of galaxies sampled: {ngal_tot} using {c} z-shells") - fits.close() - print("Saved simulation to: ", out_file) - - if self.camb: - self.get_camb_cls() - - def get_camb_cls(self, sav=True): - """Lensing C_ell from CAMB source windows for the mock n(z).""" - if self.bin_nz is None or self.z is None: - print("ERROR: run galaxies_simulation() first to populate n(z)") - return None - lmax = self.config.lmax - sources = [ - camb.sources.SplinedSourceWindow( - z=self.z, W=self.bin_nz[i], source_type="lensing" - ) - for i in range(self.config.nbins) - ] - self.pars.SourceWindows = sources - self.pars.InitPower.set_params(As=2.1e-9, ns=0.965) - results = camb.get_results(self.pars) - cl_camb = results.get_source_cls_dict(lmax=lmax, raw_cl=True) - - dic = {"ell": np.arange(lmax + 1) + 1} - for key in cl_camb: - if "P" not in key: - a, b = key.replace("W", "").replace("x", "-").split("-") - dic[f"{int(a) - 1}-{int(b) - 1}"] = cl_camb[key] - if sav: - out = f"{self.root}/camb_cls.fits" - fits = fitsio.FITS(out, "rw", clobber=True) - fits.write(dic) - fits.close() - self.camb_cls = dic - return dic - - -def _cosmology(pars): - """GLASS cosmology wrapper from CAMB params (lazy import).""" - from cosmology import Cosmology - - return Cosmology.from_camb(pars) - - -if __name__ == "__main__": - print("Starting the simulation") - start_time = time.time() - new_sky = Sky() - print("--- init %s seconds ---" % (time.time() - start_time)) - - start_time = time.time() - new_sky.galaxies_simulation() - print("--- simulation %s seconds ---" % (time.time() - start_time)) diff --git a/scripts/xip_xim.py b/scripts/xip_xim.py index 1345f01a..763752ce 100755 --- a/scripts/xip_xim.py +++ b/scripts/xip_xim.py @@ -20,6 +20,8 @@ def params_default(): params = { "input_path": "shape_catalog_ngmix.fits", + "tomo_bin1": None, + "tomo_bin2": None, "key_ra": "RA", "key_dec": "DEC", "key_e1": "e1", @@ -30,20 +32,27 @@ def params_default(): "theta_max": 200, "n_theta": 20, "output_path": "./xip_xim.txt", + "key_tomo_bin_col": "tomo_bin_id", } short_options = { "input_path": "-i", "output_path": "-o", + "tomo_bin1": "-b1", + "tomo_bin2": "-b2", } types = { "sign_e1": "int", "sign_e2": "int", + "tomo_bin1": "int", + "tomo_bin2": "int", } help_strings = { "input_path": "shear catalogue input path, default={}", + "tomo_bin1": "First tomographic bin, default={} (non-tomographic)", + "tomo_bin2": "Second tomographic bin, default={} (non-tomographic)", "key_ra": "column name for right ascension, default={}", "key_dec": "column name for declination, default={}", "key_e1": "column name for ellipticity component 1, default={}", @@ -54,6 +63,7 @@ def params_default(): "theta_max": "maximum angular scale [arcmin], default={}", "n_theta": "number of angular scales, default={}", "output_path": "output path, default={}", + "key_tomo_bin_col": "column name for tomography bin, default={}", } return params, short_options, types, help_strings @@ -132,18 +142,52 @@ def main(argv=None): "Signs for ellipticity components =" + f" ({params['sign_e1']:+d}, {params['sign_e2']:+d})" ) - g1 = data[params["key_e1"]] * params["sign_e1"] - g2 = data[params["key_e2"]] * params["sign_e2"] - cat = treecorr.Catalog( - ra=data[params["key_ra"]], - dec=data[params["key_dec"]], - g1=g1, - g2=g2, - w=data["w"], - ra_units=coord_units, - dec_units=coord_units, - ) - + tomographic = params["tomo_bin1"] is not None and params["tomo_bin2"] is not None + cat2 = None + if tomographic: + if params["verbose"]: + print( + f"Calculating 2PCF for tomographic bins {params['tomo_bin1']} and {params['tomo_bin2']}" + ) + tomo_bin1_idx = data[params["key_tomo_bin_col"]] == params["tomo_bin1"] + tomo_bin2_idx = data[params["key_tomo_bin_col"]] == params["tomo_bin2"] + + g1 = data[params["key_e1"]] * params["sign_e1"] + g2 = data[params["key_e2"]] * params["sign_e2"] + + cat1 = treecorr.Catalog( + ra=data[params["key_ra"]][tomo_bin1_idx], + dec=data[params["key_dec"]][tomo_bin1_idx], + g1=g1[tomo_bin1_idx], + g2=g2[tomo_bin1_idx], + w=data["w"][tomo_bin1_idx], + ra_units=coord_units, + dec_units=coord_units, + ) + if params["tomo_bin1"] != params["tomo_bin2"]: + cat2 = treecorr.Catalog( + ra=data[params["key_ra"]][tomo_bin2_idx], + dec=data[params["key_dec"]][tomo_bin2_idx], + g1=g1[tomo_bin2_idx], + g2=g2[tomo_bin2_idx], + w=data["w"][tomo_bin2_idx], + ra_units=coord_units, + dec_units=coord_units, + ) + else: + if params["verbose"]: + print("Calculating non-tomographic 2PCF") + g1 = data[params["key_e1"]] * params["sign_e1"] + g2 = data[params["key_e2"]] * params["sign_e2"] + cat1 = treecorr.Catalog( + ra=data[params["key_ra"]], + dec=data[params["key_dec"]], + g1=g1, + g2=g2, + w=data["w"], + ra_units=coord_units, + dec_units=coord_units, + ) # Set treecorr config info for correlation sep_units = "arcmin" TreeCorrConfig = { @@ -159,7 +203,7 @@ def main(argv=None): # Compute correlation if params["verbose"]: print("Correlating...") - gg.process(cat, cat) + gg.process(cat1, cat2=cat2) # Write to file if params["verbose"]: diff --git a/src/sp_validation/cosmo_val/catalog_characterization.py b/src/sp_validation/cosmo_val/catalog_characterization.py index 8f8ecd2e..e2df6d8e 100644 --- a/src/sp_validation/cosmo_val/catalog_characterization.py +++ b/src/sp_validation/cosmo_val/catalog_characterization.py @@ -148,13 +148,17 @@ def area(self): @property def n_eff_gal(self): if not hasattr(self, "_n_eff_gal"): - self.calculate_n_eff_gal() + self.calculate_n_eff_gal(tomography=False) + if self.compute_tomography: + self.calculate_n_eff_gal(tomography=True) return self._n_eff_gal @property def ellipticity_dispersion(self): if not hasattr(self, "_ellipticity_dispersion"): - self.calculate_ellipticity_dispersion() + self.calculate_ellipticity_dispersion(tomography=False) + if self.compute_tomography: + self.calculate_ellipticity_dispersion(tomography=True) return self._ellipticity_dispersion def _get_binned_catalog_mask(self, ver): @@ -200,29 +204,81 @@ def calculate_area_from_binned_catalog(self, ver): return area - def calculate_n_eff_gal(self): + def calculate_n_eff_gal(self, tomography=False): self.print_start("Calculating effective number of galaxy") - n_eff_gal = {} + if not hasattr(self, "_n_eff_gal"): + self._n_eff_gal = {} for ver in self.versions: self.print_magenta(ver) + if ver not in self._n_eff_gal: + self._n_eff_gal[ver] = {} + + if tomography: + tomo_bin_ids, tomo_bin_pairs = self._get_tomo_bins(ver) + + if tomo_bin_ids is None or tomo_bin_pairs is None: + raise ValueError( + f"Version {ver} does not have tomography information." + ) + + else: + tomo_bin_ids, tomo_bin_pairs = ["all"], [("all", "all")] + with self.results[ver].temporarily_read_data(): w = self._read_shear_cols(ver, "w_col") - n_eff_gal[ver] = n_eff_density(w, self.area[ver]) - print(f"n_eff_gal = {n_eff_gal[ver]:.2f} gal./arcmin^-2") + for tomo_bin_id in tomo_bin_ids: + if tomo_bin_id == "all": + self._n_eff_gal[ver][f"tomo_bin_{tomo_bin_id}"] = n_eff_density( + w, self.area[ver] + ) + else: + tomo_bin = self._read_shear_cols(ver, "tomo_bin_col") + mask = tomo_bin == tomo_bin_id + self._n_eff_gal[ver][f"tomo_bin_{tomo_bin_id}"] = n_eff_density( + w[mask], self.area[ver] + ) + print( + f"n_eff_gal for tomo_bin_{tomo_bin_id} = {self._n_eff_gal[ver][f'tomo_bin_{tomo_bin_id}']:.2f} gal./arcmin^-2" + ) - self._n_eff_gal = n_eff_gal self.print_done("Effective number of galaxy calculation finished") - def calculate_ellipticity_dispersion(self): + def calculate_ellipticity_dispersion(self, tomography=False): self.print_start("Calculating ellipticity dispersion") - ellipticity_dispersion = {} + if not hasattr(self, "_ellipticity_dispersion"): + self._ellipticity_dispersion = {} for ver in self.versions: self.print_magenta(ver) + if ver not in self._ellipticity_dispersion: + self._ellipticity_dispersion[ver] = {} + + if tomography: + tomo_bin_ids, tomo_bin_pairs = self._get_tomo_bins(ver) + + if tomo_bin_ids is None or tomo_bin_pairs is None: + raise ValueError( + f"Version {ver} does not have tomography information." + ) + + else: + tomo_bin_ids, tomo_bin_pairs = ["all"], [("all", "all")] + with self.results[ver].temporarily_read_data(): e1, e2, w = self._read_shear_cols(ver, "e1_col", "e2_col", "w_col") - ellipticity_dispersion[ver] = ellipticity_dispersion_stat(e1, e2, w) - print(f"Ellipticity dispersion = {ellipticity_dispersion[ver]:.4f}") - self._ellipticity_dispersion = ellipticity_dispersion + for tomo_bin_id in tomo_bin_ids: + if tomo_bin_id == "all": + self._ellipticity_dispersion[ver][f"tomo_bin_{tomo_bin_id}"] = ( + ellipticity_dispersion_stat(e1, e2, w) + ) + else: + tomo_bin = self._read_shear_cols(ver, "tomo_bin_col") + mask = tomo_bin == tomo_bin_id + self._ellipticity_dispersion[ver][f"tomo_bin_{tomo_bin_id}"] = ( + ellipticity_dispersion_stat(e1[mask], e2[mask], w[mask]) + ) + print( + f"Ellipticity dispersion for tomo_bin_{tomo_bin_id} = {self._ellipticity_dispersion[ver][f'tomo_bin_{tomo_bin_id}']:.4f}" + ) def plot_footprints(self): self.print_start("Plotting footprints:") diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 57f172bf..4fbd2a89 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -1,5 +1,6 @@ # %% import copy +import itertools import os from pathlib import Path @@ -51,8 +52,18 @@ class _LeakageObject(run_object.LeakageObject): entries = None - def read_data(self): - self._dat = io.open_entry(self.entries["shear"]) + def read_data(self, selection=None): + """Read the shear entry, keeping the rows ``selection`` marks. + + ``selection`` is an optional boolean array over the rows of + ``io.open_entry(entries["shear"])``, in that order. + """ + dat = io.open_entry(self.entries["shear"]) + if selection is not None: + if len(selection) != len(dat): + raise ValueError("Selection array has different length than catalogue") + dat = dat[selection] + self._dat = dat # %% @@ -118,8 +129,13 @@ class CosmologyValidation( Output directory. If None, the catalog config's paths.output. rho_tau_method : {'lsq', 'mcmc'}, default 'lsq' Fitting method for PSF leakage systematics parameters. - cov_estimate_method : {'th', 'jk'}, default 'th' - Covariance estimation: 'th' for semi-analytic theory, 'jk' for jackknife. + cov_estimate_method : {'th', 'jk', 'sim'}, default 'th' + Covariance estimation: 'th' for semi-analytic theory, 'jk' for jackknife, + 'sim' for a covariance measured on simulations. + n_sim_cov : int, default 300 + Number of simulations the 'sim' rho/tau covariance was measured on. + The covariance file does not record it; it sets the Hartlap debiasing + of the inverse covariance in the PSF-leakage fits. compute_cov_rho : bool, default True Whether to compute covariance for rho statistics during PSF analysis. n_cov : int, default 100 @@ -154,7 +170,7 @@ class CosmologyValidation( Number of ell bins for pseudo-C_ell analysis (used with binning='powspace'). ell_step : int, default 10 Bin width in ell for linear binning (used with binning='linear'). - pol_factor : bool, default True + pol_factor : int, default -1 Apply polarization correction factor in pseudo-C_ell calculations. nrandom_cell : int, default 10 Number of random realizations for C_ell error estimation. @@ -163,6 +179,10 @@ class CosmologyValidation( noise debiasing, making those realizations reproducible run-to-run. cosmo_params : dict, optional Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. + compute_tomography : bool, default False + Whether to compute tomographic correlation functions and pseudo-C_ell. + force_run : bool, default False + If True, forces re-computation of results even if cached outputs exist. Attributes ---------- @@ -264,6 +284,7 @@ def __init__( cov_estimate_method="th", compute_cov_rho=True, n_cov=100, + n_sim_cov=300, theta_min=0.1, theta_max=250, nbins=20, @@ -280,7 +301,7 @@ def __init__( power=1 / 2, n_ell_bins=32, ell_step=10, - pol_factor=True, + pol_factor=-1, cell_method="map", noise_bias_method="analytic", fiducial_input_inka="coupled", @@ -288,11 +309,14 @@ def __init__( cell_seed=8192, path_onecovariance=None, cosmo_params=None, + compute_tomography=False, + force_run=False, ): self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method self.compute_cov_rho = compute_cov_rho self.n_cov = n_cov + self.n_sim_cov = n_sim_cov self.theta_min = theta_min self.theta_max = theta_max self.npatch = npatch @@ -308,7 +332,12 @@ def __init__( self.power = power self.n_ell_bins = n_ell_bins self.ell_step = ell_step + + # bool is an int subclass: True would pass `in (-1, 1)` and mean "no flip" + if isinstance(pol_factor, bool) or pol_factor not in (-1, 1): + raise ValueError(f"pol_factor must be -1 or 1, got {pol_factor!r}") self.pol_factor = pol_factor + self.nrandom_cell = nrandom_cell self.cell_seed = cell_seed self.cell_method = cell_method @@ -316,6 +345,8 @@ def __init__( self.fiducial_input_inka = fiducial_input_inka self.nside_mask = nside_mask self.path_onecovariance = path_onecovariance + self.compute_tomography = compute_tomography + self.force_run = force_run assert self.cell_method in ["map", "catalog"], ( "cell_method must be 'map' or 'catalog'" @@ -450,7 +481,8 @@ def _output_path(self, *parts): def sacc_nz(self, version): """Single-bin ``nz`` mapping ``{0: (z, nz)}`` for the SACC writers. - The round is single-bin, so the whole survey n(z) is bin 0. + The SACC parts carry the ("all", "all") pair only, so the whole-survey + n(z) of :meth:`get_redshift` is bin 0. """ return {0: tuple(self.get_redshift(version))} @@ -463,7 +495,12 @@ def sacc_metadata(self, version): } def get_redshift(self, version): - """Load redshift distribution for a catalog version. + """Load the whole-survey redshift distribution of a catalog version. + + The ``redshift_path`` file holds z in column 0 and one n(z) column per + tomographic bin (a single column for a non-tomographic version); the + whole-survey n(z) is their sum, as + ``read_redshift_distribution(version, is_tomography=False)`` returns it. Parameters ---------- @@ -477,7 +514,7 @@ def get_redshift(self, version): nz : ndarray n(z) probability density """ - return np.loadtxt(self.cc[version]["shear"]["redshift_path"], unpack=True) + return self.read_redshift_distribution(version, is_tomography=False) def _write_catalog_config(self): with self.catalog_config_path.open("w") as file: @@ -541,11 +578,20 @@ def results_objectwise(self): self._results_objectwise = self.init_results(objectwise=True) return self._results_objectwise - def basename(self, version, treecorr_config=None, npatch=None): + def basename( + self, + version, + tomo_bin_a="all", + tomo_bin_b=None, + treecorr_config=None, + npatch=None, + ): cfg = treecorr_config or self.treecorr_config patches = npatch or self.npatch + tomo_bin_a_str = f"tomo_bin_{tomo_bin_a}" + tomo_bin_b_str = f"_tomo_bin_{tomo_bin_b}" if tomo_bin_b is not None else "" return ( - f"{version}_minsep={cfg['min_sep']}" + f"{version}_{tomo_bin_a_str}{tomo_bin_b_str}_minsep={cfg['min_sep']}" f"_maxsep={cfg['max_sep']}" f"_nbins={cfg['nbins']}" f"_npatch={patches}" @@ -609,8 +655,9 @@ def colors(self): def summarize_bmodes(self, fiducial_scale_cut=(12, 83), versions=None): """Print and return B-mode PTE summary across all statistics. - Collects PTEs from pure E/B, COSEBIs, and pseudo-Cl at the specified - fiducial scale cut. Statistics that haven't been computed show '--'. + Collects the PTEs of the non-tomographic ``("all", "all")`` pair from + pure E/B, COSEBIs, and pseudo-Cl at the specified fiducial scale cut. + Statistics that haven't been computed show '--'. Parameters ---------- @@ -627,47 +674,105 @@ def summarize_bmodes(self, fiducial_scale_cut=(12, 83), versions=None): versions = versions or self.versions summary = {} cov_methods = set() + pair = "tomo_bin_all_tomo_bin_all" for ver in versions: row = {} # Pure E/B PTEs from stored results - if ver in self._pure_eb_results: - res = self._pure_eb_results[ver] - edges = (res["left_edges"], res["right_edges"]) + pure_eb = self._pure_eb_results.get(ver, {}).get(pair) + if pure_eb is not None: + edges = (pure_eb["left_edges"], pure_eb["right_edges"]) try: for stat in ("xip_B", "xim_B", "combined"): row[stat] = _get_pte_from_scale_cut( - res["pte_matrices"][stat], edges, fiducial_scale_cut + pure_eb["pte_matrices"][stat], edges, fiducial_scale_cut ) - except (KeyError, RuntimeError): + except RuntimeError: + # The scale cut selects no bin of this grid. pass - cov_methods.add(covariance_label(res["npatch"])) + cov_methods.add(covariance_label(pure_eb["npatch"])) # COSEBIs PTE from stored results - if ver in self._cosebis_results: - cosebis_res = self._cosebis_results[ver] - has_multi_scale_cuts = all(isinstance(k, tuple) for k in cosebis_res) - if has_multi_scale_cuts: - key = find_conservative_scale_cut_key( - cosebis_res, fiducial_scale_cut - ) - row["COSEBIS"] = cosebis_res[key]["pte_B"] - elif "pte_B" in cosebis_res: - row["COSEBIS"] = cosebis_res["pte_B"] - - # Pseudo-Cl BB PTE (_pseudo_cls is lazy; check existence without - # triggering computation) - if hasattr(self, "_pseudo_cls") and ver in self._pseudo_cls: - try: - cl_bb = self.pseudo_cls[ver]["pseudo_cl"]["BB"] - cov_bb = self.pseudo_cls[ver]["cov"]["COVAR_BB_BB"].data - _, _, row["C_l_BB"] = chi2_and_pte(cl_bb, cov_bb) - cov_methods.add("Gaussian (NaMaster)") - except (KeyError, AttributeError): - pass + cosebis = self._cosebis_results.get(ver, {}).get(pair) + if cosebis is not None: + if all(isinstance(k, tuple) for k in cosebis): + key = find_conservative_scale_cut_key(cosebis, fiducial_scale_cut) + row["COSEBIS"] = cosebis[key]["pte_B"] + else: + row["COSEBIS"] = cosebis["pte_B"] + + # Pseudo-Cl BB PTE, once both the spectrum and its covariance are + # loaded (_pseudo_cls is lazy; read it without triggering computation) + pseudo_cl = getattr(self, "_pseudo_cls", {}).get(ver, {}).get(pair, {}) + if "pseudo_cl" in pseudo_cl and "cov" in pseudo_cl: + cl_bb = pseudo_cl["pseudo_cl"]["BB"] + cov_bb = pseudo_cl["cov"]["COVAR_BB_BB"].data + _, _, row["C_l_BB"] = chi2_and_pte(cl_bb, cov_bb) + cov_methods.add("Gaussian (NaMaster)") summary[ver] = row print_bmode_summary(summary, fiducial_scale_cut, cov_methods) return summary + + def _get_tomo_bins(self, version): + """ + Return the tomo_bin_ids for a given version. If the version does not have tomography, return None. + + Returns + ------- + tomo_bin_ids : list or None + List of unique tomographic bin IDs for the version, or None if no tomography is available + tomo_bin_pairs : list of tuples or None + List of unique pairs of tomographic bin IDs (including self-pairs) for the version, or None if no tomography is available + """ + if "tomo_bin_col" in self.cc[version]["shear"]: + self.print_cyan( + f"Extracting tomography information from version {version}." + ) + cat_gal = io.open_entry(self.cc[version]["shear"]) + tomo_bin = cat_gal[self.cc[version]["shear"]["tomo_bin_col"]] + tomo_bin_ids = np.unique(tomo_bin) + tomo_bin_ids = tomo_bin_ids[ + tomo_bin_ids > 0 + ] # Exclude zero or negative bins + self.print_cyan( + f"Found {len(tomo_bin_ids)} tomographic bins for version {version}: {tomo_bin_ids}." + ) + + tomo_bin_pairs = list( + itertools.combinations_with_replacement(tomo_bin_ids, 2) + ) + return tomo_bin_ids, tomo_bin_pairs + else: + self.print_cyan(f"Version {version} does not have tomography information.") + return None, None + + def _get_tomo_bins_for_versions(self, versions, tomography): + """ + Return a dictionary of tomo_bin_ids and tomo_bin_pairs for each version in versions. + + Parameters + ---------- + versions : list of str + List of catalog version identifiers + tomography : bool + If True, assumes tomography else returns the format for non-tomographic versions. + + Returns + ------- + dict + Dictionary with version as key and a dictionary containing 'ids' and 'pairs' as values. + Example: {version1: {'ids': tomo_bin_ids1, 'pairs': tomo_bin_pairs1}, ...} + """ + tomo_bins = {} + for ver in versions: + if tomography: + tomo_bin_ids, tomo_bin_pairs = self._get_tomo_bins(ver) + else: + tomo_bin_ids, tomo_bin_pairs = ["all"], [("all", "all")] + + tomo_bins[ver] = {"ids": tomo_bin_ids, "pairs": tomo_bin_pairs} + + return tomo_bins diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index 312aadcd..079720bf 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -29,6 +29,7 @@ def calculate_cosebis( cov_path=None, scale_cuts=None, evaluate_all_scale_cuts=False, + compute_tomography=False, min_sep=None, max_sep=None, nbins=None, @@ -60,7 +61,8 @@ def calculate_cosebis( Number of COSEBIs modes to compute. Defaults to 10. cov_path : str, optional Path to theoretical covariance matrix. When provided, enables analytic - covariance calculation. + covariance calculation for a single pair. Cannot be combined with + ``compute_tomography=True``. scale_cuts : list of tuples, optional Explicit list of (min_theta, max_theta) scale cuts to evaluate. Overrides evaluate_all_scale_cuts when provided. @@ -68,6 +70,8 @@ def calculate_cosebis( If True, evaluates COSEBIs for all possible scale cut combinations using the reporting binning parameters. Ignored when scale_cuts is provided. Defaults to False. + compute_tomography : bool, optional + If True, computes COSEBIs for all tomographic bin combinations. Defaults to False.s min_sep : float, optional Minimum separation for reporting binning (only used when evaluate_all_scale_cuts=True). Defaults to self.treecorr_config["min_sep"]. @@ -88,6 +92,12 @@ def calculate_cosebis( """ self.print_start(f"Computing {version} COSEBIs") + if cov_path is not None and compute_tomography: + raise ValueError( + "cov_path holds a single ξ± covariance; it cannot serve every " + "tomographic bin pair." + ) + # Set up parameters with defaults npatch = npatch or self.npatch @@ -99,44 +109,61 @@ def calculate_cosebis( f"Computing fine-binned 2PCF with {nbins_int} bins from {min_sep_int} to " f"{max_sep_int} arcmin" ) - gg = self.calculate_2pcf(version, npatch=npatch, **treecorr_config) - if scale_cuts is not None: - # Explicit scale cuts provided - print(f"Evaluating {len(scale_cuts)} explicit scale cuts") - results = calculate_cosebis( - gg=gg, nmodes=nmodes, scale_cuts=scale_cuts, cov_path=cov_path - ) - elif evaluate_all_scale_cuts: - # Use reporting binning parameters or inherit from class config - binning = self._binning(min_sep, max_sep, nbins) - min_sep, max_sep, nbins = ( - binning["min_sep"], - binning["max_sep"], - binning["nbins"], - ) + ggs = self.calculate_2pcf_version( + version, + npatch=npatch, + compute_tomography=compute_tomography, + **treecorr_config, + ) + results = { + bin_key: None for bin_key in ggs.keys() + } # Initialize results dictionary - # Generate scale cuts using np.geomspace (no TreeCorr needed) - bin_edges = np.geomspace(min_sep, max_sep, nbins + 1) - generated_cuts = [ - (bin_edges[start], bin_edges[stop]) - for start in range(nbins) - for stop in range(start + 1, nbins + 1) - ] + for bin_key in ggs.keys(): + # LG TO-DO: account for tomographic scale-cuts + if scale_cuts is not None: + # Explicit scale cuts provided + print(f"Evaluating {len(scale_cuts)} explicit scale cuts") + results[bin_key] = calculate_cosebis( + gg=ggs[bin_key], + nmodes=nmodes, + scale_cuts=scale_cuts, + cov_path=cov_path, + ) + elif evaluate_all_scale_cuts: + # Use reporting binning parameters or inherit from class config + binning = self._binning(min_sep, max_sep, nbins) + min_sep, max_sep, nbins = ( + binning["min_sep"], + binning["max_sep"], + binning["nbins"], + ) - print(f"Evaluating {len(generated_cuts)} scale cut combinations") + # Generate scale cuts using np.geomspace (no TreeCorr needed) + bin_edges = np.geomspace(min_sep, max_sep, nbins + 1) + generated_cuts = [ + (bin_edges[start], bin_edges[stop]) + for start in range(nbins) + for stop in range(start + 1, nbins + 1) + ] - # Call b_modes function with scale cuts list - results = calculate_cosebis( - gg=gg, nmodes=nmodes, scale_cuts=generated_cuts, cov_path=cov_path - ) - else: - # Single scale cut behavior: use full range - results = calculate_cosebis( - gg=gg, nmodes=nmodes, scale_cuts=None, cov_path=cov_path - ) - # Extract single results dict from scale_cuts dictionary - results = next(iter(results.values())) + print(f"Evaluating {len(generated_cuts)} scale cut combinations") + + # Call b_modes function with scale cuts list + results[bin_key] = calculate_cosebis( + gg=ggs[bin_key], + nmodes=nmodes, + scale_cuts=generated_cuts, + cov_path=cov_path, + ) + else: + # Single scale cut behavior: use full range + results[bin_key] = calculate_cosebis( + gg=ggs[bin_key], nmodes=nmodes, scale_cuts=None, cov_path=cov_path + ) + # Extract single results dict from scale_cuts dictionary + results[bin_key] = next(iter(results[bin_key].values())) return results @@ -152,6 +179,7 @@ def plot_cosebis( cov_path=None, scale_cuts=None, # Explicit scale cuts evaluate_all_scale_cuts=False, # Grid-based scale cuts + compute_tomography=False, min_sep=None, max_sep=None, nbins=None, # Reporting binning @@ -187,6 +215,8 @@ def plot_cosebis( evaluate_all_scale_cuts : bool Whether to evaluate all scale cuts from reporting binning grid (default: False). Ignored when scale_cuts is provided. + compute_tomography : bool + Compute and plot each tomographic bin pair instead of the all-galaxy pair. min_sep, max_sep, nbins : float, float, int, optional Reporting binning parameters. Only used when evaluate_all_scale_cuts=True. fiducial_scale_cut : tuple, optional @@ -218,7 +248,7 @@ def plot_cosebis( # Get or calculate results for this version if results is None: # Calculate COSEBIs using instance method - results = self.calculate_cosebis( + results_tomo = self.calculate_cosebis( version, min_sep_int=min_sep_int, max_sep_int=max_sep_int, @@ -228,55 +258,66 @@ def plot_cosebis( cov_path=cov_path, scale_cuts=scale_cuts, evaluate_all_scale_cuts=evaluate_all_scale_cuts, + compute_tomography=compute_tomography, min_sep=min_sep, max_sep=max_sep, nbins=nbins, ) - - # Generate plots using specialized plotting functions - # Extract single result for plotting if multiple scale cuts were evaluated - multiple_scale_cuts = isinstance(results, dict) and all( - isinstance(k, tuple) for k in results - ) - if multiple_scale_cuts: - # Multiple scale cuts: use fiducial_scale_cut if provided, otherwise use - # full range (largest scale cut) - plot_results = results[ - find_conservative_scale_cut_key(results, fiducial_scale_cut) - if fiducial_scale_cut is not None - else max(results, key=lambda x: x[1] - x[0]) - ] + elif results and all( + isinstance(key, str) and key.startswith("tomo_bin_") for key in results + ): + results_tomo = results else: - # Single result - plot_results = results - - plot_cosebis_modes( - plot_results, - version, - out_stub + "_cosebis.png", - fiducial_scale_cut=fiducial_scale_cut, - ) - - plot_cosebis_covariance_matrix( - plot_results, version, var_method, out_stub + "_covariance.png" - ) + results_tomo = {"tomo_bin_all_tomo_bin_all": results} - # Generate scale cut heatmap if we have multiple scale cuts - if multiple_scale_cuts and len(results) > 1: - # Create temporary gg object with correct binning for mapping - treecorr_config_temp = self._binning(min_sep, max_sep, nbins) - gg_temp = self.calculate_2pcf( - version, npatch=npatch, **treecorr_config_temp + ggs_temp = None + for bin_key, bin_results in results_tomo.items(): + bin_stub = ( + out_stub + if bin_key == "tomo_bin_all_tomo_bin_all" + else f"{out_stub}_{bin_key}" + ) + multiple_scale_cuts = isinstance(bin_results, dict) and all( + isinstance(k, tuple) for k in bin_results ) + if multiple_scale_cuts: + plot_results = bin_results[ + find_conservative_scale_cut_key(bin_results, fiducial_scale_cut) + if fiducial_scale_cut is not None + else max(bin_results, key=lambda x: x[1] - x[0]) + ] + else: + plot_results = bin_results - plot_cosebis_scale_cut_heatmap( - results, - (gg_temp.left_edges, gg_temp.right_edges), + plot_cosebis_modes( + plot_results, version, - out_stub + "_scalecut_ptes.png", + bin_stub + "_cosebis.png", fiducial_scale_cut=fiducial_scale_cut, ) + plot_cosebis_covariance_matrix( + plot_results, version, var_method, bin_stub + "_covariance.png" + ) - # Save data products and store on instance - save_cosebis_results(results, out_stub + "_data.npz", fiducial_scale_cut) - self._cosebis_results[version] = results + if multiple_scale_cuts and len(bin_results) > 1: + if ggs_temp is None: + treecorr_config_temp = self._binning(min_sep, max_sep, nbins) + ggs_temp = self.calculate_2pcf_version( + version, + npatch=npatch, + compute_tomography=compute_tomography, + **treecorr_config_temp, + ) + gg_temp = ggs_temp[bin_key] + plot_cosebis_scale_cut_heatmap( + bin_results, + (gg_temp.left_edges, gg_temp.right_edges), + version, + bin_stub + "_scalecut_ptes.png", + fiducial_scale_cut=fiducial_scale_cut, + ) + + save_cosebis_results( + bin_results, bin_stub + "_data.npz", fiducial_scale_cut + ) + self._cosebis_results[version] = results_tomo diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 7c8406e2..686cda01 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -7,31 +7,44 @@ on pymaster (NaMaster), healpy, and OneCovariance. """ +import colorsys import configparser +import itertools import os import healpy as hp import matplotlib.pyplot as plt import numpy as np -import pymaster as nmt from astropy.io import fits -from cs_util.cosmo import get_theo_c_ell +from matplotlib.colors import to_rgb + +import sp_validation.pseudo_cl as spv_pseudo_cl from .. import sacc_io from ..io import open_entry -from ..pseudo_cl import ( - apply_random_rotation, - get_n_gal_map, - get_pseudo_cls_catalog, - get_pseudo_cls_map, - make_namaster_bin, -) from ..rho_tau import get_params_rho_tau -from ..statistics import chi2_and_pte, cov_from_one_covariance +from ..statistics import cov_from_one_covariance from .sacc_writers import BIN as SACC_BIN from .sacc_writers import pseudo_cl_to_sacc +def _apply_pixel_window_to_fiducial_cl(fiducial_cl, nside, cell_method): + """Apply ``pw²(ℓ)`` to fiducial spectra only for HEALPix-map measurements. + + Each spectrum array uses index ``ell``, so the window spans the matching + ``ell = 0..len(spectrum)-1`` grid. Catalogue spectra are point-sampled and + must remain unchanged. + """ + if cell_method != "map" or not fiducial_cl: + return fiducial_cl + + lmax = len(next(iter(fiducial_cl.values()))) - 1 + pixwin = hp.pixwin(nside, lmax=lmax) + return { + key: np.asarray(cl, dtype=float) * pixwin**2 for key, cl in fiducial_cl.items() + } + + def plot_pseudo_cl_spectrum(datasets, spectrum, output_path): """Two-panel ℓC_ℓ / C_ℓ figure for one spectrum across catalogue versions. @@ -72,250 +85,588 @@ def plot_pseudo_cl_spectrum(datasets, spectrum, output_path): class PseudoClMixin: + # ---------------- Pseudo-Cl properties ---------------- # @property def pseudo_cls(self): if not hasattr(self, "_pseudo_cls"): - self.calculate_pseudo_cl() - self.calculate_pseudo_cl_eb_cov() + self.calculate_pseudo_cl(compute_tomography=False) + self.calculate_pseudo_cl_inka_cov( + compute_tomography=False, load_all_block=True + ) + if self.compute_tomography: + self.calculate_pseudo_cl(compute_tomography=True) + self.calculate_pseudo_cl_inka_cov( + compute_tomography=True, load_all_block=True + ) return self._pseudo_cls @property def pseudo_cls_onecov(self): if not hasattr(self, "_pseudo_cls_onecov"): - self.calculate_pseudo_cl_onecovariance() + self.calculate_pseudo_cl_onecovariance(tomography=False) + if self.compute_tomography: + self.calculate_pseudo_cl_onecovariance(tomography=True) return self._pseudo_cls_onecov - def get_namaster_bin(self, lmin, lmax, b_lmax): - """Build NaMaster binning object (thin wrapper, state -> primitive).""" - return make_namaster_bin( - lmin, - lmax, - b_lmax, - self.binning, - ell_step=self.ell_step, - n_ell_bins=self.n_ell_bins, - power=self.power, - ) - - def get_variance_map(self, nside, e1, e2, w, unique_pix, idx_rep): + # ---------------- Pseudo-Cl calculation methods ---------------- # + def calculate_pseudo_cl(self, compute_tomography=True, out_path=None): """ - Create a variance map from the input catalog. + Compute the pseudo-Cl of a `CosmologyValidation` inputs with tomography. + + With ``compute_tomography=False`` the single ``("all", "all")`` pair is + computed and written as a SACC part, by default to + ``pseudo_cl_{ver}.sacc``; ``out_path`` overrides that destination (one + version only). With ``compute_tomography=True`` every tomographic bin + pair is written to its own FITS file and ``out_path`` is not accepted. """ + if out_path is not None: + if compute_tomography: + raise ValueError( + "calculate_pseudo_cl(out_path=...) names the non-tomographic " + "SACC part; call it with compute_tomography=False" + ) + if len(self.versions) != 1: + raise ValueError( + "calculate_pseudo_cl(out_path=...) writes one part to one " + f"path, but {len(self.versions)} versions are configured; " + "call per version" + ) + + out_dir = self._output_path("pseudo_cl") + os.makedirs(out_dir, exist_ok=True) + + if compute_tomography: + self.print_start("Computing tomographic pseudo-Cl's") + else: + self.print_start("Computing non-tomographic pseudo-Cl's") + + self._pseudo_cls = getattr(self, "_pseudo_cls", {}) + + for ver in self.versions: + self.print_magenta(ver) + + if ver not in self.pseudo_cls.keys(): + self._pseudo_cls[ver] = {} - variance_map = np.zeros(hp.nside2npix(nside)) + if compute_tomography: + tomo_bin_ids, tomo_bin_pairs = self._get_tomo_bins(ver) - variance_map[unique_pix] = np.bincount( - idx_rep, weights=(e1**2 + e2**2) / 2 * w**2 + if tomo_bin_ids is None or tomo_bin_pairs is None: + raise ValueError( + f"Version {ver} does not have tomography information." + ) + + else: + tomo_bin_pairs = [("all", "all")] + + # Loop on the different tomographic bin pairs + for bin_key1, bin_key2 in tomo_bin_pairs: + self.print_cyan(f"Tomo Bin Pair: ({bin_key1}, {bin_key2})") + + if ( + f"tomo_bin_{bin_key1}_tomo_bin_{bin_key2}" + not in self._pseudo_cls[ver].keys() + ): + self._pseudo_cls[ver][ + f"tomo_bin_{bin_key1}_tomo_bin_{bin_key2}" + ] = {} + + pair_out_path = out_path or self._output_path_pseudo_cl( + ver, tomo_bin_pair=(bin_key1, bin_key2) + ) + if os.path.exists(pair_out_path) and not self.force_run: + self.print_done( + f"Skipping Pseudo-Cl's calculation, {pair_out_path} exists" + ) + self._pseudo_cls[ver][f"tomo_bin_{bin_key1}_tomo_bin_{bin_key2}"][ + "pseudo_cl" + ] = self._load_pseudo_cl(pair_out_path, (bin_key1, bin_key2)) + continue + + if self.cell_method == "map": + self.calculate_pseudo_cl_map( + ver, self.nside, pair_out_path, bin_key1, bin_key2 + ) + elif self.cell_method == "catalog": + self.calculate_pseudo_cl_catalog( + ver, pair_out_path, bin_key1, bin_key2 + ) + else: + raise ValueError(f"Unknown cell method: {self.cell_method}") + + self.print_done("Done pseudo-Cl's") + + def calculate_pseudo_cl_map(self, ver, nside, out_path, tomo_bin_a, tomo_bin_b): + assert (tomo_bin_a == "all" and tomo_bin_b == "all") or ( + isinstance(tomo_bin_a, (int, np.integer)) + and isinstance(tomo_bin_b, (int, np.integer)) + ), "tomo_bin_a and tomo_bin_b must be either both 'all' or both integers." + + params = get_params_rho_tau(self.cc[ver]) + + self.print_cyan( + f"Computing pseudo-Cl's for tomographic bins {tomo_bin_a} and {tomo_bin_b}..." ) - return variance_map + # Load data and create shear and noise maps + cat_gal = open_entry(self.cc[ver]["shear"]) - def get_field_and_workspace_from_map(self, mask, lmax, b): - """ - Create a NaMaster field and workspace from the input map. - """ + # Get the tomographic bin + cat_gal_a = self._get_tomographic_bin(params, cat_gal, tomo_bin_a) + cat_gal_b = self._get_tomographic_bin(params, cat_gal, tomo_bin_b) + + del cat_gal + + self.print_cyan("Creating maps and computing Cl's...") + # Get the pixels and indices for the catalogs + unique_pix_a, idx_a, idx_rep_a = self.get_pixels(params, nside, cat_gal_a) + unique_pix_b, idx_b, idx_rep_b = self.get_pixels(params, nside, cat_gal_b) + + # Create number density maps for each tomographic bin + n_gal_map_a = self.get_n_gal_map( + params, + nside, + cat_gal_a, + unique_pix=unique_pix_a, + idx=idx_a, + idx_rep=idx_rep_a, + ) + n_gal_map_b = self.get_n_gal_map( + params, + nside, + cat_gal_b, + unique_pix=unique_pix_b, + idx=idx_b, + idx_rep=idx_rep_b, + ) - nside = hp.npix2nside(len(mask)) + # Create shear maps for each tomographic bin + shear_map_a_e1, shear_map_a_e2 = self.get_shear_map( + params, + nside, + cat_gal_a, + unique_pix=unique_pix_a, + idx=idx_a, + idx_rep=idx_rep_a, + n_gal_map=n_gal_map_a, + ) + shear_map_a = shear_map_a_e1 + 1j * shear_map_a_e2 + del shear_map_a_e1, shear_map_a_e2 - # Create NaMaster field - f = nmt.NmtField( - mask=mask, - maps=[np.zeros(hp.nside2npix(nside)), np.zeros(hp.nside2npix(nside))], - lmax=lmax, + shear_map_b_e1, shear_map_b_e2 = self.get_shear_map( + params, + nside, + cat_gal_b, + unique_pix=unique_pix_b, + idx=idx_b, + idx_rep=idx_rep_b, + n_gal_map=n_gal_map_b, ) + shear_map_b = shear_map_b_e1 + 1j * shear_map_b_e2 + del shear_map_b_e1, shear_map_b_e2 - # Create NaMaster workspace - wsp = nmt.NmtWorkspace.from_fields(f, f, b) + # Compute the pseudo-Cl's + ell_eff, cl_shear, wsp = self.get_pseudo_cls_map( + shear_map_a, n_gal_map_a, shear_map_b=shear_map_b, mask_b=n_gal_map_b + ) - return f, wsp + # Remove the noise bias for auto-correlations. + if tomo_bin_a == tomo_bin_b: + # Compute the noise bias using noise_bias_method + cl_noise = self.get_noise_bias_from_gaussian_real( + params, + nside, + cat_gal_a, + unique_pix=unique_pix_a, + idx=idx_a, + idx_rep=idx_rep_a, + n_gal_map=n_gal_map_a, + wsp=wsp, + ) - def calculate_pseudo_cl_eb_cov(self): - """ - Compute a theoretical Gaussian covariance of the Pseudo-Cl for EE, EB and BB. + # Subtract the noise bias from the pseudo-Cl's + cl_shear = cl_shear - cl_noise + + self.print_cyan("Saving pseudo-Cl's...") + tomo_bin_pair = (tomo_bin_a, tomo_bin_b) + self._save_pseudo_cl( + ver, out_path, tomo_bin_pair, ell_eff, cl_shear, wsp, nside=nside + ) + + self._pseudo_cls[ver][f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}"][ + "pseudo_cl" + ] = self._load_pseudo_cl(out_path, tomo_bin_pair) + + def calculate_pseudo_cl_catalog(self, ver, out_path, tomo_bin_a, tomo_bin_b): + assert (tomo_bin_a == "all" and tomo_bin_b == "all") or ( + isinstance(tomo_bin_a, (int, np.integer)) + and isinstance(tomo_bin_b, (int, np.integer)) + ), "tomo_bin_a and tomo_bin_b must be either both 'all' or both integers." + + params = get_params_rho_tau(self.cc[ver]) + + # Load data and create shear and noise maps + cat_gal = open_entry(self.cc[ver]["shear"]) + + ell_eff, cl_shear, wsp = self.get_pseudo_cls_catalog( + catalog=cat_gal, params=params, tomo_bin_a=tomo_bin_a, tomo_bin_b=tomo_bin_b + ) + + self.print_cyan("Saving pseudo-Cl's...") + tomo_bin_pair = (tomo_bin_a, tomo_bin_b) + self._save_pseudo_cl(ver, out_path, tomo_bin_pair, ell_eff, cl_shear, wsp) + + self._pseudo_cls[ver][f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}"][ + "pseudo_cl" + ] = self._load_pseudo_cl(out_path, tomo_bin_pair) + + def calculate_pseudo_cl_inka_cov( + self, compute_tomography=True, load_all_block=False + ): + """Compute the Gaussian iNKA covariance for EE, EB, and BB. + + Apply the HEALPix pixel window ``pw²(ℓ)`` to each fiducial spectrum only + when ``cell_method == "map"``. Catalogue-based spectra have no pixel + window and use the unmodified fiducial. """ self.print_start("Computing Pseudo-Cl covariance") nside = self.nside - try: - self._pseudo_cls - except AttributeError: - self._pseudo_cls = {} + self._pseudo_cls = getattr(self, "_pseudo_cls", {}) for ver in self.versions: self.print_magenta(ver) + out_dir_block = self._output_path(f"pseudo_cl/iNKA_block_{ver}/") + os.makedirs(out_dir_block, exist_ok=True) + if ver not in self._pseudo_cls.keys(): self._pseudo_cls[ver] = {} - out_path = self._output_path(f"pseudo_cl_cov_{ver}.fits") - if os.path.exists(out_path): - self.print_done( - f"Skipping Pseudo-Cl covariance calculation, {out_path} exists" - ) - self._pseudo_cls[ver]["cov"] = fits.open(out_path) - else: - params = get_params_rho_tau(self.cc[ver], survey=ver) + out_path_merged = self._output_path_pseudo_cl_cov( + ver, "iNKA", tomography=compute_tomography + ) + tomo_str = "tomo" if compute_tomography else "non_tomo" - self.print_cyan(f"Extracting the fiducial power spectrum for {ver}") + if compute_tomography: + tomo_bin_ids, tomo_bin_pairs = self._get_tomo_bins(ver) - lmax = 2 * self.nside - ell = np.arange(1, lmax + 1) - pw = hp.pixwin(nside, lmax=lmax) - if pw.shape[0] != len(ell) + 1: + if tomo_bin_ids is None or tomo_bin_pairs is None: raise ValueError( - "Unexpected pixwin length for lmax=" - f"{lmax}: got {pw.shape[0]}, expected {len(ell) + 1}" - ) - pw = pw[1 : len(ell) + 1] - - # Load redshift distribution and calculate theory C_ell - path_redshift_distr = self.cc[ver]["shear"]["redshift_path"] - z, dndz = np.loadtxt(path_redshift_distr, unpack=True) - fiducial_cl = ( - get_theo_c_ell( - ell=ell, - z=z, - nz=dndz, - backend="ccl", - cosmo=self.cosmo, + f"Version {ver} does not have tomography information." ) - * pw**2 + + else: + tomo_bin_pairs = [("all", "all")] + + if os.path.exists(out_path_merged) and not self.force_run: + self.print_done( + f"Skipping Pseudo-Cl iNKA covariance calculation, {out_path_merged} exists" + ) + self._pseudo_cls[ver][f"cov_iNKA_{tomo_str}"] = fits.open( + out_path_merged ) - self.print_cyan("Getting a binning, n_gal_map, field and workspace.") + if load_all_block: + self.print_done("Loading all the iNKA covariance blocks") + n_spectra = len(tomo_bin_pairs) - lmin = 8 - lmax = 2 * self.nside - b_lmax = lmax - 1 + # indices into tomo_bin_pairs for all unique covariance blocks + block_indices = list( + itertools.combinations_with_replacement(range(n_spectra), 2) + ) - b = self.get_namaster_bin(lmin, lmax, b_lmax) + for index_a, index_b in block_indices: + bin_key_a1, bin_key_a2 = tomo_bin_pairs[index_a] + bin_key_b1, bin_key_b2 = tomo_bin_pairs[index_b] + + load_path = self._output_path_iNKA_block_cov( + ver, + tomo_bin_quad=( + bin_key_a1, + bin_key_a2, + bin_key_b1, + bin_key_b2, + ), + ) - # Load data and create shear and noise maps - cat_gal = open_entry(self.cc[ver]["shear"]) + if os.path.exists(load_path): + self._pseudo_cls[ver][ + f"spectra_{bin_key_a1}{bin_key_a2}_spectra_{bin_key_b1}{bin_key_b2}" + ] = fits.open(load_path) + + if bin_key_a1 == bin_key_b1 and bin_key_a2 == bin_key_b2: + self._pseudo_cls[ver][ + f"tomo_bin_{bin_key_a1}_tomo_bin_{bin_key_a2}" + ]["cov"] = fits.open(load_path) + else: + raise FileNotFoundError( + "The file does not exist, please run `cosmo_val` with `force_run` to True." + ) - n_gal, unique_pix, _idx, idx_rep = self.get_n_gal_map( - params, nside, cat_gal - ) + continue - f, wsp = self.get_field_and_workspace_from_map(n_gal, b_lmax, b) + # Initialise dictionnary to store field and workspace + n_gal_map_dict = {} + field_dict = {} + wsp_dict = {} - if self.noise_bias_method == "randoms": - self.print_cyan("Getting a sample of Cls with noise bias.") + self.print_cyan(f"Extracting the fiducial power spectrum for {ver}") - cl_noise, f, wsp = self.get_sample( - params, - self.nside, - b_lmax, - b, - cat_gal, - n_gal, - n_gal, - unique_pix, - idx_rep, - np.random.default_rng(self.cell_seed), - ) + fiducial_cl = self.get_fiducial_cl(ver, compute_tomography) - noise_bias_cl = np.mean(cl_noise, axis=0) + # Add a zero multipole and drop the last value to align + # Namaster and fiducial binning + fiducial_cl = { + key: np.concatenate(([0.0], np.asarray(value[:-1], dtype=float))) + for key, value in fiducial_cl.items() + } + fiducial_cl = _apply_pixel_window_to_fiducial_cl( + fiducial_cl, nside, self.cell_method + ) - elif self.noise_bias_method == "analytic": - self.print_cyan("Getting analytic noise bias.") + self.print_cyan( + "Estimating and adding the noise bias to the fiducial power spectra" + ) + self.print_cyan(f"Method used: {self.noise_bias_method}") - e1, e2, w = ( - cat_gal[self.cc[ver]["shear"]["e1_col"]], - cat_gal[self.cc[ver]["shear"]["e2_col"]], - cat_gal[self.cc[ver]["shear"]["w_col"]], - ) - variance_map = self.get_variance_map( - self.nside, e1, e2, w, unique_pix, idx_rep - ) + params = get_params_rho_tau(self.cc[ver]) + cat_gal = open_entry(self.cc[ver]["shear"]) - noise_bias = hp.nside2pixarea(self.nside) * np.mean(variance_map) + lmin, lmax, b_lmax = spv_pseudo_cl.pseudo_cl_geometry(self.nside) + b = self.get_namaster_bin(lmin, lmax, b_lmax) - noise_bias_cl = np.zeros((4, lmax)) - noise_bias_cl[0, :] = noise_bias - noise_bias_cl[3, :] = noise_bias + # Compute the fields and workspaces + for bin_key1, bin_key2 in tomo_bin_pairs: + self.print_cyan( + f"Computing fields and workspaces for {bin_key1}, {bin_key2}" + ) - noise_bias_cl = wsp.decouple_cell(noise_bias_cl) # Decouple + # Get the tomographic bins + cat_gal_a = self._get_tomographic_bin(params, cat_gal, bin_key1) + cat_gal_b = self._get_tomographic_bin(params, cat_gal, bin_key2) - else: - raise ValueError( - f"Noise bias method {self.noise_bias_method} not recognized. It should be 'randoms' or 'analytic'." + # Compute the n_gal_maps and the wsp object + unique_pix_a, idx_a, idx_rep_a = self.get_pixels( + params, nside, cat_gal_a + ) + unique_pix_b, idx_b, idx_rep_b = self.get_pixels( + params, nside, cat_gal_b + ) + + # Compute the number density maps + n_gal_map_a = self.get_n_gal_map(params, nside, cat_gal_a) + n_gal_map_b = self.get_n_gal_map(params, nside, cat_gal_b) + + # Get the shear maps + shear_map_a_e1, shear_map_a_e2 = self.get_shear_map( + params, + self.nside, + cat_gal_a, + unique_pix=unique_pix_a, + idx=idx_a, + idx_rep=idx_rep_a, + ) + shear_map_b_e1, shear_map_b_e2 = self.get_shear_map( + params, + self.nside, + cat_gal_b, + unique_pix=unique_pix_b, + idx=idx_b, + idx_rep=idx_rep_b, + ) + + # Get the fields and workspaces + field_a, field_b, wsp = spv_pseudo_cl.get_field_and_workspace_from_map( + b, + mask_a=n_gal_map_a, + e1_map_a=shear_map_a_e1, + e2_map_a=shear_map_a_e2, + mask_b=n_gal_map_b, + e1_map_b=shear_map_b_e1, + e2_map_b=shear_map_b_e2, + pol_factor=self.pol_factor, + return_wsp=True, + ) + + # Save in the dictionnaries + if f"W{bin_key1}" not in n_gal_map_dict: + n_gal_map_dict[f"W{bin_key1}"] = n_gal_map_a + if f"W{bin_key2}" not in n_gal_map_dict: + n_gal_map_dict[f"W{bin_key2}"] = n_gal_map_b + if f"W{bin_key1}" not in field_dict: + field_dict[f"W{bin_key1}"] = field_a + if f"W{bin_key2}" not in field_dict: + field_dict[f"W{bin_key2}"] = field_b + if bin_key1 <= bin_key2 and f"W{bin_key1}xW{bin_key2}" not in wsp_dict: + wsp_dict[f"W{bin_key1}xW{bin_key2}"] = wsp + + for bin_key1, bin_key2 in tomo_bin_pairs: + if bin_key1 == bin_key2: + self.print_cyan( + f"Adding the noise bias for tomographic bins {bin_key1}, {bin_key2}" ) - # Unbin, then fill the data vector below lmin with the lowest-ell value - noise_bias_cl = b.unbin_cell(noise_bias_cl) - lowest_ell = b.get_ell_list(0)[0] - noise_bias_cl[:, :lowest_ell] = noise_bias_cl[:, [lowest_ell]] + cat_gal_ = self._get_tomographic_bin(params, cat_gal, bin_key1) + + noise_bias_cl = self.get_noise_bias(params, nside, cat_gal_) + + # If using the analytical noise bias, we need to decouple it + if self.noise_bias_method == "analytic": + self.print_cyan( + "Decoupling the noise bias for the analytic method..." + ) + wsp = wsp_dict[f"W{bin_key1}xW{bin_key2}"] + noise_bias_cl = wsp.decouple_cell(noise_bias_cl) + + # Unbin, then continue the white noise below lmin at the + # lowest band's value + noise_bias_cl = b.unbin_cell(noise_bias_cl) + lowest_ell = b.get_ell_list(0)[0] + noise_bias_cl[:, :lowest_ell] = noise_bias_cl[:, [lowest_ell]] - self.print_cyan("Adding noise bias to the fiducial Cls.") + else: + noise_bias_cl = np.zeros((4, 2 * nside)) - fiducial_cl = ( + # Update the fiducial_cl dictionnary + fiducial_cl[f"W{bin_key1}xW{bin_key2}"] = ( np.array( [ - fiducial_cl, - 0.0 * fiducial_cl, - 0.0 * fiducial_cl, - 0.0 * fiducial_cl, + fiducial_cl[f"W{bin_key1}xW{bin_key2}"], + 0.0 * fiducial_cl[f"W{bin_key1}xW{bin_key2}"], + 0.0 * fiducial_cl[f"W{bin_key1}xW{bin_key2}"], + 0.0 * fiducial_cl[f"W{bin_key1}xW{bin_key2}"], ] ) + noise_bias_cl ) - if self.fiducial_input_inka == "coupled": - self.print_cyan("Coupling the fiducial Cls.") + if self.fiducial_input_inka == "coupled": + # Couple the cell if required + self.print_cyan("Coupling the fiducial Cls.") + for bin_key1, bin_key2 in tomo_bin_pairs: + # Get the wsp object + n_gal_map_a = n_gal_map_dict[f"W{bin_key1}"] + n_gal_map_b = n_gal_map_dict[f"W{bin_key2}"] + wsp = wsp_dict[f"W{bin_key1}xW{bin_key2}"] coupling_mat = wsp.get_coupling_matrix() coupling_mat_re = np.reshape( coupling_mat, (4, lmax, 4, lmax), order="F" ) - fiducial_cl = np.tensordot(coupling_mat_re, fiducial_cl) / np.mean( - n_gal**2 - ) # couple and divide by the mean of the mask squared + fiducial_cl[f"W{bin_key1}xW{bin_key2}"] = np.tensordot( + coupling_mat_re, fiducial_cl[f"W{bin_key1}xW{bin_key2}"] + ) / np.mean( + n_gal_map_a * n_gal_map_b + ) # couple and divide by the product of the mask + + # To compute all the blocks in the covariance, + # we must compute all the (i, j, k, l) bin combinations + # Or equivalently, compute the covariance for all spectra pairs + n_spectra = len(tomo_bin_pairs) + + # indices into tomo_bin_pairs for all unique covariance blocks + block_indices = list( + itertools.combinations_with_replacement(range(n_spectra), 2) + ) + # Loop on the different tomographic bin pairs to compute the covariance + for index_a, index_b in block_indices: + bin_key_a1, bin_key_a2 = tomo_bin_pairs[index_a] + bin_key_b1, bin_key_b2 = tomo_bin_pairs[index_b] + self.print_cyan( + f"Tomo Bin Quad: ({bin_key_a1}, {bin_key_a2}, {bin_key_b1}, {bin_key_b2})" + ) + + if ( + bin_key_a1 == bin_key_b1 + and bin_key_a2 == bin_key_b2 + and ( + f"tomo_bin_{bin_key_a1}_tomo_bin_{bin_key_a2}" + not in self._pseudo_cls[ver].keys() + ) + ): + self._pseudo_cls[ver][ + f"tomo_bin_{bin_key_a1}_tomo_bin_{bin_key_a2}" + ] = {} + + out_path = self._output_path_iNKA_block_cov( + ver, tomo_bin_quad=(bin_key_a1, bin_key_a2, bin_key_b1, bin_key_b2) + ) + + if os.path.exists(out_path) and not self.force_run: + self.print_done( + f"Skipping Pseudo-Cl covariance block Cl ({bin_key_a1, bin_key_a2}), Cl ({bin_key_b1, bin_key_b2}) calculation, {out_path} exists" + ) + + self._pseudo_cls[ver][ + f"spectra_{bin_key_a1}{bin_key_a2}_spectra_{bin_key_b1}{bin_key_b2}" + ] = fits.open(out_path) + + if bin_key_a1 == bin_key_b1 and bin_key_a2 == bin_key_b2: + self._pseudo_cls[ver][ + f"tomo_bin_{bin_key_a1}_tomo_bin_{bin_key_a2}" + ]["cov"] = fits.open(out_path) + + continue self.print_cyan("Computing the Pseudo-Cl covariance") - cw = nmt.NmtCovarianceWorkspace.from_fields(f, f, f, f) - - # Get actual number of ell bins from binning scheme - n_ell_actual = b.get_n_bands() - - covar_22_22 = nmt.gaussian_covariance( - cw, - 2, - 2, - 2, - 2, - fiducial_cl, - fiducial_cl, - fiducial_cl, - fiducial_cl, - wsp, - wb=wsp, - ).reshape([n_ell_actual, 4, n_ell_actual, 4]) + input_cl_a1_b1 = ( + fiducial_cl[f"W{bin_key_a1}xW{bin_key_b1}"] + if bin_key_a1 <= bin_key_b1 + else fiducial_cl[f"W{bin_key_b1}xW{bin_key_a1}"] + ) + input_cl_a1_b2 = ( + fiducial_cl[f"W{bin_key_a1}xW{bin_key_b2}"] + if bin_key_a1 <= bin_key_b2 + else fiducial_cl[f"W{bin_key_b2}xW{bin_key_a1}"] + ) + input_cl_a2_b1 = ( + fiducial_cl[f"W{bin_key_a2}xW{bin_key_b1}"] + if bin_key_a2 <= bin_key_b1 + else fiducial_cl[f"W{bin_key_b1}xW{bin_key_a2}"] + ) + input_cl_a2_b2 = ( + fiducial_cl[f"W{bin_key_a2}xW{bin_key_b2}"] + if bin_key_a2 <= bin_key_b2 + else fiducial_cl[f"W{bin_key_b2}xW{bin_key_a2}"] + ) - self.print_cyan("Saving Pseudo-Cl covariance") + covar_22_22 = spv_pseudo_cl.get_pseudo_cl_iNKA_covariance( + input_cl_a1_b1, + input_cl_a1_b2, + input_cl_a2_b1, + input_cl_a2_b2, + field_dict[f"W{bin_key_a1}"], + field_dict[f"W{bin_key_a2}"], + field_dict[f"W{bin_key_b1}"], + field_dict[f"W{bin_key_b2}"], + wsp_a=wsp_dict[f"W{bin_key_a1}xW{bin_key_a2}"], + wsp_b=wsp_dict[f"W{bin_key_b1}xW{bin_key_b2}"], + b=b, + ) - # covar_22_22 is indexed [ell, pol_a, ell, pol_b]; store each of the - # 16 EE/EB/BE/BB cross-blocks as a named HDU (row-major pol order). - # Append rather than construct from a list so astropy promotes the - # first HDU to a PrimaryHDU on write. - pols = ["EE", "EB", "BE", "BB"] - hdu = fits.HDUList() - for i, pa in enumerate(pols): - for j, pb in enumerate(pols): - hdu.append( - fits.ImageHDU( - covar_22_22[:, i, :, j], name=f"COVAR_{pa}_{pb}" - ) - ) + self.print_cyan("Saving Pseudo-Cl covariance") - hdu.writeto(out_path, overwrite=True) + self._pseudo_cls[ver][ + f"spectra_{bin_key_a1}{bin_key_a2}_spectra_{bin_key_b1}{bin_key_b2}" + ] = self._save_iNKA_covariance(covar_22_22, out_path) - self._pseudo_cls[ver]["cov"] = hdu + if bin_key_a1 == bin_key_b1 and bin_key_a2 == bin_key_b2: + self._pseudo_cls[ver][ + f"tomo_bin_{bin_key_a1}_tomo_bin_{bin_key_a2}" + ]["cov"] = self._pseudo_cls[ver][ + f"spectra_{bin_key_a1}{bin_key_a2}_spectra_{bin_key_b1}{bin_key_b2}" + ] + # Merge the covariance blocks + self._pseudo_cls[ver][f"cov_iNKA_{tomo_str}"] = self._merge_iNKA_covariance( + ver, tomography=compute_tomography + ) + self.print_done(f"Done Pseudo-Cl covariance calculation for {ver}") self.print_done("Done Pseudo-Cl covariance") - def calculate_pseudo_cl_onecovariance(self): + def calculate_pseudo_cl_onecovariance(self, tomography=False): """ Compute the pseudo-Cl covariance using OneCovariance. """ @@ -335,18 +686,24 @@ def calculate_pseudo_cl_onecovariance(self): if not os.path.exists(template_config): raise ValueError(f"Template config file {template_config} does not exist") - self._pseudo_cls_onecov = {} + if not hasattr(self, "_pseudo_cls_onecov"): + self._pseudo_cls_onecov = {} for ver in self.versions: self.print_magenta(ver) - - out_dir = self._output_path(f"pseudo_cl_cov_onecov_{ver}/") + self._pseudo_cls_onecov.setdefault(ver, {}) + out_dir = self._output_path( + "pseudo_cl/", f"pseudo_cl_cov_onecov_{ver}_tomography_{tomography}" + ) os.makedirs(out_dir, exist_ok=True) - if os.path.exists( - os.path.join(out_dir, "covariance_list_3x2pt_pure_Cell.dat") + if ( + os.path.exists( + os.path.join(out_dir, "covariance_list_3x2pt_pure_Cell.dat") + ) + and not self.force_run ): self.print_done(f"Skipping OneCovariance calculation, {out_dir} exists") - self._load_onecovariance_cov(out_dir, ver) + self._load_onecovariance_cov(out_dir, ver, tomography) else: mask_path = self.cc[ver]["mask"] if not os.path.exists(mask_path): @@ -359,7 +716,9 @@ def calculate_pseudo_cl_onecovariance(self): self.cc[ver]["shear"]["redshift_path"] ) - config_path = os.path.join(out_dir, f"config_onecov_{ver}.ini") + config_path = os.path.join( + out_dir, f"config_onecov_{ver}_tomography_{tomography}.ini" + ) self.print_cyan( f"Modifying OneCovariance config file and saving it to {config_path}" @@ -371,6 +730,7 @@ def calculate_pseudo_cl_onecovariance(self): mask_path, redshift_distr_path, ver, + tomography, ) self.print_cyan("Running OneCovariance...") @@ -382,12 +742,288 @@ def calculate_pseudo_cl_onecovariance(self): f"OneCovariance command failed with return code {ret}" ) self.print_cyan("OneCovariance completed successfully.") - self._load_onecovariance_cov(out_dir, ver) + self._load_onecovariance_cov(out_dir, ver, tomography) self.print_done("Done Pseudo-Cl covariance with OneCovariance") + def calculate_pseudo_cl_g_ng_cov(self, tomography=False, gaussian_part="iNKA"): + assert gaussian_part in ["iNKA", "OneCovariance"], ( + "gaussian_part must be 'iNKA' or 'OneCovariance'" + ) + self.print_start( + f"Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part" + ) + + if not hasattr(self, "_pseudo_cls_cov_g_ng"): + self._pseudo_cls_cov_g_ng = {} + + for ver in self.versions: + self.print_magenta(ver) + self._pseudo_cls_cov_g_ng.setdefault(ver, {}) + key_to_use = "tomo" if tomography else "non_tomo" + out_file = self._output_path( + "pseudo_cl", + f"pseudo_cl_cov_g_ng_{gaussian_part}_{ver}_tomography_{tomography}.fits", + ) + if os.path.exists(out_file) and not self.force_run: + self.print_done( + f"Skipping Gaussian and Non-Gaussian covariance calculation, {out_file} exists" + ) + cov_hdu = fits.open(out_file) + self._pseudo_cls_cov_g_ng[ver][key_to_use] = cov_hdu + continue + if gaussian_part == "iNKA": + gaussian_cov = self.pseudo_cls[ver][f"cov_iNKA_{key_to_use}"][ + "COVAR_EE_EE" + ].data + non_gaussian_cov = ( + self.pseudo_cls_onecov[ver][key_to_use]["all_cov"] + - self.pseudo_cls_onecov[ver][key_to_use]["gaussian_cov"] + ) + full_cov = gaussian_cov + non_gaussian_cov + elif gaussian_part == "OneCovariance": + gaussian_cov = self.pseudo_cls_onecov[ver][key_to_use]["gaussian_cov"] + non_gaussian_cov = ( + self.pseudo_cls_onecov[ver][key_to_use]["all_cov"] + - self.pseudo_cls_onecov[ver][key_to_use]["gaussian_cov"] + ) + full_cov = self.pseudo_cls_onecov[ver][key_to_use]["all_cov"] + else: + raise ValueError(f"Unknown gaussian_part: {gaussian_part}") + self.print_cyan("Saving Gaussian and Non-Gaussian covariance...") + hdu = fits.HDUList() + hdu.append(fits.ImageHDU(gaussian_cov, name="COVAR_GAUSSIAN")) + hdu.append(fits.ImageHDU(non_gaussian_cov, name="COVAR_NON_GAUSSIAN")) + hdu.append(fits.ImageHDU(full_cov, name="COVAR_FULL")) + hdu.writeto(out_file, overwrite=True) + self._pseudo_cls_cov_g_ng[ver][key_to_use] = hdu + self.print_done( + f"Done Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part" + ) + + # ---------------- Utility functions for pseudo-Cl calculations ---------------- # + def get_namaster_bin(self, lmin, lmax, b_lmax): + """Build NaMaster binning object (thin wrapper, state -> primitive).""" + return spv_pseudo_cl.make_namaster_bin( + lmin, + lmax, + b_lmax, + self.binning, + ell_step=self.ell_step, + n_ell_bins=self.n_ell_bins, + power=self.power, + ) + + def get_pixels(self, params, nside, cat_gal): + """Get unique pixels and indices for a catalog (thin wrapper -> primitive).""" + return spv_pseudo_cl.get_pixels( + cat_gal[params["ra_col"]], cat_gal[params["dec_col"]], nside + ) + + def get_n_gal_map( + self, params, nside, cat_gal, unique_pix=None, idx=None, idx_rep=None + ): + """Weighted galaxy number-density map (thin wrapper -> primitive).""" + return spv_pseudo_cl.get_n_gal_map( + nside, + cat_gal[params["ra_col"]], + cat_gal[params["dec_col"]], + weights=cat_gal[params["w_col"]], + unique_pix=unique_pix, + idx=idx, + idx_rep=idx_rep, + ) + + def get_shear_map( + self, + params, + nside, + cat_gal, + unique_pix=None, + idx=None, + idx_rep=None, + n_gal_map=None, + ): + """Weighted shear map (thin wrapper -> primitive).""" + return spv_pseudo_cl.get_shear_map( + cat_gal[params["ra_col"]], + cat_gal[params["dec_col"]], + cat_gal[params["e1_col"]], + cat_gal[params["e2_col"]], + cat_gal[params["w_col"]], + nside, + unique_pix=unique_pix, + idx=idx, + idx_rep=idx_rep, + n_gal_map=n_gal_map, + ) + + def get_noise_realisation( + self, + params, + nside, + cat_gal, + n_gal=None, + unique_pix=None, + idx=None, + idx_rep=None, + rng=None, + ): + """ + Get a single Gaussian noise realization (thin wrapper -> primitive). + """ + return spv_pseudo_cl.get_noise_realisation( + cat_gal[params["ra_col"]], + cat_gal[params["dec_col"]], + cat_gal[params["e1_col"]], + cat_gal[params["e2_col"]], + cat_gal[params["w_col"]], + nside, + n_gal_map=n_gal, + unique_pix=unique_pix, + idx=idx, + idx_rep=idx_rep, + rng=rng, + ) + + def get_noise_bias_from_gaussian_real( + self, + params, + nside, + cat_gal, + unique_pix=None, + idx=None, + idx_rep=None, + n_gal_map=None, + wsp=None, + ): + """Noise-bias from Gaussian realisations (thin wrapper, state -> primitive)""" + return spv_pseudo_cl.get_noise_bias_from_gaussian_real( + cat_gal[params["ra_col"]], + cat_gal[params["dec_col"]], + cat_gal[params["e1_col"]], + cat_gal[params["e2_col"]], + cat_gal[params["w_col"]], + nside, + nrandom_cell=self.nrandom_cell, + binning=self.binning, + ell_step=self.ell_step, + n_ell_bins=self.n_ell_bins, + power=self.power, + unique_pix=unique_pix, + idx=idx, + idx_rep=idx_rep, + n_gal_map=n_gal_map, + wsp=wsp, + seed=self.cell_seed, + ) + + def get_noise_bias_analytical( + self, params, nside, cat_gal, unique_pix=None, idx=None, idx_rep=None + ): + """Noise-bias from analytical prescription (thin wrapper, state -> primitive)""" + return spv_pseudo_cl.get_noise_bias_analytical( + cat_gal[params["ra_col"]], + cat_gal[params["dec_col"]], + cat_gal[params["e1_col"]], + cat_gal[params["e2_col"]], + cat_gal[params["w_col"]], + lmax=2 * nside, + nside=nside, + unique_pix=unique_pix, + idx=idx, + idx_rep=idx_rep, + ) + + def get_noise_bias(self, params, nside, cat_gal): + """Noise-bias estimation (thin wrapper, state -> primitive)""" + return spv_pseudo_cl.get_noise_bias( + cat_gal[params["ra_col"]], + cat_gal[params["dec_col"]], + cat_gal[params["e1_col"]], + cat_gal[params["e2_col"]], + cat_gal[params["w_col"]], + nside, + noise_bias_method=self.noise_bias_method, + binning=self.binning, + ell_step=self.ell_step, + n_ell_bins=self.n_ell_bins, + power=self.power, + nrandom_cell=self.nrandom_cell, + seed=self.cell_seed, + ) + + def get_pseudo_cls_map( + self, map_a, mask_a, wsp=None, shear_map_b=None, mask_b=None + ): + """Map-based pseudo-cl (thin wrapper, state -> primitive).""" + return spv_pseudo_cl.get_pseudo_cls_map( + map_a, + mask_a, + self.nside, + self.binning, + shear_map_b=shear_map_b, + mask_b=mask_b, + pol_factor=self.pol_factor, + wsp=wsp, + ell_step=self.ell_step, + n_ell_bins=self.n_ell_bins, + power=self.power, + ) + + def get_pseudo_cls_catalog( + self, catalog, params, wsp=None, tomo_bin_a="all", tomo_bin_b="all" + ): + """Catalog-based pseudo-cl (thin wrapper, state -> primitive).""" + return spv_pseudo_cl.get_pseudo_cls_catalog( + catalog, + params, + self.nside, + self.binning, + tomo_bin_a=tomo_bin_a, + tomo_bin_b=tomo_bin_b, + pol_factor=self.pol_factor, + wsp=wsp, + ell_step=self.ell_step, + n_ell_bins=self.n_ell_bins, + power=self.power, + ) + + def read_redshift_distribution(self, ver, is_tomography): + path_redshift_distr = self.cc[ver]["shear"]["redshift_path"] + redshift_distribution = np.loadtxt(path_redshift_distr) + z = redshift_distribution[:, 0] + dndz = redshift_distribution[:, 1:] + + # Here it is assumed that the tomographic redshift distribution sum to the non-tomographic one and that the latter is normalised + if not is_tomography: + dndz = np.sum(dndz, axis=1) + + return z, dndz + + def get_fiducial_cl(self, ver, is_tomography): + """Get a theory prediction for the angular power spectra (thin wrapper, state -> primitive).""" + lmax = 2 * self.nside + + z, dndz = self.read_redshift_distribution(ver, is_tomography) + + fiducial_cl = spv_pseudo_cl.get_fiducial_cl(z, dndz, lmax, self.cosmo) + + # If non-tomographic, change the key to 'WallxWall' + if not is_tomography: + fiducial_cl = {"WallxWall": fiducial_cl["W1xW1"]} + + return fiducial_cl + def _modify_onecov_config( - self, template_config, config_path, out_dir, mask_path, redshift_distr_path, ver + self, + template_config, + config_path, + out_dir, + mask_path, + redshift_distr_path, + ver, + tomography, ): """ Modify OneCovariance configuration file with correct mask, redshift distribution, @@ -403,6 +1039,10 @@ def _modify_onecov_config( Path to the mask file redshift_distr_path : str Path to the redshift distribution file + ver : str + Version identifier for the current analysis + tomography : bool + Whether to compute tomography or not """ config = configparser.ConfigParser() # Load the template configuration @@ -414,10 +1054,42 @@ def _modify_onecov_config( config["survey specs"]["mask_directory"] = mask_folder config["survey specs"]["mask_file_lensing"] = mask_base config["survey specs"]["survey_area_lensing_in_deg2"] = str(self.area[ver]) - config["survey specs"]["ellipticity_dispersion"] = str( - self.ellipticity_dispersion[ver] + + # Update ellipticity dispersion and effective number density + # Account for tomography if needed + if tomography: + tomo_bin_ids, _ = self._get_tomo_bins(ver) + else: + tomo_bin_ids = ["all"] + input_ellipticity_distribution = ", ".join( + str(self.ellipticity_dispersion[ver][f"tomo_bin_{bin_id}"]) + for bin_id in tomo_bin_ids ) - config["survey specs"]["n_eff_lensing"] = str(self.n_eff_gal[ver]) + input_n_eff_gal = ", ".join( + str(self.n_eff_gal[ver][f"tomo_bin_{bin_id}"]) for bin_id in tomo_bin_ids + ) + config["survey specs"]["ellipticity_dispersion"] = ( + input_ellipticity_distribution + ) + config["survey specs"]["n_eff_lensing"] = input_n_eff_gal + + # Handle the case where the redshift distribution file is tomographic + # Converts it to a non-tomographic distribution if needed + if not tomography: + # Load the redshift distribution + z, dndz = self.read_redshift_distribution(ver, is_tomography=True) + # Sum over tomographic bins to get the non-tomographic distribution + dndz_non_tomo = np.sum(dndz, axis=1) + # Save the non-tomographic distribution to a new file + non_tomo_redshift_distr_path = os.path.join( + out_dir, f"redshift_distribution_{ver}_non_tomo.txt" + ) + np.savetxt( + non_tomo_redshift_distr_path, + np.column_stack((z, dndz_non_tomo)), + header="z dN/dz", + ) + redshift_distr_path = non_tomo_redshift_distr_path # Update redshift distribution path redshift_distr_base = os.path.basename(os.path.abspath(redshift_distr_path)) @@ -425,6 +1097,8 @@ def _modify_onecov_config( config["redshift"]["z_directory"] = redshift_distr_folder config["redshift"]["zlens_file"] = redshift_distr_base + self._update_onecov_cosmo_params(config, self.cosmo) + # Update output directory config["output settings"]["directory"] = out_dir @@ -432,7 +1106,48 @@ def _modify_onecov_config( with open(config_path, "w") as f: config.write(f) - def _load_onecovariance_cov(self, out_dir, ver): + def _update_onecov_cosmo_params(self, config, cosmo): + """ + Update the cosmological parameters in the OneCovariance configuration. + + Parameters + ---------- + config : configparser.ConfigParser + The configuration object to update. + cosmo + The cosmology object containing the parameters to set. + """ + # Update cosmo params section + config["cosmo"]["h"] = str(cosmo["H0"] / 100.0) + config["cosmo"]["omega_m"] = str(cosmo["Omega_m"]) + config["cosmo"]["omega_b"] = str(cosmo["Omega_b"]) + config["cosmo"]["omega_de"] = str(cosmo["Omega_l"]) + config["cosmo"]["sigma8"] = str(cosmo.sigma8()) + config["cosmo"]["ns"] = str(cosmo["n_s"]) + config["cosmo"]["w0"] = str(cosmo["w0"]) + config["cosmo"]["wa"] = str(cosmo["wa"]) + config["cosmo"]["neff"] = str(cosmo["Neff"]) + config["cosmo"]["m_nu"] = str( + np.sum( + cosmo["m_nu"] + ) # cosmo["m_nu"] is an array of neutrino masses in eV. OneCovariance expects the sum of neutrino masses in eV. + ) + + # Update powspec evaluation section + cosmo_dict = cosmo.to_dict() + + config["powspec evaluation"]["non_linear_model"] = str( + cosmo_dict.get("extra_parameters", {}) + .get("camb", {}) + .get("halofit_version", "mead2020_feedback") # Default to mead2020 feedback + ) # TODO: check what would be the default if not provided as input to align with this + config["powspec evaluation"]["HMCode_logT_AGN"] = str( + cosmo_dict.get("extra_parameters", {}) + .get("camb", {}) + .get("HMCode_logT_AGN", 7.8) # OneCovariance default + ) + + def _load_onecovariance_cov(self, out_dir, ver, tomography): self.print_cyan(f"Loading OneCovariance results from {out_dir}") cov_one_cov = np.genfromtxt( os.path.join(out_dir, "covariance_list_3x2pt_pure_Cell.dat") @@ -440,327 +1155,543 @@ def _load_onecovariance_cov(self, out_dir, ver): gaussian_one_cov = cov_from_one_covariance(cov_one_cov, gaussian=True) all_one_cov = cov_from_one_covariance(cov_one_cov, gaussian=False) - self._pseudo_cls_onecov[ver] = { - "gaussian_cov": gaussian_one_cov, - "all_cov": all_one_cov, - } - - def calculate_pseudo_cl_g_ng_cov(self, gaussian_part="iNKA"): - assert gaussian_part in ["iNKA", "OneCovariance"], ( - "gaussian_part must be 'iNKA' or 'OneCovariance'" - ) - self.print_start( - f"Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part" + key_to_update = "tomo" if tomography else "non_tomo" + self._pseudo_cls_onecov.setdefault(ver, {}).update( + { + key_to_update: { + "gaussian_cov": gaussian_one_cov, + "all_cov": all_one_cov, + } + } ) - self._pseudo_cls_cov_g_ng = {} + def _get_tomographic_bin(self, params, cat_gal, tomo_bin): + """Extract tomographic bin from a given catalogue""" + if tomo_bin == "all": + return cat_gal + else: + tomo_bin_id = cat_gal[params["tomo_bin_col"]] + mask = tomo_bin_id == tomo_bin + return cat_gal[mask] - for ver in self.versions: - self.print_magenta(ver) - out_file = self._output_path( - f"pseudo_cl_cov_g_ng_{gaussian_part}_{ver}.fits" - ) - if os.path.exists(out_file): - self.print_done( - f"Skipping Gaussian and Non-Gaussian covariance calculation, {out_file} exists" - ) - cov_hdu = fits.open(out_file) - self._pseudo_cls_cov_g_ng[ver] = cov_hdu - continue - if gaussian_part == "iNKA": - gaussian_cov = self.pseudo_cls[ver]["cov"]["COVAR_EE_EE"].data - non_gaussian_cov = ( - self.pseudo_cls_onecov[ver]["all_cov"] - - self.pseudo_cls_onecov[ver]["gaussian_cov"] - ) - full_cov = gaussian_cov + non_gaussian_cov - elif gaussian_part == "OneCovariance": - gaussian_cov = self.pseudo_cls_onecov[ver]["gaussian_cov"] - non_gaussian_cov = ( - self.pseudo_cls_onecov[ver]["all_cov"] - - self.pseudo_cls_onecov[ver]["gaussian_cov"] - ) - full_cov = self.pseudo_cls_onecov[ver]["all_cov"] - else: - raise ValueError(f"Unknown gaussian_part: {gaussian_part}") - self.print_cyan("Saving Gaussian and Non-Gaussian covariance...") - hdu = fits.HDUList() - hdu.append(fits.ImageHDU(gaussian_cov, name="COVAR_GAUSSIAN")) - hdu.append(fits.ImageHDU(non_gaussian_cov, name="COVAR_NON_GAUSSIAN")) - hdu.append(fits.ImageHDU(full_cov, name="COVAR_FULL")) - hdu.writeto(out_file, overwrite=True) - self._pseudo_cls_cov_g_ng[ver] = hdu - self.print_done( - f"Done Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part" - ) + def _output_path_pseudo_cl(self, ver, tomo_bin_pair): + """Default pseudo-Cl product path of one bin pair. - def calculate_pseudo_cl(self, out_path=None): + The ``("all", "all")`` pair is the SACC part ``pseudo_cl_{ver}.sacc``; + a tomographic pair is a FITS table under ``pseudo_cl/``. """ - Compute the pseudo-Cl of given catalogs. + if tuple(tomo_bin_pair) == ("all", "all"): + return self._output_path(f"pseudo_cl_{ver}.sacc") + bin_key1, bin_key2 = tomo_bin_pair + return self._output_path( + "pseudo_cl", + f"pseudo_cl_from_{self.cell_method}_tomo_bin_{bin_key1}_tomo_bin_{bin_key2}_{ver}_binning_{self.binning}_nbins_{self.n_ell_bins}.fits", + ) - ``out_path`` is the exact destination the part is born at (one version - only); ``None`` defaults each part to ``pseudo_cl_{ver}.sacc``. - """ - self.print_start("Computing pseudo-Cl's") + def _output_path_pseudo_cl_cov(self, ver, method, tomography): + is_tomo = "tomo" if tomography else "non_tomo" + return self._output_path( + "pseudo_cl", + f"pseudo_cl_cov_{is_tomo}_{ver}_from_{method}_binning_{self.binning}_nbins_{self.n_ell_bins}.fits", + ) - nside = self.nside + def _output_path_iNKA_block_cov(self, ver, tomo_bin_quad): + bin_key_a1, bin_key_a2, bin_key_b1, bin_key_b2 = tomo_bin_quad + return self._output_path( + "pseudo_cl", + f"iNKA_block_{ver}", + f"pseudo_cl_cov_from_iNKA_tomo_bin_{bin_key_a1}_tomo_bin_{bin_key_a2}_tomo_bin_{bin_key_b1}_tomo_bin_{bin_key_b2}_{ver}_binning_{self.binning}_nbins_{self.n_ell_bins}.fits", + ) - if out_path is not None and len(self.versions) != 1: - raise ValueError( - "calculate_pseudo_cl(out_path=...) writes one part to one path, " - f"but {len(self.versions)} versions are configured; call per version" + def _save_pseudo_cl( + self, ver, out_path, tomo_bin_pair, ell_eff, cl_all, wsp, nside=None + ): + """Write one pair as SACC or FITS, forwarding ``nside`` for map spectra.""" + if tuple(tomo_bin_pair) == ("all", "all"): + self.pseudo_cl_to_sacc_part( + ver, out_path, ell_eff, cl_all, wsp, nside=nside ) + else: + self.save_pseudo_cl(ell_eff, cl_all, out_path) - try: - self._pseudo_cls - except AttributeError: - self._pseudo_cls = {} - for ver in self.versions: - self.print_magenta(ver) - - self._pseudo_cls[ver] = {} - - ver_out_path = out_path or self._output_path(f"pseudo_cl_{ver}.sacc") - if os.path.exists(ver_out_path): - self.print_done( - f"Skipping Pseudo-Cl's calculation, {ver_out_path} exists" - ) - self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc( - ver_out_path - ) - elif self.cell_method == "map": - self.calculate_pseudo_cl_map(ver, nside, ver_out_path) - elif self.cell_method == "catalog": - self.calculate_pseudo_cl_catalog(ver, ver_out_path) - else: - raise ValueError(f"Unknown cell method: {self.cell_method}") - - self.print_done("Done pseudo-Cl's") + def _load_pseudo_cl(self, out_path, tomo_bin_pair): + """Read one bin pair's pseudo-Cl product written by ``_save_pseudo_cl``.""" + if tuple(tomo_bin_pair) == ("all", "all"): + return self._load_pseudo_cl_sacc(out_path) + return fits.getdata(out_path) @staticmethod def _load_pseudo_cl_sacc(out_path): - """Read a pseudo-Cl SACC part into the ELL/EE/EB/BB dict.""" + """Read the single-field auto-spectrum, whose BE equals EB.""" # Readback of a part this producer just wrote — a legitimate pre-blind # consumer, so the fail-closed load is opted out of. s = sacc_io.load(out_path, allow_unblinded=True) ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(s, SACC_BIN) - return {"ELL": ell, "EE": ee, "EB": eb, "BB": bb} + return {"ELL": ell, "EE": ee, "EB": eb, "BE": eb.copy(), "BB": bb} - def calculate_pseudo_cl_map(self, ver, nside, out_path): - params = get_params_rho_tau(self.cc[ver], survey=ver) - - # Load data and create shear and noise maps - cat_gal = open_entry(self.cc[ver]["shear"]) + def pseudo_cl_to_sacc_part( + self, version, out_path, ell_eff, cl_all, wsp, nside=None + ): + """Write the pseudo-Cl SACC part with its shared bandpower window. - w = cat_gal[params["w_col"]] - self.print_cyan("Creating maps and computing Cl's...") - n_gal_map, unique_pix, _idx, idx_rep = self.get_n_gal_map( - params, nside, cat_gal + ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB). + Map-based callers pass ``nside`` so the window includes ``pw²(ℓ)``; + catalogue-based callers leave it unset because their spectra have no + HEALPix pixel window. No covariance is attached here. + """ + s = pseudo_cl_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + ell_eff, + cl_all, + wsp, + nside=nside, ) - mask = n_gal_map != 0 + sacc_io.save(s, out_path, type="data") - shear_map_e1 = np.zeros(hp.nside2npix(nside)) - shear_map_e2 = np.zeros(hp.nside2npix(nside)) + def save_pseudo_cl(self, ell_eff, pseudo_cl, out_path): + """ + Save a tomographic bin pair's pseudo-Cl's to a FITS file. - e1 = cat_gal[params["e1_col"]] - e2 = cat_gal[params["e2_col"]] + Parameters + ---------- + pseudo_cl : np.array + Pseudo-Cl's to save. + out_path : str + Path to save the pseudo-Cl's to. + """ + # Create columns of the fits file + col1 = fits.Column(name="ELL", format="D", array=ell_eff) + col2 = fits.Column(name="EE", format="D", array=pseudo_cl[0]) + col3 = fits.Column(name="EB", format="D", array=pseudo_cl[1]) + col4 = fits.Column(name="BE", format="D", array=pseudo_cl[2]) + col5 = fits.Column(name="BB", format="D", array=pseudo_cl[3]) + coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) + cell_hdu = fits.BinTableHDU.from_columns(coldefs, name="PSEUDO_CELL") + + cell_hdu.writeto(out_path, overwrite=True) + + def _save_iNKA_covariance(self, covar, out_path): + # covar_22_22 is indexed [ell, pol_a, ell, pol_b]; store each of the + # 16 EE/EB/BE/BB cross-blocks as a named HDU (row-major pol order). + # Append rather than construct from a list so astropy promotes the + # first HDU to a PrimaryHDU on write. + pols = ["EE", "EB", "BE", "BB"] + hdu = fits.HDUList() + for i, pa in enumerate(pols): + for j, pb in enumerate(pols): + hdu.append(fits.ImageHDU(covar[:, i, :, j], name=f"COVAR_{pa}_{pb}")) + + hdu.writeto(out_path, overwrite=True) + + return hdu + + def _merge_iNKA_covariance(self, ver, tomography): + """ + Merge the iNKA covariance matrices for a given version to get the data vector covariance. + """ + out_path = self._output_path_pseudo_cl_cov( + ver, method="iNKA", tomography=tomography + ) - del cat_gal + if tomography: + # Merge the tomographic covariance matrices + tomo_bin_ids, tomo_bin_pairs = self._get_tomo_bins(ver) - shear_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1 * w) - shear_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2 * w) - shear_map_e1[mask] /= n_gal_map[mask] - shear_map_e2[mask] /= n_gal_map[mask] + # Get the number of bins from the pseudo_cls attribute + # The non-tomographic pseudo-cl are computed from the call + # to this attribute if not already computed. + n_ell = self._pseudo_cls[ver]["tomo_bin_all_tomo_bin_all"]["pseudo_cl"][ + "ELL" + ].shape[0] - shear_map = shear_map_e1 + 1j * shear_map_e2 + if tomo_bin_ids is None or tomo_bin_pairs is None: + raise AssertionError( + f"Tomographic bin IDs of version {ver} is not available." + ) - del shear_map_e1, shear_map_e2 + n_spectra = len(tomo_bin_pairs) + block_indices = list( + itertools.combinations_with_replacement(range(n_spectra), 2) + ) - ell_eff, cl_shear, wsp = self.get_pseudo_cls_map(shear_map, n_gal_map) + pols = ["EE", "EB", "BE", "BB"] + covar = fits.HDUList() + for pa in pols: + for pb in pols: + full_cov = np.zeros((n_spectra * n_ell, n_spectra * n_ell)) + for index_a, index_b in block_indices: + bin_key_a1, bin_key_a2 = tomo_bin_pairs[index_a] + bin_key_b1, bin_key_b2 = tomo_bin_pairs[index_b] + + block_path = self._output_path_iNKA_block_cov( + ver, + tomo_bin_quad=( + bin_key_a1, + bin_key_a2, + bin_key_b1, + bin_key_b2, + ), + ) + sl_a = slice(index_a * n_ell, (index_a + 1) * n_ell) + sl_b = slice(index_b * n_ell, (index_b + 1) * n_ell) + + with fits.open(block_path) as block_cov: + full_cov[sl_a, sl_b] = block_cov[f"COVAR_{pa}_{pb}"].data + if index_a != index_b: + # Cov(P_b, Q_a) = Cov(Q_a, P_b).T. + full_cov[sl_b, sl_a] = block_cov[ + f"COVAR_{pb}_{pa}" + ].data.T + + covar.append(fits.ImageHDU(full_cov, name=f"COVAR_{pa}_{pb}")) + + else: + block_path = self._output_path_iNKA_block_cov( + ver, tomo_bin_quad=("all", "all", "all", "all") + ) + covar = fits.open(block_path) - cl_noise = np.zeros_like(cl_shear) - rng = np.random.default_rng(self.cell_seed) + covar.writeto(out_path, overwrite=True) - for i in range(self.nrandom_cell): - noise_map_e1 = np.zeros(hp.nside2npix(nside)) - noise_map_e2 = np.zeros(hp.nside2npix(nside)) + return covar - e1_rot, e2_rot = self.apply_random_rotation(e1, e2, rng) + # ---------------- Plotting functions for pseudo-Cl's ---------------- # + def plot_pseudo_cl( + self, + pol_list, + versions=None, + ell_factor="ell", + cov_type="iNKA", + offset=0.15, + tomography=True, + savefig=None, + show=True, + ): + """ + Plot the pseudo-Cl for EE power spectrum. - noise_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1_rot * w) - noise_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2_rot * w) + Parameters + ---------- + pol_list : list + List of polarization types to plot (e.g., ["EE", "BB"]). + ell_factor : {"None", "ell", "ell(ell+1)"} + Factor to multiply the ell values by. + cov_type : {"iNKA"} + Type of covariance to use. + tomography : bool, optional + Whether to plot tomographic power spectra. + savefig : str, optional + Path to save the figure. + show : bool, optional + Whether to show the figure. + + Returns + ------- + fig, ax : matplotlib.figure.Figure, matplotlib.axes.Axes + Figure and axes objects for the plot. + """ + if versions is None: + versions = self.versions + else: + for ver in versions: + if ver not in self.versions: + raise ValueError( + f"Version {ver} is not available. Available versions: {self.versions}" + ) + # Check that the method for the covariance is valid + if cov_type not in ["iNKA"]: + raise ValueError( + f"Invalid covariance type: {cov_type}. Valid options are: ['iNKA']" + ) - noise_map_e1[mask] /= n_gal_map[mask] - noise_map_e2[mask] /= n_gal_map[mask] + # Check that the ell_factor is valid + if ell_factor not in ["None", "ell", "ell(ell+1)"]: + raise ValueError( + f"Invalid ell_factor: {ell_factor}. Valid options are: ['None', 'ell', 'ell(ell+1)']" + ) - noise_map = noise_map_e1 + 1j * noise_map_e2 - del noise_map_e1, noise_map_e2 + def get_ell_factor(ell): + """Given some ell, return the ell_factor for the plot.""" + if ell_factor == "None": + return 1 + elif ell_factor == "ell": + return ell + elif ell_factor == "ell(ell+1)": + return ell * (ell + 1) + + # Check that all items in the list are valid polarisation + valid_pols = ["EE", "BB", "EB", "BE"] + for pol in pol_list: + if pol not in valid_pols: + raise ValueError( + f"Invalid polarization type: {pol}. Valid options are: {valid_pols}" + ) - _, cl_noise_, _ = self.get_pseudo_cls_map(noise_map, n_gal_map, wsp) - cl_noise += cl_noise_ + fmt_dict = {"EE": "o", "BB": "s", "EB": "^", "BE": "v"} + # From all the versions, get the maximum number of tomo_bin_ids + tomo_bins = self._get_tomo_bins_for_versions(versions, tomography=tomography) - cl_noise /= self.nrandom_cell - del e1, e2, w - try: - del e1_rot, e2_rot - except NameError: # Continue if the random generation has been skipped. - pass - del n_gal_map + max_key = max(tomo_bins, key=lambda k: len(tomo_bins[k]["ids"])) + n_tomo_bins_plot = len(tomo_bins[max_key]["ids"]) + reference_tomo_bin_pairs = tomo_bins[max_key]["pairs"] - # Noise realizations are now reproducible (seeded rng from self.cell_seed). - cl_shear = cl_shear - cl_noise + fig, axs = plt.subplots( + n_tomo_bins_plot, + n_tomo_bins_plot, + figsize=(12, 12), + sharex=True, + sharey=True, + ) - self.print_cyan("Saving pseudo-Cl's...") - self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) + for j, ver in enumerate(versions): + # Plot the pseudo-cl for each tomo bin of the considered version + for tomo_bin_a, tomo_bin_b in tomo_bins[ver]["pairs"]: + if tomography: + ax = axs[tomo_bin_b - 1, tomo_bin_a - 1] + else: + ax = axs + ver_tomo_info = self._pseudo_cls[ver][ + f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}" + ] + ver_label = self.cc[ver]["label"] if "label" in self.cc[ver] else ver + ver_color = ( + self.cc[ver]["colour"] if "colour" in self.cc[ver] else "black" + ) - self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) + pseudo_cls = ver_tomo_info["pseudo_cl"] + cov = ver_tomo_info["cov"] + + ell = pseudo_cls["ELL"] + + ell_widths = np.diff(ell) + ell_widths = np.append( + ell_widths, ell_widths[-1] + ) # Assume last bin width is same as second last + + # Better jittering: symmetric around original ell values + jitter_fraction = (j - (len(versions) - 1) / 2) * offset + jittered_ell = ell + jitter_fraction * ell_widths + ell_factor_ = get_ell_factor(ell) + + for pol in pol_list: + pol_color = self.get_pol_color(ver_color, pol, pol_list) + ax.errorbar( + jittered_ell, + ell_factor_ * pseudo_cls[pol], + yerr=np.sqrt(np.diag(cov[f"COVAR_{pol}_{pol}"].data)) + * ell_factor_, + fmt=fmt_dict[pol], + label=ver_label + f" {pol}", + color=pol_color, + capsize=2, + ) - def calculate_pseudo_cl_catalog(self, ver, out_path): - params = get_params_rho_tau(self.cc[ver], survey=ver) + # Draw to extract the yaxis text offset + fig.canvas.draw() - # Load data and create shear and noise maps - cat_gal = open_entry(self.cc[ver]["shear"]) + if ell_factor == "None": + ell_label = r"$C_\ell$" + elif ell_factor == "ell": + ell_label = r"$\ell C_\ell$" + else: + ell_label = r"$\ell(\ell+1) C_\ell$" - ell_eff, cl_shear, wsp = self.get_pseudo_cls_catalog( - catalog=cat_gal, params=params - ) + for tomo_bin_a, tomo_bin_b in reference_tomo_bin_pairs: + if tomography: + ax = axs[tomo_bin_b - 1, tomo_bin_a - 1] + else: + ax = axs + ax.text( + 0.8, + 0.95, + f"{tomo_bin_a}-{tomo_bin_b}", + transform=ax.transAxes, + verticalalignment="top", + ) + ax.axhline(0, color="black", ls="--") + ax.set_xlim(ell.min(), ell.max()) + ax.set_xscale("squareroot") + ax.set_xticks(np.array([100, 400, 900, 1600])) + ax.minorticks_on() + minor_ticks = [i * 10 for i in range(1, 10)] + [ + i * 100 for i in range(1, 21) + ] + ax.xaxis.set_ticks(minor_ticks, minor=True) + ax.tick_params(axis="both", which="both", direction="in") + if tomo_bin_b == 6 or tomo_bin_b == "all": + ax.set_xlabel(r"$\ell$") + if tomo_bin_a == 1 or tomo_bin_a == "all": + text_offset = ax.yaxis.get_offset_text().get_text() + ax.yaxis.get_offset_text().set_visible(False) + ax.set_ylabel(f"{ell_label}{text_offset}") + else: + ax.yaxis.get_offset_text().set_visible(False) + if tomo_bin_a != tomo_bin_b: + ax = axs[tomo_bin_a - 1, tomo_bin_b - 1] + ax.set_visible(False) - self.print_cyan("Saving pseudo-Cl's...") - self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) + # Setup the legend + plt.subplots_adjust(hspace=0.0, wspace=0.0) # Remove space between subplots - self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) + legend_ax = self._add_grouped_legend(fig, versions, self.cc, pol_list, fmt_dict) - def get_n_gal_map(self, params, nside, cat_gal): - """Weighted galaxy number-density map (thin wrapper -> primitive).""" - return get_n_gal_map( - nside, - cat_gal[params["ra_col"]], - cat_gal[params["dec_col"]], - weights=cat_gal[params["w_col"]], - ) + if savefig is not None: + plt.savefig(savefig, dpi=300, bbox_inches="tight") + if show: + plt.show() - def get_gaussian_real( - self, params, nside, lmax, cat_gal, n_gal, mask, unique_pix, idx_rep, rng=None + return fig, axs, legend_ax + + def _add_grouped_legend( + self, + fig, + versions, + cc, + pol_list, + fmt_dict, + row_height=0.35, + label_width=2, + col_width=0.9, + gap_below=0.10, + capsize=2, + fontsize=10, ): - e1_rot, e2_rot = self.apply_random_rotation( - cat_gal[params["e1_col"]], cat_gal[params["e2_col"]], rng - ) - noise_map_e1 = np.zeros(hp.nside2npix(nside)) - noise_map_e2 = np.zeros(hp.nside2npix(nside)) + """ + Add a custom legend below the figure with one row per version: + ... - w = cat_gal[params["w_col"]] - noise_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1_rot * w) - noise_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2_rot * w) - noise_map_e1[mask] /= n_gal[mask] - noise_map_e2[mask] /= n_gal[mask] + The box size (in inches) scales with the number of versions (rows) + and polarizations (columns), then gets converted to figure-fraction + coordinates so it works for any figsize. - return noise_map_e1 + 1j * noise_map_e2 + Parameters + ---------- + row_height : float + Height per row (per version), in inches. + label_width : float + Width reserved for the version label column, in inches. + col_width : float + Width per polarization column (marker + pol label), in inches. + gap_below : float + Vertical gap between the bottom of the subplot grid and the + top of the legend box, in inches. + """ + n_rows = len(versions) + n_pol = len(pol_list) - def get_sample( - self, - params, - nside, - lmax, - b, - cat_gal, - n_gal, - mask, - unique_pix, - idx_rep, - rng=None, - ): - noise_map = self.get_gaussian_real( - params, nside, lmax, cat_gal, n_gal, mask, unique_pix, idx_rep, rng - ) + # Desired box size in inches + box_width_in = label_width + n_pol * col_width + box_height_in = n_rows * row_height - f = nmt.NmtField(mask=mask, maps=[noise_map.real, noise_map.imag], lmax=lmax) + fig_w_in, fig_h_in = fig.get_size_inches() - wsp = nmt.NmtWorkspace.from_fields(f, f, b) + # Convert to figure-fraction + width_frac = box_width_in / fig_w_in + height_frac = box_height_in / fig_h_in + gap_frac = gap_below / fig_h_in - cl_noise = nmt.compute_coupled_cell(f, f) - cl_noise = wsp.decouple_cell(cl_noise) + left = 0.5 - width_frac / 2 # centered horizontally + bottom = -gap_frac - height_frac # just below the subplot grid (y=0) - return cl_noise, f, wsp + legend_ax = fig.add_axes([left, bottom, width_frac, height_frac]) + legend_ax.axis("off") + legend_ax.set_xlim(0, 1) + legend_ax.set_ylim(0, 1) - def get_pseudo_cls_map(self, map, mask, wsp=None): - """Map-based pseudo-cl (thin wrapper, state -> primitive).""" - return get_pseudo_cls_map( - map, - mask, - self.nside, - self.binning, - pol_factor=self.pol_factor, - wsp=wsp, - ell_step=self.ell_step, - n_ell_bins=self.n_ell_bins, - power=self.power, - ) + # Fractions *within* legend_ax's own 0-1 coordinate system, derived + # from the same inch-based proportions so columns stay consistent + label_frac = label_width / box_width_in + col_frac = col_width / box_width_in - def get_pseudo_cls_catalog(self, catalog, params, wsp=None): - """Catalog-based pseudo-cl (thin wrapper, state -> primitive).""" - return get_pseudo_cls_catalog( - catalog, - params, - self.nside, - self.binning, - pol_factor=self.pol_factor, - wsp=wsp, - ell_step=self.ell_step, - n_ell_bins=self.n_ell_bins, - power=self.power, - ) + dummy_yerr = 0.15 / n_rows - def apply_random_rotation(self, e1, e2, rng=None): - """Random ellipticity rotation (thin wrapper -> primitive). + for i, ver in enumerate(versions): + y = 1.0 - (i + 0.5) / n_rows - Pass a seeded ``rng`` for reproducible noise realizations. - """ - return apply_random_rotation(e1, e2, rng) + ver_label = cc[ver]["label"] if "label" in cc[ver] else ver + ver_color = cc[ver]["colour"] if "colour" in cc[ver] else "black" - def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): - """Write the pseudo-Cl SACC part (EE/BB/EB + shared bandpower window). + legend_ax.text( + 0.02 * label_frac, + y, + ver_label, + ha="left", + va="center", + fontweight="bold", + fontsize=fontsize, + ) + + for k, pol in enumerate(pol_list): + pol_color = self.get_pol_color(ver_color, pol, pol_list) + x_marker = label_frac + k * col_frac + 0.15 * col_frac + x_text = x_marker + 0.15 * col_frac + + legend_ax.errorbar( + [x_marker], + [y], + yerr=dummy_yerr, + fmt=fmt_dict[pol], + color=pol_color, + markersize=6, + capsize=capsize, + clip_on=False, + ) + legend_ax.text( + x_text, + y, + pol, + ha="left", + va="center", + fontsize=fontsize - 1, + ) - ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); - the writer takes the shared bandpower window from ``wsp``. No covariance - is attached here. + return legend_ax + + def get_pol_color(self, base_color, pol, pol_list, lightness_range=(-0.25, 0.25)): """ - s = pseudo_cl_to_sacc( - self.sacc_nz(version), - self.sacc_metadata(version), - ell_eff, - cl_all, - wsp, - ) - sacc_io.save(s, out_path, type="data") + Given a base version color, return a shade variant for a given + polarization, by adjusting lightness in HLS space while keeping + hue and saturation fixed. - def plot_pseudo_cl(self): - """Plot the EE/EB/BB pseudo-Cl spectra for every version.""" - self.print_cyan("Plotting pseudo-Cl's") - - for spectrum in ("EE", "EB", "BB"): - datasets = { - ver: { - "ell": self.pseudo_cls[ver]["pseudo_cl"]["ELL"], - "cl": self.pseudo_cls[ver]["pseudo_cl"][spectrum], - "cov": self.pseudo_cls[ver]["cov"][ - f"COVAR_{spectrum}_{spectrum}" - ].data, - "style": { - "marker": self.cc[ver]["marker"], - "colour": self.cc[ver]["colour"], - }, - } - for ver in self.versions - } - plot_pseudo_cl_spectrum( - datasets, spectrum, self._output_path(f"cell_{spectrum.lower()}.png") + Parameters + ---------- + base_color : str or tuple + Any matplotlib-recognized color (hex, name, RGB tuple, etc.) + pol : str + The polarization this color is for (e.g. "EE"). + pol_list : list + Full list of polarizations being plotted, used to compute + this pol's relative position in the lightness range. + lightness_range : tuple of float + (min_offset, max_offset) added to the base lightness, spread + evenly across pol_list. Negative = darker, positive = lighter. + Values are in HLS lightness units (0-1 scale), so keep these + modest (e.g. +/-0.25) to avoid washing out to white or black. + + Returns + ------- + tuple + RGB color tuple in [0, 1] range, usable directly in matplotlib. + """ + r, g, b = to_rgb(base_color) + h, lightness, s = colorsys.rgb_to_hls(r, g, b) + + n_pol = len(pol_list) + idx = pol_list.index(pol) + + if n_pol == 1: + l_offset = 0.0 + else: + # spread idx evenly across [lightness_range[0], lightness_range[1]] + frac = idx / (n_pol - 1) # 0 to 1 + l_offset = lightness_range[0] + frac * ( + lightness_range[1] - lightness_range[0] ) - for ver in self.versions: - cl_bb = self.pseudo_cls[ver]["pseudo_cl"]["BB"] - cov_bb = self.pseudo_cls[ver]["cov"]["COVAR_BB_BB"].data - chi2_bb, _, pte_bb = chi2_and_pte(cl_bb, cov_bb) - print( - f" {ver}: C_l^BB PTE = {pte_bb:.4f} " - f"(chi2/dof = {float(chi2_bb):.1f}/{len(cl_bb)})" - ) + lightness_new = min( + max(lightness + l_offset, 0.05), 0.95 + ) # clamp to avoid pure black/white + + r_new, g_new, b_new = colorsys.hls_to_rgb(h, lightness_new, s) + return (r_new, g_new, b_new) diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index 8a402bd5..b189e622 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -6,8 +6,10 @@ """ import os +from pathlib import Path import matplotlib.pyplot as plt +import matplotlib.ticker as mticker import numpy as np import treecorr from cs_util import plots as cs_plots @@ -17,6 +19,7 @@ from uncertainties import ufloat from .. import sacc_io +from ..io import open_entry from ..rho_tau import ( get_rho_tau_w_cov, get_samples, @@ -24,29 +27,105 @@ from .sacc_writers import rho_tau_to_sacc +# TODO: Reorganise the order of functions so it is more readable class PSFSystematicsMixin: - def calculate_rho_tau_stats(self): - """Measure ρ/τ statistics per version and write each version's SACC part.""" + # --- property definitions --- + @property + def rho_stat_handler(self): + if not hasattr(self, "_rho_stat_handler"): + self.calculate_rho_tau_stats(tomography=False) + if self.compute_tomography: + self.calculate_rho_tau_stats(tomography=True) + return self._rho_stat_handler + + @property + def tau_stat_handler(self): + if not hasattr(self, "_tau_stat_handler"): + self.calculate_rho_tau_stats(tomography=False) + if self.compute_tomography: + self.calculate_rho_tau_stats(tomography=True) + return self._tau_stat_handler + + @property + def psf_fitter(self): + if not hasattr(self, "_psf_fitter"): + self._psf_fitter = PSFErrorFit( + self.rho_stat_handler, + self.tau_stat_handler, + self.rho_stat_handler.catalogs._output, + ) + return self._psf_fitter + + @property + def rho_tau_fits(self): + if not hasattr(self, "_rho_tau_fits"): + self.calculate_rho_tau_fits(tomography=False) + if self.compute_tomography: + self.calculate_rho_tau_fits(tomography=True) + return self._rho_tau_fits + + @property + def xi_psf_sys(self): + if not hasattr(self, "_xi_psf_sys"): + self.calculate_rho_tau_fits(tomography=False) + if self.compute_tomography: + self.calculate_rho_tau_fits(tomography=True) + return self._xi_psf_sys + + # --- calculate functions --- + def calculate_rho_tau_stats(self, tomography=True): + """Measure ρ/τ statistics per version and tomographic bin. + + Without ``tomography`` the single bin is ``"all"``, and its SACC part is + written per version (``rho_tau_to_sacc_part``). + """ out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" if not os.path.exists(out_dir): os.mkdir(out_dir) self.print_start("Rho stats") for ver in self.versions: - base = self.basename(ver) - rho_stat_handler, tau_stat_handler = get_rho_tau_w_cov( - self.cc, - ver, - self.treecorr_config, - out_dir, - base, - method=self.cov_estimate_method, - cov_rho=self.compute_cov_rho, - npatch=self.npatch, - ) - self.rho_tau_to_sacc_part( - ver, out_dir, base, rho_stat_handler, tau_stat_handler - ) + # Get the tomographic bins + if tomography: + tomo_bin_ids, tomo_bin_pairs = self._get_tomo_bins(ver) + + if tomo_bin_ids is None or tomo_bin_pairs is None: + raise ValueError( + f"Version {ver} does not have tomography information." + ) + + else: + tomo_bin_ids, tomo_bin_pairs = ["all"], [("all", "all")] + + # Get the selection for stars + mask_star = self._get_star_mask(ver) + + for tomo_bin_id in tomo_bin_ids: + self.print_cyan(f"Computing for the tomographic bin: {tomo_bin_id}") + base_rho = self.basename(ver) + base_tau = self.basename(ver, tomo_bin_a=tomo_bin_id) + + # Get the selection for galaxies + mask_gal = self._get_galaxy_mask(ver, tomo_bin_id) + + # Compute rho and tau statistics + rho_stat_handler, tau_stat_handler = get_rho_tau_w_cov( + self.cc, + ver, + self.treecorr_config, + out_dir, + base_rho, + base_tau, + mask_star=mask_star, + mask_gal=mask_gal, + method=self.cov_estimate_method, + cov_rho=self.compute_cov_rho, + npatch=self.npatch, + ) + if not tomography: + self.rho_tau_to_sacc_part( + ver, out_dir, base_tau, rho_stat_handler, tau_stat_handler + ) self.print_done("Rho stats finished") self._rho_stat_handler = rho_stat_handler @@ -80,233 +159,221 @@ def rho_tau_to_sacc_part( out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") sacc_io.save(s, out_path, type="data") - @property - def rho_stat_handler(self): - if not hasattr(self, "_rho_stat_handler"): - self.calculate_rho_tau_stats() - return self._rho_stat_handler - - @property - def tau_stat_handler(self): - if not hasattr(self, "_tau_stat_handler"): - self.calculate_rho_tau_stats() - return self._tau_stat_handler - - def plot_rho_stats(self, abs=False): - filenames = [f"rho_stats_{self.basename(ver)}.fits" for ver in self.versions] - - savefig = "rho_stats.png" - self.rho_stat_handler.plot_rho_stats( - filenames, - self.colors, - self.versions, - savefig=savefig, - legend="outside", - abs=abs, - show=True, - close=True, - ) + def calculate_rho_tau_fits(self, tomography=True, track_result=True): + assert self.rho_tau_method != "none" - self.print_done( - "Rho stats plot saved to " - + f"{os.path.abspath(self.rho_stat_handler.catalogs._output)}/{savefig}", - ) + # this initializes the rho_tau_fits attribute + if not hasattr(self, "_rho_tau_fits"): + self._rho_tau_fits = {} + quantiles = [1 - self.quantile, self.quantile] - def plot_tau_stats(self, plot_tau_m=False): - filenames = [f"tau_stats_{self.basename(ver)}.fits" for ver in self.versions] + if not hasattr(self, "_xi_psf_sys"): + self._xi_psf_sys = {} + for ver in self.versions: + params = self.set_params_rho_tau( + ver, self.results[ver]._params, self.cc[ver]["psf"] + ) - savefig = "tau_stats.png" - self.tau_stat_handler.plot_tau_stats( - filenames, - self.colors, - self.versions, - savefig=savefig, - legend="outside", - plot_tau_m=plot_tau_m, - show=True, - close=True, - ) + # Get the tomographic bins + if tomography: + tomo_bin_ids, tomo_bin_pairs = self._get_tomo_bins(ver) - self.print_done( - "Tau stats plot saved to " - + f"{os.path.abspath(self.tau_stat_handler.catalogs._output)}/{savefig}", - ) + if tomo_bin_ids is None or tomo_bin_pairs is None: + raise ValueError( + f"Version {ver} does not have tomography information." + ) - def set_params_rho_tau(self, params, params_psf, patch_number, survey="other"): - params = {**params, "patch_number": patch_number} - params["ra_PSF_col"] = params_psf["ra_col"] - params["dec_PSF_col"] = params_psf["dec_col"] - params["e1_PSF_col"] = params_psf["e1_PSF_col"] - params["e2_PSF_col"] = params_psf["e2_PSF_col"] - params["e1_star_col"] = params_psf["e1_star_col"] - params["e2_star_col"] = params_psf["e2_star_col"] - params["PSF_size"] = params_psf["PSF_size"] - params["star_size"] = params_psf["star_size"] - if survey != "DES": - params["PSF_flag"] = params_psf["PSF_flag"] - params["star_flag"] = params_psf["star_flag"] - params["ra_units"] = "deg" - params["dec_units"] = "deg" + else: + tomo_bin_ids, tomo_bin_pairs = ["all"], [("all", "all")] - params["w_col"] = self.cc[survey]["shear"]["w_col"] + # Get the samples for each tomographic bin + for tomo_bin_id in tomo_bin_ids: + self.print_cyan( + f"Sample PSF error parameters for tomographic bin {tomo_bin_id}" + ) + _ = self.get_samples( + ver, params, tomo_bin_id, track_result=track_result + ) - return params + # Get the xi_psf_sys for each tomographic bin pairs + for tomo_bin_a, tomo_bin_b in tomo_bin_pairs: + xi_psf_sys_samples_plus, xi_psf_sys_samples_minus = ( + self.get_xi_psf_sys_samples(ver, params, tomo_bin_a, tomo_bin_b) + ) - @property - def psf_fitter(self): - if not hasattr(self, "_psf_fitter"): - self._psf_fitter = PSFErrorFit( - self.rho_stat_handler, - self.tau_stat_handler, - self.rho_stat_handler.catalogs._output, - ) - return self._psf_fitter + if ver not in self._xi_psf_sys.keys(): + self._xi_psf_sys[ver] = {} + + self._xi_psf_sys[ver][ + f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}" + ] = { + "mean_plus": np.mean(xi_psf_sys_samples_plus, axis=0), + "var_plus": np.var(xi_psf_sys_samples_plus, axis=0), + "quantiles_plus": np.quantile( + xi_psf_sys_samples_plus, quantiles, axis=0 + ), + "mean_minus": np.mean(xi_psf_sys_samples_minus, axis=0), + "var_minus": np.var(xi_psf_sys_samples_minus, axis=0), + "quantiles_minus": np.quantile( + xi_psf_sys_samples_minus, quantiles, axis=0 + ), + } - def calculate_rho_tau_fits(self): - assert self.rho_tau_method != "none" + def calculate_scale_dependent_leakage(self): + # TODO: Upgrade for tomography + self.print_start("Calculating scale-dependent leakage:") + for ver in self.versions: + self.print_magenta(ver) + results = self.results[ver] - # this initializes the rho_tau_fits attribute - self._rho_tau_fits = {"flat_sample_list": [], "result_list": [], "q_list": []} - quantiles = [1 - self.quantile, self.quantile] + output_base_path = self._output_path(f"leakage_{ver}/xi_for_leak_scale") + output_path_ab = f"{output_base_path}_a_b.txt" + output_path_aa = f"{output_base_path}_a_a.txt" + with self.results[ver].temporarily_read_data(): + if os.path.exists(output_path_ab) and os.path.exists(output_path_aa): + self.print_green( + f"Skipping computation, reading {output_path_ab} and " + f"{output_path_aa} instead" + ) - self._xi_psf_sys = {} - for ver in self.versions: - params = self.set_params_rho_tau( - self.results[ver]._params, - self.cc[ver]["psf"], - self.cc[ver]["patch_number"], - survey=ver, - ) + results.r_corr_gp = treecorr.GGCorrelation(self.treecorr_config) + results.r_corr_gp.read(output_path_ab) - npatch = {"sim": 300, "jk": params["patch_number"]}.get( - self.cov_estimate_method, None - ) + results.r_corr_pp = treecorr.GGCorrelation(self.treecorr_config) + results.r_corr_pp.read(output_path_aa) - base = self.basename(ver) + else: + results.compute_corr_gp_pp_alpha(output_base_path=output_base_path) - flat_samples, result, q = get_samples( - self.psf_fitter, - ver, - base, - cov_type=self.cov_estimate_method, - apply_debias=npatch, - sampler=self.rho_tau_method, - ) + results.do_alpha(fast=True) + results.do_xi_sys() - self.rho_tau_fits["flat_sample_list"].append(flat_samples) - self.rho_tau_fits["result_list"].append(result) - self.rho_tau_fits["q_list"].append(q) + self.print_done("Finished scale-dependent leakage calculation.") - self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") - nbins = self.psf_fitter.rho_stat_handler._treecorr_config["nbins"] - xi_psf_sys_samples = np.array( - [self.psf_fitter.compute_xi_psf_sys(sample) for sample in flat_samples] - ).reshape(-1, nbins) + def calculate_objectwise_leakage(self, tomography=False): + """Fit the object-wise PSF leakage, per version and tomographic bin. - self._xi_psf_sys[ver] = { - "mean": np.mean(xi_psf_sys_samples, axis=0), - "var": np.var(xi_psf_sys_samples, axis=0), - "quantiles": np.quantile(xi_psf_sys_samples, quantiles, axis=0), - } + Stores ``a11``, ``a22`` and ``aii_mean`` (as ufloats) in + ``self.leakage_coeff[ver]["tomo_bin_"]``, with ```` ``all`` when + ``tomography`` is False. A version whose catalogue lacks a required + column is dropped from ``results_objectwise`` and ``leakage_coeff``. + """ + tomo_bins = self._get_tomo_bins_for_versions( + self.versions, tomography=tomography + ) - @property - def rho_tau_fits(self): - if not hasattr(self, "_rho_tau_fits"): - self.calculate_rho_tau_fits() - return self._rho_tau_fits + self.print_start("Object-wise leakage:") + mix = True + order = "lin" + if not hasattr(self, "leakage_coeff"): + self.leakage_coeff = {} + for ver in self.versions: + if ver not in self.results_objectwise: + # Dropped earlier for a missing catalogue column + continue + self.print_magenta(ver) - def plot_rho_tau_fits(self): - out_dir = self.rho_stat_handler.catalogs._output + results_obj = self.results_objectwise[ver] + results_obj.check_params() + results_obj.update_params() + results_obj.prepare_output() - savefig = f"{out_dir}/contours_tau_stat.png" - psfleak_plots.plot_contours( - self.rho_tau_fits["flat_sample_list"], - names=["x0", "x1", "x2"], - labels=[r"\alpha", r"\beta", r"\eta"], - savefig=savefig, - legend_labels=self.versions, - legend_loc="upper right", - contour_colors=self.colors, - markers={"x0": 0, "x1": 1, "x2": 1}, - show=True, - close=True, + coeff_ver = self.leakage_coeff.setdefault(ver, {}) + try: + for tomo_bin_id in tomo_bins[ver]["ids"]: + coeff_ver[f"tomo_bin_{tomo_bin_id}"] = self._objectwise_leakage_bin( + results_obj, ver, tomo_bin_id, mix, order + ) + except KeyError as e: + print(f"{e}\nExpected key is missing from catalog.") + self.results_objectwise.pop(ver) + self.leakage_coeff.pop(ver) + + def _objectwise_leakage_bin(self, results_obj, ver, tomo_bin_id, mix, order): + """Object-wise leakage coefficients of one tomographic bin.""" + selection = ( + None if tomo_bin_id == "all" else self._get_galaxy_mask(ver, tomo_bin_id) ) - self.print_done(f"Tau contours plot saved to {os.path.abspath(savefig)}") + suffix = f"tomo_bin_{tomo_bin_id}" - plt.figure(figsize=(15, 6)) - for mcmc_result, ver, color, flat_sample in zip( - self.rho_tau_fits["result_list"], - self.versions, - self.colors, - self.rho_tau_fits["flat_sample_list"], - ): - self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") - for i in range(100): - self.psf_fitter.plot_xi_psf_sys( - flat_sample[-i + 1], ver, color, alpha=0.1 + out_path = f"{results_obj.get_out_base(mix, order, suffix=suffix)}.pkl" + if os.path.exists(out_path): + self.print_green(f"Skipping object-wise leakage, file {out_path} exists") + results_obj.par_best_fit = leakage.read_from_file(out_path) + else: + self.print_cyan(f"Computing object-wise leakage regression, {suffix}") + with results_obj.temporarily_read_data(selection=selection): + results_obj.PSF_leakage(mix=mix, order=order, suffix=suffix) + + par_best_fit = results_obj.par_best_fit + a11 = ufloat(par_best_fit["a11"].value, par_best_fit["a11"].stderr) + a22 = ufloat(par_best_fit["a22"].value, par_best_fit["a22"].stderr) + return {"a11": a11, "a22": a22, "aii_mean": 0.5 * (a11 + a22)} + + def calculate_alpha_leakage_summaries(self, tomography=False, cov_type=None): + """Summarise the scale-dependent leakage alpha(theta) = tau_0 / rho_0. + + For each version in ``leakage_coeff`` and each of its tomographic bins, + stores ``alpha_mean``, ``alpha_1`` and ``alpha_0`` (see + `_alpha_summaries`) next to the object-wise coefficients in + ``self.leakage_coeff[ver]["tomo_bin_"]``. + """ + tomo_bins = self._get_tomo_bins_for_versions( + list(self.leakage_coeff), tomography=tomography + ) + for ver, coeff_ver in self.leakage_coeff.items(): + for tomo_bin_id in tomo_bins[ver]["ids"]: + theta, alpha, alpha_err = self._load_alpha_leakage( + ver, tomo_bin_id, cov_type + ) + coeff_ver.setdefault(f"tomo_bin_{tomo_bin_id}", {}).update( + self._alpha_summaries(theta, alpha, alpha_err) ) - self.psf_fitter.plot_xi_psf_sys(mcmc_result[1], ver, color) - plt.legend() - out_path = os.path.abspath(f"{out_dir}/xi_psf_sys_samples.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"xi_psf_sys samples plot saved to {out_path}") - - plt.figure(figsize=(15, 6)) - for mcmc_result, ver, color, flat_sample in zip( - self.rho_tau_fits["result_list"], - self.versions, - self.colors, - self.rho_tau_fits["flat_sample_list"], - ): - ls = self.cc[ver]["ls"] - theta = self.psf_fitter.rho_stat_handler.rho_stats["theta"] - xi_psf_sys = self.xi_psf_sys[ver] - plt.plot(theta, xi_psf_sys["mean"], linestyle=ls, color=color) - plt.plot(theta, xi_psf_sys["quantiles"][0], linestyle=ls, color=color) - plt.plot(theta, xi_psf_sys["quantiles"][1], linestyle=ls, color=color) - plt.fill_between( - theta, - xi_psf_sys["quantiles"][0], - xi_psf_sys["quantiles"][1], - color=color, - alpha=0.25, - label=ver, - ) - plt.xscale("log") - plt.yscale("log") - plt.xlabel(r"$\theta$ [arcmin]") - plt.ylabel(r"$\xi^{\rm PSF}_{\rm sys}$") - plt.title(f"{1 - self.quantile:.1%}, {self.quantile:.1%} quantiles") - plt.legend() - out_path = os.path.abspath(f"{out_dir}/xi_psf_sys_quantiles.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"xi_psf_sys quantiles plot saved to {out_path}") + # --- utility functions --- + # The masks index the rows of the catalogue entry as open_entry reads them, + # which are the rows rho_tau._CatalogueLoader hands to shear_psf_leakage. + def _get_galaxy_mask(self, ver, tomo_bin_id): + cat_gal = open_entry(self.cc[ver]["shear"]) + if tomo_bin_id != "all": + gal_mask = cat_gal[self.cc[ver]["shear"]["tomo_bin_col"]] == tomo_bin_id + else: + gal_mask = np.ones(len(cat_gal), dtype=bool) + return gal_mask + + def _get_star_mask(self, ver): + cat_star = open_entry(self.cc[ver]["psf"]) + PSF_flag = self.cc[ver]["psf"].get("PSF_flag") + star_flag = self.cc[ver]["psf"].get("star_flag") + if PSF_flag is not None: + if star_flag is not None: + star_mask = (cat_star[PSF_flag] == 0) & (cat_star[star_flag] == 0) + else: + star_mask = cat_star[PSF_flag] == 0 + else: + star_mask = np.ones(len(cat_star), dtype=bool) + return star_mask - for mcmc_result, ver, flat_sample in zip( - self.rho_tau_fits["result_list"], - self.versions, - self.rho_tau_fits["flat_sample_list"], - ): - self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") - for yscale in ("linear", "log"): - out_path = os.path.abspath( - f"{out_dir}/xi_psf_sys_terms_{yscale}_{ver}.png" - ) - self.psf_fitter.plot_xi_psf_sys_terms( - ver, mcmc_result[1], out_path, yscale=yscale, show=True - ) - self.print_done( - f"{yscale}-scale xi_psf_sys terms plot saved to {out_path}" - ) + def set_params_rho_tau(self, ver, params, params_psf): + params = {**params} - @property - def xi_psf_sys(self): - if not hasattr(self, "_xi_psf_sys"): - self.calculate_rho_tau_fits() - return self._xi_psf_sys + params["ra_PSF_col"] = params_psf["ra_col"] + params["dec_PSF_col"] = params_psf["dec_col"] + params["e1_PSF_col"] = params_psf["e1_PSF_col"] + params["e2_PSF_col"] = params_psf["e2_PSF_col"] + params["e1_star_col"] = params_psf["e1_star_col"] + params["e2_star_col"] = params_psf["e2_star_col"] + params["PSF_size"] = params_psf["PSF_size"] + params["star_size"] = params_psf["star_size"] + params["PSF_flag"] = params_psf.get("PSF_flag") + params["star_flag"] = params_psf.get("star_flag") + params["ra_units"] = "deg" + params["dec_units"] = "deg" + + params["w_col"] = self.cc[ver]["shear"]["w_col"] + params["patch_number"] = self.cc[ver]["patch_number"] + + return params def set_params_leakage_scale(self, ver): params_in = {} @@ -367,287 +434,1235 @@ def set_params_leakage_object(self, ver): return params_in - def calculate_scale_dependent_leakage(self): - self.print_start("Calculating scale-dependent leakage:") - for ver in self.versions: - self.print_magenta(ver) - results = self.results[ver] + def set_psf_parameter_sampling_method(self, rho_tau_method): + if rho_tau_method not in ["emcee", "lsq"]: + raise ValueError("Invalid PSF parameter sampling method.") + self.rho_tau_method = rho_tau_method - output_base_path = self._output_path(f"leakage_{ver}/xi_for_leak_scale") - output_path_ab = f"{output_base_path}_a_b.txt" - output_path_aa = f"{output_base_path}_a_a.txt" - with self.results[ver].temporarily_read_data(): - if os.path.exists(output_path_ab) and os.path.exists(output_path_aa): - self.print_green( - f"Skipping computation, reading {output_path_ab} and " - f"{output_path_aa} instead" - ) + def set_psf_parameter_nsamples(self, nsamples): + self.psf_error_nsamples = nsamples - results.r_corr_gp = treecorr.GGCorrelation(self.treecorr_config) - results.r_corr_gp.read(output_path_ab) + def set_psf_parameter_nwalkers(self, nwalkers): + self.psf_error_nwalkers = nwalkers - results.r_corr_pp = treecorr.GGCorrelation(self.treecorr_config) - results.r_corr_pp.read(output_path_aa) + def get_samples(self, version, params, tomo_bin_id, track_result=False): + # Hartlap debiasing of the inverse covariance: the number of jackknife + # patches or of simulations the covariance was estimated from. + n_realisations = {"jk": params["patch_number"], "sim": self.n_sim_cov}.get( + self.cov_estimate_method + ) - else: - results.compute_corr_gp_pp_alpha(output_base_path=output_base_path) + base_rho = self.basename(version) + base_tau = self.basename(version, tomo_bin_a=tomo_bin_id) - results.do_alpha(fast=True) - results.do_xi_sys() + # Set the number of samples and walkers and fallback to defaults if not attributed. + n_samples = ( + self.psf_error_nsamples if hasattr(self, "psf_error_nsamples") else 10_000 + ) + n_walkers = ( + self.psf_error_nwalkers if hasattr(self, "psf_error_nwalkers") else 124 + ) - self.print_done("Finished scale-dependent leakage calculation.") + flat_samples, result, q = get_samples( + self.psf_fitter, + base_rho, + base_tau, + cov_type=self.cov_estimate_method, + apply_debias=n_realisations, + sampler=self.rho_tau_method, + nsamples=n_samples, + nwalkers=n_walkers, + ) + + if track_result: + if version not in self.rho_tau_fits.keys(): + self.rho_tau_fits[version] = { + "flat_samples": {}, + "result": {}, + "quantile": {}, + } + self.rho_tau_fits[version]["flat_samples"][f"tomo_bin_{tomo_bin_id}"] = ( + flat_samples + ) + self.rho_tau_fits[version]["result"][f"tomo_bin_{tomo_bin_id}"] = result + self.rho_tau_fits[version]["quantile"][f"tomo_bin_{tomo_bin_id}"] = q + + return flat_samples + + def get_xi_psf_sys_samples(self, ver, params, tomo_bin_a, tomo_bin_b): + base_rho = self.basename(ver) + self.psf_fitter.load_rho_stat(f"rho_stats_{base_rho}.fits") + nbins = self.psf_fitter.rho_stat_handler._treecorr_config["nbins"] + + # Get the samples for the given tomographic bins + flat_samples_a = self.get_samples(ver, params, tomo_bin_a, track_result=False) + flat_samples_b = self.get_samples(ver, params, tomo_bin_b, track_result=False) + + xi_psf_sys_samples_plus = np.array( + [ + self.psf_fitter.compute_xi_psf_sys(sample_a, sample_b, p_or_m="p") + for (sample_a, sample_b) in zip(flat_samples_a, flat_samples_b) + ] + ).reshape(-1, nbins) + + xi_psf_sys_samples_minus = np.array( + [ + self.psf_fitter.compute_xi_psf_sys(sample_a, sample_b, p_or_m="m") + for (sample_a, sample_b) in zip(flat_samples_a, flat_samples_b) + ] + ).reshape(-1, nbins) + + return xi_psf_sys_samples_plus, xi_psf_sys_samples_minus + + def _load_alpha_leakage(self, ver, tomo_bin_id, cov_type=None, seed=0): + """Alpha(theta) of one version and tomographic bin from its rho/tau files. + + With ``cov_type`` None the errors use the variances in the rho/tau + files; otherwise the ``cov_tau_*_.npy`` and jackknife + ``cov_rho_*_jk.npy`` covariances under ``rho_tau_stats/``. + """ + base_rho = self.basename(ver) + base_tau = self.basename(ver, tomo_bin_a=tomo_bin_id) + self.rho_stat_handler.load_rho_stats(f"rho_stats_{base_rho}.fits") + self.tau_stat_handler.load_tau_stats(f"tau_stats_{base_tau}.fits") + + if cov_type is not None: + out_dir = Path(self.cc["paths"]["output"]) / "rho_tau_stats" + cov_tau = np.load(out_dir / f"cov_tau_{base_tau}_{cov_type}.npy") + cov_rho = np.load(out_dir / f"cov_rho_{base_rho}_jk.npy") + else: + cov_tau = None + cov_rho = None + + return self._get_alpha_leakage( + self.rho_stat_handler, + self.tau_stat_handler, + cov_rho, + cov_tau, + seed=seed, + ) - def plot_scale_dependent_leakage(self): - if not hasattr(self.results[self.versions[0]], "r_corr_gp"): - self.calculate_scale_dependent_leakage() + def _get_alpha_leakage( + self, + rho_stat_handler, + tau_stat_handler, + cov_rho=None, + cov_tau=None, + n_samples=10_000, + seed=0, + ): + """ + Compute the alpha leakage parameter from the rho and tau statistics. + + alpha(theta) = tau_0,+(theta) / rho_0,+(theta); its error is the standard + deviation of that ratio over Gaussian draws of rho_0,+ and tau_0,+. + + Parameters + ---------- + rho_stat_handler : RhoStatHandler + The handler for the rho statistics. + tau_stat_handler : TauStatHandler + The handler for the tau statistics. + cov_rho : np.ndarray, optional + The covariance matrix for the rho statistics; its leading block is + the rho_0,+ covariance. If None, the variances in the rho-statistics + table are used. + cov_tau : np.ndarray, optional + The covariance matrix for the tau statistics; its leading block is + the tau_0,+ covariance. If None, the variances in the tau-statistics + table are used. + n_samples : int, optional + Number of Gaussian draws for the error. + seed : int or np.random.Generator, optional + Seed (or generator) for the draws, so that errors are reproducible. + + Returns + ------- + theta, alpha, alpha_err : np.ndarray + Angular scales, alpha leakage and its error. + """ + if cov_rho is None: + cov_rho = np.diag(rho_stat_handler.rho_stats["varrho_0_p"]) + if cov_tau is None: + cov_tau = np.diag(tau_stat_handler.tau_stats["vartau_0_p"]) + + theta = rho_stat_handler.rho_stats["theta"] + n_bins = len(theta) + alpha = ( + tau_stat_handler.tau_stats["tau_0_p"] + / rho_stat_handler.rho_stats["rho_0_p"] + ) - theta = [] - y = [] - yerr = [] - labels = [] - colors = [] - linestyles = [] - markers = [] + rng = np.random.default_rng(seed) + rho_samples = rng.multivariate_normal( + mean=rho_stat_handler.rho_stats["rho_0_p"], + cov=cov_rho[:n_bins, :n_bins], + size=n_samples, + ) + tau_samples = rng.multivariate_normal( + mean=tau_stat_handler.tau_stats["tau_0_p"], + cov=cov_tau[:n_bins, :n_bins], + size=n_samples, + ) - for ver in self.versions: - if hasattr(self.results[ver], "r_corr_gp"): - theta.append(self.results[ver].r_corr_gp.meanr) - y.append(self.results[ver].alpha_leak) - yerr.append(self.results[ver].sig_alpha_leak) - labels.append(ver) - colors.append(self.cc[ver]["colour"]) - linestyles.append(self.cc[ver]["ls"]) - markers.append(self.cc[ver]["marker"]) - - if len(theta) > 0: - # Log x - out_path = self._output_path("alpha_leak_log.png") - - title = r"$\alpha$ leakage" - xlabel = r"$\theta$ [arcmin]" - ylabel = r"$\alpha(\theta)$" - cs_plots.plot_data_1d( - theta, - y, - yerr, - title, - xlabel, - ylabel, - out_path=None, - xlog=True, - xlim=[self.theta_min_plot, self.theta_max_plot], - ylim=self.ylim_alpha, - labels=labels, - colors=colors, - linestyles=linestyles, - shift_x=True, + alpha_samples = tau_samples / rho_samples + alpha_err = np.std(alpha_samples, axis=0) + + return theta, alpha, alpha_err + + @staticmethod + def _alpha_summaries(theta, alpha, alpha_err): + """Scalar summaries of alpha(theta), as ufloats. + + Returns + ------- + dict + ``alpha_mean``: inverse-variance weighted mean of alpha(theta), with + the weighted standard deviation; ``alpha_1``: alpha and its error at + the smallest theta; ``alpha_0``: intercept c of the weighted + least-squares affine fit alpha(theta) = c + m theta, with its error + from the fit covariance (errors taken as absolute). + """ + theta = np.asarray(theta, dtype=float) + alpha = np.asarray(alpha, dtype=float) + alpha_err = np.asarray(alpha_err, dtype=float) + + weights = 1 / alpha_err**2 + mean = np.average(alpha, weights=weights) + std = np.sqrt(np.average((alpha - mean) ** 2, weights=weights)) + + i_min = np.argmin(theta) + + (_, c), cov = np.polyfit(theta, alpha, 1, w=1 / alpha_err, cov="unscaled") + + return { + "alpha_mean": ufloat(mean, std), + "alpha_1": ufloat(alpha[i_min], alpha_err[i_min]), + "alpha_0": ufloat(c, np.sqrt(cov[1, 1])), + } + + def _compute_scale_dependent_xi_psf_sys( + self, + rho_0, + tau_0_a, + tau_0_b, + cov_rho, + cov_tau_a, + cov_tau_b, + n_samples=10_000, + same_bin=False, + seed=0, + ): + """ + Compute the scale-dependent xi_psf_sys from the rho and tau statistics. + + xi_psf_sys = tau_0_a * tau_0_b / rho_0; its error is the standard + deviation over Gaussian draws of rho_0, tau_0_a and tau_0_b. + + Parameters + ---------- + rho_0 : np.ndarray + The rho_0 statistics. + tau_0_a : np.ndarray + The tau_0 statistics for tomographic bin a. + tau_0_b : np.ndarray + The tau_0 statistics for tomographic bin b. + cov_rho : np.ndarray + The covariance matrix for the rho statistics. + cov_tau_a : np.ndarray + The covariance matrix for the tau statistics for tomographic bin a. + cov_tau_b : np.ndarray + The covariance matrix for the tau statistics for tomographic bin b. + n_samples : int, optional + Number of Gaussian draws for the error. + same_bin : bool, optional + True when a and b are the same bin: tau_0_a and tau_0_b are then one + measurement, and each draw uses a single tau sample (tau^2 / rho). + seed : int or np.random.Generator, optional + Seed (or generator) for the draws, so that errors are reproducible. + """ + xi_psf_sys = (tau_0_a * tau_0_b) / rho_0 + + rng = np.random.default_rng(seed) + rho_samples = rng.multivariate_normal(mean=rho_0, cov=cov_rho, size=n_samples) + tau_samples_a = rng.multivariate_normal( + mean=tau_0_a, cov=cov_tau_a, size=n_samples + ) + if same_bin: + tau_samples_b = tau_samples_a + else: + tau_samples_b = rng.multivariate_normal( + mean=tau_0_b, cov=cov_tau_b, size=n_samples ) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"Log-scale alpha leakage plot saved to {out_path}") - - # Lin x - out_path = self._output_path("alpha_leak_lin.png") - - title = r"$\alpha$ leakage" - xlabel = r"$\theta$ [arcmin]" - ylabel = r"$\alpha(\theta)$" - cs_plots.plot_data_1d( - theta, - y, - yerr, - title, - xlabel, - ylabel, - out_path=None, - xlog=False, - xlim=[-10, self.theta_max_plot], - ylim=self.ylim_alpha, - labels=labels, - colors=colors, - linestyles=linestyles, - shift_x=False, + xi_psf_sys_samples = (tau_samples_a * tau_samples_b) / rho_samples + xi_psf_sys_err = np.std(xi_psf_sys_samples, axis=0) + + return xi_psf_sys, xi_psf_sys_err + + # --- plotting functions --- + def plot_rho_stats( + self, + versions=None, + colors=None, + abs=False, + offset=0, + savefig=None, + show=True, + close=True, + ): + """ + Plot the Rho statistics. + + Parameters + ---------- + versions : list, optional + List of versions to plot. If None, all versions are plotted. + abs : bool, optional + If True, plot the absolute values of the Rho statistics. + offset : float, optional + Offset to apply to the versions for better visualisation. + savefig : str, optional + If provided, save the figure to this file. + show : bool, optional + If True, show the figure. + close : bool, optional + If True, close the figure after saving or showing. + """ + if versions is None: + versions = self.versions + + filenames = [f"rho_stats_{self.basename(ver)}.fits" for ver in versions] + + if colors is None: + colors = [self.cc[ver]["colour"] for ver in versions] + + if len(colors) != len(versions): + raise ValueError("Colors and versions must have the same length.") + + self.rho_stat_handler.plot_rho_stats( + filenames, + colors, + versions, + offset=offset, + savefig=savefig, + legend="outside", + abs=abs, + show=show, + close=close, + ) + + if savefig is not None: + self.print_done( + "Rho stats plot saved to " + + f"{os.path.abspath(self.rho_stat_handler.catalogs._output)}/{savefig}", ) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"Lin-scale alpha leakage plot saved to {out_path}") - # Plot xi_sys - y = [] - yerr = [] - colors = [] - linestyles = [] + def plot_tau_stats( + self, + tomography=False, + cov_type=None, + versions=None, + colors=None, + offset=0, + savefig=None, + show=True, + close=True, + plot_tau_m=False, + plot_theta_times_tau=False, + fmt="", + capsize=2, + ): + if versions is None: + versions = self.versions + + if colors is None: + colors = [self.cc[ver]["colour"] for ver in versions] - for ver in self.versions: - if hasattr(self.results[ver], "C_sys_p"): - y.append(self.results[ver].C_sys_p) - yerr.append(self.results[ver].C_sys_std_p) - labels.append(ver) - colors.append(self.cc[ver]["colour"]) - linestyles.append(self.cc[ver]["ls"]) - - if len(y) > 0: - xlabel = r"$\theta$ [arcmin]" - ylabel = r"$\xi^{\rm sys}_+(\theta)$" - title = "Cross-correlation leakage" - out_path = self._output_path("xi_sys_p.png") - cs_plots.plot_data_1d( - theta, - y, - yerr, - title, - xlabel, - ylabel, - out_path=None, - labels=labels, - xlog=True, - xlim=[self.theta_min_plot, self.theta_max_plot], - colors=colors, - linestyles=linestyles, - # shift_x=True, + if len(colors) != len(versions): + raise ValueError("Colors and versions must have the same length.") + + if cov_type is None: + self.print_cyan("Using the error bars from the tau-statistics files") + else: + self.print_cyan( + f"Using the error bars from the covariance files of type: {cov_type}" ) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"xi_sys_plus plot saved to {out_path}") - y = [] - yerr = [] - for ver in self.versions: - if hasattr(self.results[ver], "C_sys_m"): - y.append(self.results[ver].C_sys_m) - yerr.append(self.results[ver].C_sys_std_m) - - if len(y) > 0: - xlabel = r"$\theta$ [arcmin]" - ylabel = r"$\xi^{\rm sys}_-(\theta)$" - title = "Cross-correlation leakage" - out_path = self._output_path("xi_sys_m.png") - cs_plots.plot_data_1d( - theta, - y, - yerr, - title, - xlabel, - ylabel, - out_path=None, - labels=labels, - xlog=True, - xlim=[self.theta_min_plot, self.theta_max_plot], - ylim=[-1e-7, 1e-6], - colors=colors, - linestyles=linestyles, - # shift_x=True, + out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" + + if tomography: + # Write the whole script for the tomography. It does not exist in shear_psf_leakage + e_obs = r"e^\mathrm{obs}" + e_psf = r"e^\mathrm{PSF}" + delta_e_psf = r"\delta e^\mathrm{PSF}" + delta_T_psf = r"\delta T^\mathrm{PSF}" + + factor_theta_label = r"\theta" if plot_theta_times_tau else r"" + + titles = [ + rf"$\tau_0 = \langle {e_obs} {e_psf} \rangle$", + rf"${factor_theta_label} \tau_2 = {factor_theta_label} \langle {e_obs} {delta_e_psf} \rangle$", + rf"${factor_theta_label} \tau_5 = {factor_theta_label} \langle {e_obs} {delta_T_psf} \rangle$", + ] + + dict_index_tau = { + 0: "0", + 1: "2", + 2: "5", + } + + # From all the versions, get the maximum number of tomo_bin_ids + tomo_bins = self._get_tomo_bins_for_versions( + versions, tomography=tomography ) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"xi_sys_minus plot saved to {out_path}") - def calculate_objectwise_leakage(self): - if not hasattr(self.results[self.versions[0]], "alpha_leak_mean"): - self.calculate_scale_dependent_leakage() + n_tomo_bins_plot = max(len(bins["ids"]) for bins in tomo_bins.values()) + + n_rows = n_tomo_bins_plot * (1 + plot_tau_m) + + fig = plt.figure(figsize=(20 * (1 + plot_tau_m), 10 * (1 + plot_tau_m))) + gs = fig.add_gridspec(n_rows, 3, wspace=0.1, hspace=0) + all_axs = gs.subplots(sharex="col") + + for k in range(n_rows): + axs = all_axs[k] + + tomo_bin_id = k // 2 + 1 if plot_tau_m else k + 1 + is_m_component = k % 2 if plot_tau_m else 0 + + for file_idx, (ver, color) in enumerate(zip(versions, colors)): + # Check if the tomo bin is valid for this version + if tomo_bin_id not in tomo_bins[ver]["ids"]: + continue + + # Load the tau-stats in the tau_stat_handler for easier read + base_tau = self.basename(ver, tomo_bin_a=tomo_bin_id) + + self.tau_stat_handler.load_tau_stats(f"tau_stats_{base_tau}.fits") + + if cov_type is not None: + cov_tau_path = ( + Path(out_dir) / f"cov_tau_{base_tau}_{cov_type}.npy" + ) + cov_tau = np.load(cov_tau_path) + + # Plot the different tau-stats per row + for i in range(3): + p_or_m = "m" if is_m_component else "p" + p_or_m_label = "-" if is_m_component else "+" + + # Get the jittered angular scale for the x-axis + theta = self.tau_stat_handler.tau_stats["theta"] + num_theta_bins = theta.shape[0] + + jittered_theta = self._get_jittered_theta( + theta, file_idx, len(versions), offset + ) + + factor_theta = ( + np.ones_like(jittered_theta) + if (i == 0) or not plot_theta_times_tau + else theta + ) + + y_plot = ( + self.tau_stat_handler.tau_stats[ + f"tau_{dict_index_tau[i]}_{p_or_m}" + ] + * factor_theta + ) + + if cov_type is None or p_or_m == "m": + cov_diag = self.tau_stat_handler.tau_stats[ + "vartau_" + dict_index_tau[i] + "_" + p_or_m + ] + else: + cov_diag = np.diag( + cov_tau[ + i * num_theta_bins : (i + 1) * num_theta_bins, + i * num_theta_bins : (i + 1) * num_theta_bins, + ] + ) + + yerr_plot = np.sqrt(cov_diag) * factor_theta + + ver_label = ( + self.cc[ver]["label"] if "label" in self.cc[ver] else ver + ) + axs[i].errorbar( + jittered_theta, + y_plot, + yerr=yerr_plot, + fmt=fmt, + label=ver_label, + capsize=capsize, + color=color, + ) + + # Set the style of the plot + for i in range(3): + axs[i].set_xscale("log") + axs[i].set_xlim(theta.min() * 0.9, theta.max() * 1.1) + if i == 0: + axs[i].set_ylabel(f"Bin {tomo_bin_id}\n `{p_or_m_label}' comp.") + if k == n_rows - 1: + axs[i].set_xlabel(r"$\theta$ [arcmin]") + if k == 0: + axs[i].set_title(titles[i]) + + # --- Force scientific notation and scaling --- + axs[i].ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) + axs[i].yaxis.offsetText.set_visible(True) + + # --- Make it look clean --- + axs[i].yaxis.set_major_locator( + mticker.MaxNLocator(nbins=5) + ) # Fewer, rounded ticks + # axs[i].yaxis.get_offset_text().set_fontsize(10) # Smaller ×10⁻⁴ label + axs[i].yaxis.get_offset_text().set_position( + (0, 1.02) + ) # Move scale factor slightly above axis + axs[i].yaxis.get_offset_text().set_ha("left") + + if k == n_rows - 1: + axs[1].legend( + loc="upper center", + bbox_to_anchor=(0.5, -0.5), # (x, y) relative to the axes + ncol=3, # number of columns + frameon=False, + ) - self.print_start("Object-wise leakage:") - mix = True - order = "lin" - for ver in self.versions: - self.print_magenta(ver) + plt.tight_layout() - results_obj = self.results_objectwise[ver] - results_obj.check_params() - results_obj.update_params() - results_obj.prepare_output() + if savefig is not None: + plt.savefig(savefig, dpi=300, bbox_inches="tight") + + if show: + plt.show() + + if close: + plt.close() + + else: + filenames = [f"tau_stats_{self.basename(ver)}.fits" for ver in versions] + cov_paths = ( + None + if cov_type is None + else [ + f"cov_tau_{self.basename(ver)}_{cov_type}.npy" for ver in versions + ] + ) + self.tau_stat_handler.plot_tau_stats( + filenames, + colors, + versions, + cov_paths=cov_paths, + offset=offset, + savefig=savefig, + legend="outside", + plot_tau_m=plot_tau_m, + plot_theta_times_tau=plot_theta_times_tau, + show=show, + close=close, + fmt=fmt, + capsize=capsize, + ) + + if savefig is not None: + self.print_done( + "Tau stats plot saved to " + + f"{os.path.abspath(self.tau_stat_handler.catalogs._output)}/{savefig}", + ) + + def plot_rho_tau_fits( + self, + tomography=False, + versions=None, + colors=None, + savefig_contours=None, + savefig_xi_psf_sys=None, + savefig_format="png", + tomo_bin_label_position=None, + nsamples_to_plot=100, + offset=0, + alpha=0.3, + times_theta=False, + show=True, + close=True, + ): + """ + Plot the Rho/Tau fits and the xi_psf_sys samples. + + Parameters + ---------- + versions : list, optional + List of versions to plot. If None, all versions are plotted. + colors : list, optional + List of colors for each version. If None, default colors are used. + savefig_contours : str, optional + If provided, save the contour plot to this file. + savefig_xi_psf_sys : str, optional + If provided, save the xi_psf_sys samples plot to this file. + nsamples_to_plot : int, optional + Number of xi_psf_sys samples to plot. Default is 100. + offset : float, optional + Offset to apply to the versions for better visualisation. + alpha : float, optional + Alpha value for the xi_psf_sys samples plot. Default is 0.3. + times_theta : bool, optional + If True, plot the xi_psf_sys multiplied by theta. + show : bool, optional + If True, show the figure. + close : bool, optional + If True, close the figure after saving or showing. + """ + if savefig_format not in ["png", "pdf", "jpg", "jpeg", "svg"]: + raise ValueError( + "Invalid savefig_format. Must be one of: 'png', 'pdf', 'jpg', 'jpeg', 'svg'." + ) + + out_dir = self.rho_stat_handler.catalogs._output + + versions = versions if versions is not None else self.versions + colors = ( + colors + if colors is not None + else [self.cc[ver]["colour"] for ver in versions] + ) + + # Print the tau-statistics constraints on the PSF error model parameters. + savefig = ( + f"{out_dir}/{savefig_contours}" if savefig_contours is not None else None + ) + self.print_cyan( + "Only plot contours for the non-tomographic case. For tomography, the contours are not plotted." + ) + sample_list = [ + self.rho_tau_fits[ver]["flat_samples"]["tomo_bin_all"] for ver in versions + ] + psfleak_plots.plot_contours( + sample_list, + names=["x0", "x1", "x2"], + labels=[r"\alpha", r"\beta", r"\eta"], + savefig=savefig, + legend_labels=versions, + legend_loc="upper right", + contour_colors=colors, + markers={"x0": 0, "x1": 1, "x2": 1}, + show=show, + close=close, + ) + if savefig_contours is not None: + self.print_done(f"Tau contours plot saved to {os.path.abspath(savefig)}") + + # Plot the xi_psf_sys samples and quantiles. + if tomography: + self.print_cyan("Plot the xi_psf_sys for the tomographic case.") + else: + self.print_cyan("Plot the xi_psf_sys for the non-tomographic case.") + + x_label = r"$\theta$ [arcmin]" + y_label_plus = ( + r"$\theta$" if times_theta else "" + ) + r"$\xi^{\rm PSF, sys}_+(\theta)$" + y_label_minus = ( + r"$\theta$" if times_theta else "" + ) + r"$\xi^{\rm PSF, sys}_-(\theta)$" + + if tomo_bin_label_position is None: + tomo_bin_label_position = (0.05, 0.9) if times_theta else (0.8, 0.95) + + y_scale = "linear" if times_theta else "log" + + out_path = ( + f"{out_dir}/{savefig_xi_psf_sys}_tomography_{tomography}_sample.{savefig_format}" + if savefig_xi_psf_sys is not None + else None + ) + + kwargs_x_y_plot_function = { + "nsamples_to_plot": nsamples_to_plot, + "alpha": alpha, + "offset": offset, + "times_theta": times_theta, + } + + self.plot_2pcf_tomography( + self._xi_psf_sys_sample_x_y_plot_function, + x_label, + y_label_plus, + y_label_minus, + tomo_bin_label_position, + extract_text_offset=times_theta, + add_index_version_to_kwargs=True, + x_scale="log", + y_scale=y_scale, + tomography=tomography, + versions=versions, + colors=colors, + savefig=out_path, + show=show, + close=close, + **kwargs_x_y_plot_function, + ) + + # Plot the xi_psf_sys mean and std. + x_label = r"$\theta$ [arcmin]" + y_label_plus = ( + r"$\theta$" if times_theta else "" + ) + r"$\xi^{\rm PSF, sys}_+(\theta)$" + y_label_minus = ( + r"$\theta$" if times_theta else "" + ) + r"$\xi^{\rm PSF, sys}_-(\theta)$" + + if tomo_bin_label_position is None: + tomo_bin_label_position = (0.05, 0.9) if times_theta else (0.8, 0.95) + + y_scale = "linear" if times_theta else "log" + + out_path = ( + f"{out_dir}/{savefig_xi_psf_sys}_tomography_{tomography}_mean_and_std.{savefig_format}" + if savefig_xi_psf_sys is not None + else None + ) + + kwargs_x_y_plot_function = { + "offset": offset, + "times_theta": times_theta, + "alpha": alpha, + } + + self.plot_2pcf_tomography( + self._xi_psf_sys_mean_and_std_x_y_plot_function, + x_label, + y_label_plus, + y_label_minus, + tomo_bin_label_position, + extract_text_offset=times_theta, + add_index_version_to_kwargs=True, + x_scale="log", + y_scale=y_scale, + tomography=tomography, + versions=versions, + colors=colors, + savefig=out_path, + show=show, + close=close, + **kwargs_x_y_plot_function, + ) + + def plot_scale_dependent_leakage( + self, + tomography=False, + cov_type=None, + versions=None, + colors=None, + offset=0, + savefig=None, + show=True, + close=True, + plot_theta_times_tau=False, + ylim_alpha=False, + fmt="", + capsize=2, + ): + # First plot alpha leakage + self.plot_scale_dependent_alpha( + tomography=tomography, + cov_type=cov_type, + versions=versions, + colors=colors, + offset=offset, + savefig=savefig, + show=show, + close=close, + fmt=fmt, + capsize=capsize, + ylim_alpha=ylim_alpha, + ) + + # Second plot xi_sys + self.plot_2pcf_tomography( + self._scale_dependent_xi_psf_sys_x_y_plot_function, + x_label=r"$\theta$ [arcmin]", + y_label_plus=(r"$\theta$" if plot_theta_times_tau else "") + + r"$\xi^{\rm PSF, sys}_+(\theta)$", + y_label_minus=(r"$\theta$" if plot_theta_times_tau else "") + + r"$\xi^{\rm PSF, sys}_-(\theta)$", + tomo_bin_label_position=(0.05, 0.9) + if not plot_theta_times_tau + else (0.8, 0.95), + extract_text_offset=plot_theta_times_tau, + add_index_version_to_kwargs=True, + x_scale="log", + y_scale="linear" if plot_theta_times_tau else "log", + tomography=tomography, + versions=versions, + colors=colors, + savefig=savefig.replace(".png", "_xi_psf_sys.png") + if savefig is not None + else None, + show=show, + close=close, + offset=offset, + cov_type=cov_type, + times_theta=plot_theta_times_tau, + fmt=fmt, + capsize=capsize, + ) + + def plot_scale_dependent_alpha( + self, + tomography=False, + cov_type=None, + versions=None, + colors=None, + offset=0, + savefig=None, + show=True, + close=True, + fmt="", + capsize=2, + ylim_alpha=False, + ): + if versions is None: + versions = self.versions + + if colors is None: + colors = [self.cc[ver]["colour"] for ver in versions] - # Skip read_data() and copy catalogue from scale leakage instance instead - # results_obj._dat = self.results[ver].dat_shear + if len(colors) != len(versions): + raise ValueError("Colors and versions must have the same length.") + + if cov_type is None: + self.print_cyan("Using the error bars from the tau-statistics files") + else: + self.print_cyan( + f"Using the error bars from the covariance files of type: {cov_type}" + ) - out_base = results_obj.get_out_base(mix, order) - out_path = f"{out_base}.pkl" - if os.path.exists(out_path): - self.print_green( - f"Skipping object-wise leakage, file {out_path} exists" + tomo_bins = self._get_tomo_bins_for_versions(versions, tomography=tomography) + + n_tomo_bins_plot = max(len(bins["ids"]) for bins in tomo_bins.values()) + + fig, axs = plt.subplots( + n_tomo_bins_plot, 1, figsize=(8, 3 * n_tomo_bins_plot), sharex=True + ) + + # Iterate upon each version + for ver, color in zip(versions, colors): + label = self.cc[ver]["label"] if "label" in self.cc[ver] else ver + # Iterate upon each tomographic bin + for tomo_bin_id in tomo_bins[ver]["ids"]: + theta, alpha, alpha_err = self._load_alpha_leakage( + ver, tomo_bin_id, cov_type ) - results_obj.par_best_fit = leakage.read_from_file(out_path) - else: - self.print_cyan("Computing object-wise leakage regression") - - # Run - with results_obj.temporarily_read_data(): - try: - results_obj.PSF_leakage() - except KeyError as e: - print(f"{e}\nExpected key is missing from catalog.") - # remove the results object for this version - self.results_objectwise.pop(ver) - - # Gather coefficients - leakage_coeff = {} - for ver in self.results_objectwise: - results = self.results[ver] - par_best_fit = self.results_objectwise[ver].par_best_fit - - # Object-wise leakage - a11 = ufloat(par_best_fit["a11"].value, par_best_fit["a11"].stderr) - a22 = ufloat(par_best_fit["a22"].value, par_best_fit["a22"].stderr) - leakage_coeff[ver] = { - "a11": a11, - "a22": a22, - "aii_mean": 0.5 * (a11 + a22), - # Scale-dependent leakage: mean - "alpha_mean": ufloat(results.alpha_leak_mean, results.alpha_leak_std), - # Scale-dependent leakage: value at smallest scale - "alpha_1": ufloat(results.alpha_leak[0], results.sig_alpha_leak[0]), - # Scale-dependent leakage: value extrapolated to 0 using affine model - "alpha_0": ufloat( - results.alpha_affine_best_fit["c"].value, - results.alpha_affine_best_fit["c"].stderr, - ), - } - self.leakage_coeff = leakage_coeff + jittered_theta = self._get_jittered_theta( + theta, versions.index(ver), len(versions), offset + ) - def plot_objectwise_leakage(self): - if not hasattr(self, "leakage_coeff"): - self.calculate_objectwise_leakage() + if tomo_bin_id == "all": + ax = axs + else: + ax = axs[tomo_bin_id - 1] + + ax.errorbar( + jittered_theta, + alpha, + yerr=alpha_err, + fmt=fmt, + label=f"{label}", + capsize=capsize, + color=color, + ) + + if tomography: + for i, ax in enumerate(axs): + ax.set_xscale("log") + ax.set_xlim(self.theta_min_plot, self.theta_max_plot) + if ylim_alpha: + ax.set_ylim(self.ylim_alpha) + ax.set_ylabel(rf"$\alpha_{i + 1}(\theta)$") + if i == len(axs) - 1: + ax.set_xlabel(r"$\theta$ [arcmin]") + ax.text(0.05, 0.9, f"Tomo bin {i + 1}", transform=ax.transAxes) + else: + axs.set_xscale("log") + axs.set_xlim(self.theta_min_plot, self.theta_max_plot) + if ylim_alpha: + axs.set_ylim(self.ylim_alpha) + axs.set_ylabel(r"$\alpha_{\rm all}(\theta)$") + axs.set_xlabel(r"$\theta$ [arcmin]") + axs.text(0.05, 0.9, "All tomographic bins", transform=axs.transAxes) + + if tomography: + handles, labels = axs[0].get_legend_handles_labels() + else: + handles, labels = axs.get_legend_handles_labels() + + fig.legend( + handles, + labels, + loc="upper center", + bbox_to_anchor=(0.5, -0.02), + ncol=3, + frameon=False, + ) + + if savefig is not None: + plt.savefig(savefig, dpi=300, bbox_inches="tight") + self.print_done(f"Scale-dependent alpha leakage plot saved to {savefig}") + + if show: + plt.show() + + if close: + plt.close() + + def plot_objectwise_leakage(self, tomography=False, cov_type=None): + """Plot object-wise leakage against the scale-dependent alpha. + + x is the object-wise , y the alpha(theta) summaries from the + rho/tau products (fill style: mean, smallest scale, affine intercept). + Marker shape is the version; colour is the version (non-tomographic) or + the tomographic bin. Saved to ``leakage_coefficients.png``, or + ``leakage_coefficients_tomo.png`` when ``tomography`` is True. + """ + tomo_bins = self._get_tomo_bins_for_versions( + self.versions, tomography=tomography + ) + existing = getattr(self, "leakage_coeff", {}) + if any( + ver in self.results_objectwise + and any(f"tomo_bin_{i}" not in existing.get(ver, {}) for i in bins["ids"]) + for ver, bins in tomo_bins.items() + ): + self.calculate_objectwise_leakage(tomography=tomography) + self.calculate_alpha_leakage_summaries(tomography=tomography, cov_type=cov_type) self.print_start("Plotting object-wise leakage:") cs_plots.figure(figsize=(15, 15)) - linestyles = ["-", "--", ":"] - fillstyles = ["full", "none", "left", "right", "bottom", "top"] - - for ver in self.results_objectwise: - label = ver - for key, ls, fs in zip( - ["alpha_mean", "alpha_1", "alpha_0"], linestyles, fillstyles - ): - x = self.leakage_coeff[ver]["aii_mean"].nominal_value - dx = self.leakage_coeff[ver]["aii_mean"].std_dev - y = self.leakage_coeff[ver][key].nominal_value - dy = self.leakage_coeff[ver][key].std_dev - - eb = plt.errorbar( - x, - y, - xerr=dx, - yerr=dy, - fmt=self.cc[ver]["marker"], - color=self.cc[ver]["colour"], - fillstyle=fs, - label=label, + summaries = { + "alpha_mean": (r"$\bar\alpha$", "-", "full"), + "alpha_1": (r"$\alpha(\theta_{\rm min})$", "--", "none"), + "alpha_0": (r"$\alpha(0)$", ":", "left"), + } + max_bins = max(len(bins["ids"]) for bins in tomo_bins.values()) + bin_colours = plt.get_cmap("viridis")(np.linspace(0, 0.9, max_bins)) + + handles = [] + for ver in self.leakage_coeff: + marker = self.cc[ver]["marker"] + ver_colour = "k" if tomography else self.cc[ver]["colour"] + for i_bin, tomo_bin_id in enumerate(tomo_bins[ver]["ids"]): + coeff = self.leakage_coeff[ver][f"tomo_bin_{tomo_bin_id}"] + colour = bin_colours[i_bin] if tomography else ver_colour + for key, (_, ls, fs) in summaries.items(): + eb = plt.errorbar( + coeff["aii_mean"].nominal_value, + coeff[key].nominal_value, + xerr=coeff["aii_mean"].std_dev, + yerr=coeff[key].std_dev, + fmt=marker, + color=colour, + fillstyle=fs, + ) + eb[-1][0].set_linestyle(ls) + handles.append( + plt.Line2D( + [], + [], + marker=marker, + ls="", + color=ver_colour, + label=self.cc[ver].get("label", ver), ) - label = None - eb[-1][0].set_linestyle(ls) + ) + + if tomography: + handles += [ + plt.Line2D([], [], marker="s", ls="", color=c, label=f"bin {i + 1}") + for i, c in enumerate(bin_colours) + ] + handles += [ + plt.Line2D([], [], marker="o", ls=ls, color="grey", fillstyle=fs, label=lab) + for lab, ls, fs in summaries.values() + ] # y=x line xlim = 0.02 x = [-xlim, xlim] - y = x - plt.plot(x, y, "k:", linewidth=0.5) + plt.plot(x, x, "k:", linewidth=0.5) - plt.legend() - plt.xlabel(r"tr $a$ (object-wise)") - plt.ylabel(r"$\alpha$ (scale-dependent)") - out_path = self._output_path("leakage_coefficients.png") + plt.legend(handles=handles) + plt.xlabel(r"$\langle a_{ii} \rangle$ (object-wise)") + plt.ylabel(r"$\alpha$ (scale-dependent, $\tau_0 / \rho_0$)") + out_path = self._output_path( + f"leakage_coefficients{'_tomo' if tomography else ''}.png" + ) cs_plots.savefig(out_path, close_fig=False) cs_plots.show() self.print_done(f"Object-wise leakage coefficients plot saved to {out_path}") + + # --- utlility functions for plotting --- + + def _xi_psf_sys_sample_x_y_plot_function( + self, + ax_plus, + ax_minus, + version, + tomo_bin_a, + tomo_bin_b, + idx, + versions, + color, + offset, + nsamples_to_plot, + times_theta, + alpha, + ): + # Load the rho-stats to compute the xi_psf_sys samples + base_rho = self.basename(version) + self.psf_fitter.load_rho_stat(f"rho_stats_{base_rho}.fits") + + # Get the angular scales for the xi_psf_sys plots + theta = self.psf_fitter.rho_stat_handler.rho_stats["theta"] + + # Add the offset to the theta values for better visualisation + jittered_theta = self._get_jittered_theta(theta, idx, len(versions), offset) + + # Get the parameters for the rho-tau fit + params = self.set_params_rho_tau( + version, self.results[version]._params, self.cc[version]["psf"] + ) + + # Get the xi_psf_sys samples + xi_psf_sys_samples_plus, xi_psf_sys_samples_minus = self.get_xi_psf_sys_samples( + version, params, tomo_bin_a, tomo_bin_b + ) + + y_plus = xi_psf_sys_samples_plus[-nsamples_to_plot:] * ( + theta if times_theta else 1 + ) + y_minus = xi_psf_sys_samples_minus[-nsamples_to_plot:] * ( + theta if times_theta else 1 + ) + + ax_plus.plot( + jittered_theta, + y_plus.T, + color=color, + alpha=alpha, + ) + + ax_minus.plot(jittered_theta, y_minus.T, color=color, alpha=alpha) + + def _xi_psf_sys_mean_and_std_x_y_plot_function( + self, + ax_plus, + ax_minus, + version, + tomo_bin_a, + tomo_bin_b, + idx, + versions, + color, + offset, + times_theta, + alpha, + ): + # Load the rho-stats to compute the xi_psf_sys mean and std. + base_rho = self.basename(version) + self.psf_fitter.load_rho_stat(f"rho_stats_{base_rho}.fits") + + # Get the angular scales for the xi_psf_sys plots + theta = self.psf_fitter.rho_stat_handler.rho_stats["theta"] + + # Add the offset to the theta values for better visualisation + jittered_theta = self._get_jittered_theta(theta, idx, len(versions), offset) + + xi_psf_sys = self.xi_psf_sys[version][ + f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}" + ] + + def plot_axis(ax, ax_type): + """Plot depending on the axis type ('plus' or 'minus').""" + ax.plot( + jittered_theta, + xi_psf_sys[f"mean_{ax_type}"] * (theta if times_theta else 1), + color=color, + ) + ax.plot( + jittered_theta, + xi_psf_sys[f"quantiles_{ax_type}"][0] * (theta if times_theta else 1), + color=color, + ) + ax.plot( + jittered_theta, + xi_psf_sys[f"quantiles_{ax_type}"][1] * (theta if times_theta else 1), + color=color, + ) + ax.fill_between( + jittered_theta, + xi_psf_sys[f"quantiles_{ax_type}"][0] * (theta if times_theta else 1), + xi_psf_sys[f"quantiles_{ax_type}"][1] * (theta if times_theta else 1), + color=color, + alpha=alpha, + ) + + # Plot the plus axis + plot_axis(ax_plus, "plus") + + # Plot the minus axis + plot_axis(ax_minus, "minus") + + def _scale_dependent_xi_psf_sys_x_y_plot_function( + self, + ax_plus, + ax_minus, + version, + tomo_bin_a, + tomo_bin_b, + idx, + versions, + color, + offset, + cov_type, + times_theta, + fmt, + capsize, + ): + # Load the rho-stats and the tau-stats + base_rho = self.basename(version) + base_tau_a = self.basename(version, tomo_bin_a=tomo_bin_a) + base_tau_b = self.basename(version, tomo_bin_a=tomo_bin_b) + self.rho_stat_handler.load_rho_stats(f"rho_stats_{base_rho}.fits") + + theta = self.rho_stat_handler.rho_stats["theta"] + n_bins = len(theta) + rho_0_p = self.rho_stat_handler.rho_stats["rho_0_p"] + rho_0_m = self.rho_stat_handler.rho_stats["rho_0_m"] + + tau_stats = {} + for key, base_tau in (("a", base_tau_a), ("b", base_tau_b)): + self.tau_stat_handler.load_tau_stats(f"tau_stats_{base_tau}.fits") + tau_stats[key] = { + col: self.tau_stat_handler.tau_stats[col] + for col in ("tau_0_p", "tau_0_m", "vartau_0_p", "vartau_0_m") + } + tau_0_p_a = tau_stats["a"]["tau_0_p"] + tau_0_m_a = tau_stats["a"]["tau_0_m"] + tau_0_p_b = tau_stats["b"]["tau_0_p"] + tau_0_m_b = tau_stats["b"]["tau_0_m"] + + if cov_type is not None: + outdir = f"{self.cc['paths']['output']}/rho_tau_stats" + cov_tau_path = Path(outdir) / f"cov_tau_{base_tau_a}_{cov_type}.npy" + cov_tau_p_a = np.load(cov_tau_path)[:n_bins, :n_bins] + cov_tau_path = Path(outdir) / f"cov_tau_{base_tau_b}_{cov_type}.npy" + cov_tau_p_b = np.load(cov_tau_path)[:n_bins, :n_bins] + cov_rho_path = Path(outdir) / f"cov_rho_{base_rho}_jk.npy" + cov_rho_p = np.load(cov_rho_path)[:n_bins, :n_bins] + else: + cov_tau_p_a = np.diag(tau_stats["a"]["vartau_0_p"]) + cov_tau_p_b = np.diag(tau_stats["b"]["vartau_0_p"]) + cov_rho_p = np.diag(self.rho_stat_handler.rho_stats["varrho_0_p"]) + + cov_tau_m_a = np.diag(tau_stats["a"]["vartau_0_m"]) + cov_tau_m_b = np.diag(tau_stats["b"]["vartau_0_m"]) + cov_rho_m = np.diag(self.rho_stat_handler.rho_stats["varrho_0_m"]) + + # Compute the scale-dependent xi_psf_sys and its error bars + same_bin = tomo_bin_a == tomo_bin_b + xi_psf_sys_plus, xi_psf_sys_plus_err = self._compute_scale_dependent_xi_psf_sys( + rho_0_p, + tau_0_p_a, + tau_0_p_b, + cov_rho_p, + cov_tau_p_a, + cov_tau_p_b, + same_bin=same_bin, + ) + xi_psf_sys_minus, xi_psf_sys_minus_err = ( + self._compute_scale_dependent_xi_psf_sys( + rho_0_m, + tau_0_m_a, + tau_0_m_b, + cov_rho_m, + cov_tau_m_a, + cov_tau_m_b, + same_bin=same_bin, + ) + ) + + jittered_theta = self._get_jittered_theta(theta, idx, len(versions), offset) + + y_plus = xi_psf_sys_plus * (theta if times_theta else 1) + y_plus_err = xi_psf_sys_plus_err * (theta if times_theta else 1) + y_minus = xi_psf_sys_minus * (theta if times_theta else 1) + y_minus_err = xi_psf_sys_minus_err * (theta if times_theta else 1) + + ax_plus.errorbar( + jittered_theta, + y_plus, + yerr=y_plus_err, + fmt=fmt, + capsize=capsize, + color=color, + ) + + ax_minus.errorbar( + jittered_theta, + y_minus, + yerr=y_minus_err, + fmt=fmt, + capsize=capsize, + color=color, + ) + + def _get_jittered_theta(self, theta, idx, n_versions, offset): + """Get the jittered theta values for better visualisation.""" + theta_widths = np.diff(theta) + theta_widths = np.append(theta_widths, theta_widths[-1]) + jitter_fraction = (idx - (n_versions - 1) / 2) * offset + jittered_theta = theta + jitter_fraction * theta_widths + return jittered_theta + + def _get_ax_plus(self, axs, tomo_bin_a, tomo_bin_b): + if (tomo_bin_a == "all") ^ (tomo_bin_b == "all"): + raise ValueError( + "Invalid combination of tomographic bins: 'all' and a specific bin." + ) + + if tomo_bin_a == "all" and tomo_bin_b == "all": + return axs[0] + + else: + nrows = axs.shape[0] + return axs[nrows - tomo_bin_b, tomo_bin_a - 1] + + def _get_ax_minus(self, axs, tomo_bin_a, tomo_bin_b): + if (tomo_bin_a == "all") ^ (tomo_bin_b == "all"): + raise ValueError( + "Invalid combination of tomographic bins: 'all' and a specific bin." + ) + + if tomo_bin_a == "all" and tomo_bin_b == "all": + return axs[2] + + else: + ncols = axs.shape[1] + return axs[tomo_bin_b - 1, ncols - tomo_bin_a] + + def _set_ax_visibility_to_false(self, axs, n_tomo_bins_plot): + """Set the visibility of empty axes to False for better visualization.""" + if n_tomo_bins_plot == 1: + axs[1].set_visible(False) + else: + for i in range(n_tomo_bins_plot): + axs[i, n_tomo_bins_plot - i].set_visible(False) + + +# %% diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 81bd22f0..503b02e8 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -30,6 +30,7 @@ def calculate_pure_eb( max_sep_int=300, nbins_int=1000, npatch=None, + compute_tomography=False, cov_path_int=None, ): """ @@ -51,45 +52,63 @@ def calculate_pure_eb( extend beyond the reporting range on both sides. npatch : int, optional Jackknife patch count. Defaults to self.npatch. + compute_tomography : bool, optional + Whether to compute the pure E/B modes for every tomographic bin pair + instead of the non-tomographic pair. Defaults to False. cov_path_int : str, optional Analytic ξ± covariance on the integration grid. Without it the - covariance is the jackknife of the integration-grid ξ±. + covariance is the jackknife of the integration-grid ξ±. One file + holds one pair's covariance, so it cannot be combined with + ``compute_tomography``. Returns ------- dict - The results of :func:`~sp_validation.b_modes.calculate_pure_eb_correlation`: - the six pure-mode arrays, their covariance ``cov`` and its - ``npatch`` record (``None`` for an analytic covariance), the - reporting grid and the integration-grid ξ±. + Mapping of ``"tomo_bin_{b1}_tomo_bin_{b2}"`` (the single key + ``"tomo_bin_all_tomo_bin_all"`` without tomography) to the results of + :func:`~sp_validation.b_modes.calculate_pure_eb_correlation`: the six + pure-mode arrays, their covariance ``cov`` and its ``npatch`` record + (``None`` for an analytic covariance), the reporting grid and the + integration-grid ξ±. """ self.print_start(f"Computing {version} pure E/B") + if cov_path_int is not None and compute_tomography: + raise ValueError( + "cov_path_int holds a single ξ± covariance; it cannot serve every " + "tomographic bin pair." + ) + reporting = self._binning(min_sep, max_sep, nbins) - gg_int = self.calculate_2pcf( + reporting_edges = np.geomspace( + reporting["min_sep"], reporting["max_sep"], reporting["nbins"] + 1 + ) + ggs_int = self.calculate_2pcf_version( version, npatch=npatch, + compute_tomography=compute_tomography, **self._binning(min_sep_int, max_sep_int, nbins_int), ) - if cov_path_int is not None: - cov_xi, npatch = np.loadtxt(cov_path_int), None - else: - cov_xi = gg_int.estimate_cov("jackknife", cross_patch_weight="match") - npatch = gg_int.npatch1 - - return calculate_pure_eb_correlation( - gg_int.meanr, - gg_int.xip, - gg_int.xim, - gg_int.weight, - np.append(gg_int.left_edges, gg_int.right_edges[-1]), - cov_xi, - np.geomspace( - reporting["min_sep"], reporting["max_sep"], reporting["nbins"] + 1 - ), - npatch=npatch, - ) + results = {} + for bin_key, gg_int in ggs_int.items(): + if cov_path_int is not None: + cov_xi, cov_npatch = np.loadtxt(cov_path_int), None + else: + cov_xi = gg_int.estimate_cov("jackknife", cross_patch_weight="match") + cov_npatch = gg_int.npatch1 + results[bin_key] = calculate_pure_eb_correlation( + gg_int.meanr, + gg_int.xip, + gg_int.xim, + gg_int.weight, + np.append(gg_int.left_edges, gg_int.right_edges[-1]), + cov_xi, + reporting_edges, + npatch=cov_npatch, + ) + + return results def plot_pure_eb( self, @@ -104,13 +123,14 @@ def plot_pure_eb( max_sep_int=300, nbins_int=1000, npatch=None, + compute_tomography=False, cov_path_int=None, results=None, ): """ Generate comprehensive pure E/B mode analysis plots. - Creates four types of plots for each version: + Creates four types of plots for each version and bin pair: 1. Integration vs Reporting comparison 2. E/B/Ambiguous correlation functions 3. 2D PTE heatmaps @@ -133,13 +153,17 @@ def plot_pure_eb( (default: 0.08-300 arcmin, 1000 bins) npatch : int, optional Number of patches for jackknife covariance. Uses self.npatch if None. + compute_tomography : bool, optional + Whether to compute and plot every tomographic bin pair instead of the + non-tomographic pair. Defaults to False. cov_path_int : str, optional Analytic ξ± covariance on the integration grid; the jackknife is used without it. results : dict or list, optional - Precalculated results to avoid recomputation. Can be a single results dict - for one version, or a list of results dicts for multiple versions. - If None (default), results will be calculated using calculate_pure_eb. + Precalculated :meth:`calculate_pure_eb` results (``{bin_key: result}``) + to avoid recomputation: a single such dict for one version, or a list + of them for multiple versions. If None (default), results will be + calculated using calculate_pure_eb. Notes ----- @@ -189,16 +213,8 @@ def plot_pure_eb( results_list = [None] * len(versions) for idx, version in enumerate(versions): - # Generate standardized output filename stub - out_stub = ( - f"{output_dir}/{version}_eb_minsep={min_sep}_" - f"maxsep={max_sep}_nbins={nbins}_minsepint={min_sep_int}_" - f"maxsepint={max_sep_int}_nbinsint={nbins_int}_npatch={npatch}_" - f"varmethod={var_method}" - ) - # Get or calculate results for this version - version_results = results_list[idx] or self.calculate_pure_eb( + bin_results = results_list[idx] or self.calculate_pure_eb( version, min_sep=min_sep, max_sep=max_sep, @@ -207,45 +223,61 @@ def plot_pure_eb( max_sep_int=max_sep_int, nbins_int=nbins_int, npatch=npatch, + compute_tomography=compute_tomography, cov_path_int=cov_path_int, ) + self._pure_eb_results[version] = {} - # Calculate E/B statistics for all bin combinations - version_results = calculate_eb_statistics(version_results) + for bin_key, version_results in bin_results.items(): + # Generate standardized output filename stub + out_stub = ( + f"{output_dir}/{version}_{bin_key}_eb_minsep={min_sep}_" + f"maxsep={max_sep}_nbins={nbins}_minsepint={min_sep_int}_" + f"maxsepint={max_sep_int}_nbinsint={nbins_int}_npatch={npatch}_" + f"varmethod={var_method}" + ) + label = ( + version + if bin_key == "tomo_bin_all_tomo_bin_all" + else f"{version} {bin_key}" + ) - # Integration vs Reporting comparison plot - plot_integration_vs_reporting( - version_results, - out_stub + "_integration_vs_reporting.png", - version, - ) + # Calculate E/B statistics for all bin combinations + version_results = calculate_eb_statistics(version_results) - # E/B/Ambiguous correlation functions plot - plot_pure_eb_correlations( - version_results, - out_stub + "_xis.png", - version, - fiducial_xip_scale_cut=fiducial_xip_scale_cut, - fiducial_xim_scale_cut=fiducial_xim_scale_cut, - ) + # Integration vs Reporting comparison plot + plot_integration_vs_reporting( + version_results, + out_stub + "_integration_vs_reporting.png", + label, + ) - # 2D PTE heatmaps plot - plot_pte_2d_heatmaps( - version_results, - version, - out_stub + "_ptes.png", - fiducial_xip_scale_cut=fiducial_xip_scale_cut, - fiducial_xim_scale_cut=fiducial_xim_scale_cut, - ) + # E/B/Ambiguous correlation functions plot + plot_pure_eb_correlations( + version_results, + out_stub + "_xis.png", + label, + fiducial_xip_scale_cut=fiducial_xip_scale_cut, + fiducial_xim_scale_cut=fiducial_xim_scale_cut, + ) - # Covariance matrix plot - plot_eb_covariance_matrix( - version_results["cov"], - covariance_label(version_results["npatch"]), - out_stub + "_covariance.png", - version, - ) + # 2D PTE heatmaps plot + plot_pte_2d_heatmaps( + version_results, + label, + out_stub + "_ptes.png", + fiducial_xip_scale_cut=fiducial_xip_scale_cut, + fiducial_xim_scale_cut=fiducial_xim_scale_cut, + ) + + # Covariance matrix plot + plot_eb_covariance_matrix( + version_results["cov"], + covariance_label(version_results["npatch"]), + out_stub + "_covariance.png", + label, + ) - # Save data products and store on instance - save_pure_eb_results(version_results, out_stub + "_data.npz") - self._pure_eb_results[version] = version_results + # Save data products and store on instance + save_pure_eb_results(version_results, out_stub + "_data.npz") + self._pure_eb_results[version][bin_key] = version_results diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 89dad2a9..7dea3542 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -1,334 +1,231 @@ """Real-space two-point diagnostics for cosmology validation. This mixin holds the real-space machinery: the TreeCorr two-point correlation -function (2PCF) ξ± measurement and its plots, the ratio of PSF systematics to -the cosmic-shear signal, and the aperture-mass dispersion ⟨M_ap²⟩ measurement -and plots. It depends on TreeCorr. +function (2PCF) ξ± measurement, the aperture-mass dispersion ⟨M_ap²⟩ +measurement, and the per-bin-pair plots of both. It depends on TreeCorr. """ import os import matplotlib.pyplot as plt -import matplotlib.ticker as mticker import numpy as np import treecorr -from cs_util import plots as cs_plots + +from sp_validation.statistics import jackknife_patch_centers class RealSpaceMixin: - def calculate_2pcf(self, ver, npatch=None, **treecorr_config): + def calculate_2pcf_version( + self, + ver, + npatch=None, + compute_tomography=False, + **treecorr_config, + ): """ - Calculate the two-point correlation function (2PCF) ξ± for a given catalog + Calculate the two-point correlation function (2PCF) ξ± for a single catalog version with TreeCorr. + This is the per-version child function. Use :meth:`calculate_2pcf` to run over every + version in ``self.versions`` in one call. + By default the class instance's `npatch` and `treecorr_config` entries are - used to - initialize the TreeCorr Catalog and GGCorrelation objects, but may be - overridden - by passing keyword arguments. + used to initialize the TreeCorr Catalog and GGCorrelation objects, but may + be overridden by passing keyword arguments. Parameters: ver (str): The catalog version to process. - npatch (int, optional): The number of patches to use for the calculation. - Defaults to the instance's `npatch` attribute. + npatch (int, optional): The number of patches to use for the + calculation. Defaults to the instance's `npatch` attribute. + + compute_tomography (bool, optional): Whether to compute tomographic + correlations. Defaults to False. - **treecorr_config: Additional TreeCorr configuration parameters that will - override the instance's default `treecorr_config`. For example, `min_sep=1`. + **treecorr_config: Additional TreeCorr configuration parameters that + will override the instance's default `treecorr_config`. For example, + `min_sep=1`. Returns: - treecorr.GGCorrelation: The TreeCorr GGCorrelation object containing the - computed 2PCF results. + dict: Mapping of ``"tomo_bin_{b1}_tomo_bin_{b2}"`` to the corresponding + treecorr.GGCorrelation object. For the non-tomographic case the single + key is ``"tomo_bin_all_tomo_bin_all"``. Notes: - - If the output file for the given configuration already exists, the - calculation is skipped, and the results are loaded from the file. - - If a patch file for the given configuration does not exist, it is - created during the process. - - The ``.txt`` TreeCorr dump is the only raw byproduct written here. + - The non-tomographic pair is written to the columns-only TreeCorr + dump ``xi_{basename}.txt``. If that file already exists, the pair is + read back from it instead of being recomputed. + - Seeded patch centres are computed once from the full catalogue and + shared by every tomographic bin pair. """ - self.print_magenta(f"Computing {ver} ξ±") - npatch = npatch or self.npatch treecorr_config = { **self._binning(**treecorr_config), "var_method": "jackknife" if int(npatch) > 1 else "shot", } - gg = treecorr.GGCorrelation(treecorr_config) + if compute_tomography: + tomo_bin_ids, tomo_bin_pairs = self._get_tomo_bins(ver) + if tomo_bin_ids is None or tomo_bin_pairs is None: + raise ValueError(f"Version {ver} does not have tomography information.") + self.print_magenta( + f"Computing tomographic ξ± for {ver} with {len(tomo_bin_pairs)} bins." + ) + else: + self.print_magenta(f"Computing non-tomographic ξ± for {ver}.") + tomo_bin_pairs = [("all", "all")] + + ggs = {f"tomo_bin_{b1}_tomo_bin_{b2}": None for b1, b2 in tomo_bin_pairs} + to_compute = [] + for bin1, bin2 in tomo_bin_pairs: + if (bin1, bin2) == ("all", "all"): + out_fname = self._xi_txt_path(ver, treecorr_config, npatch) + if os.path.exists(out_fname): + self.print_done(f"Skipping 2PCF calculation, {out_fname} exists") + gg = treecorr.GGCorrelation(treecorr_config) + gg.read(out_fname) + ggs["tomo_bin_all_tomo_bin_all"] = gg + continue + to_compute.append((bin1, bin2)) + + if to_compute: + cols = self._shear_columns(ver, compute_tomography) + patch_centers = self._patch_centers(cols, npatch) + + for bin1, bin2 in to_compute: + gg = treecorr.GGCorrelation(treecorr_config) + + cat_gal1 = self._bin_catalog(cols, bin1, npatch, patch_centers) + cat_gal2 = ( + self._bin_catalog(cols, bin2, npatch, patch_centers) + if bin1 != bin2 + else None + ) - # If the output file already exists, skip the calculation - out_fname = self._output_path( - f"{ver}_xi_minsep={treecorr_config['min_sep']}_maxsep={treecorr_config['max_sep']}_nbins={treecorr_config['nbins']}_npatch={npatch}.txt" - ) + gg.process(cat_gal1, cat2=cat_gal2) + + if (bin1, bin2) == ("all", "all"): + # Columns only. The covariance matrix lives in the SACC part; + # a per-patch ξ± realisation is an unblinded data vector + # nothing reads; and TreeCorr cannot read back a text file + # carrying the matrix without the per-patch results. + gg.write( + self._xi_txt_path(ver, treecorr_config, npatch), + write_patch_results=False, + write_cov=False, + ) - if os.path.exists(out_fname): - self.print_done(f"Skipping 2PCF calculation, {out_fname} exists") - gg.read(out_fname) + ggs[f"tomo_bin_{bin1}_tomo_bin_{bin2}"] = gg - else: - # Load data and create a catalog - with self.results[ver].temporarily_read_data(): - g1, g2 = self._calibrated_g(ver) - w = self._read_shear_cols(ver, "w_col") - - # Use patch file if it exists - patch_file = self._output_path(f"{ver}_patches_npatch={npatch}.dat") - - cat_gal = treecorr.Catalog( - ra=self.results[ver].dat_shear["RA"], - dec=self.results[ver].dat_shear["Dec"], - g1=g1, - g2=g2, - w=w, - ra_units=self.treecorr_config["ra_units"], - dec_units=self.treecorr_config["dec_units"], - npatch=npatch, - patch_centers=patch_file if os.path.exists(patch_file) else None, - ) + self.print_done(f"Done 2PCF for {ver}.") - # If no patch file exists, save the current patches - if not os.path.exists(patch_file): - cat_gal.write_patch_centers(patch_file) + return ggs - # Process the catalog & write the correlation functions - gg.process(cat_gal) - # Columns only. The covariance matrix lives in the SACC part; a - # per-patch ξ± realisation is an unblinded data vector nothing reads; - # and TreeCorr cannot read back a text file carrying the matrix - # without the per-patch results. - gg.write(out_fname, write_patch_results=False, write_cov=False) + def _xi_txt_path(self, ver, treecorr_config, npatch): + """Path of the non-tomographic ξ± TreeCorr dump for a version.""" + return self._output_path( + f"xi_{self.basename(ver, treecorr_config=treecorr_config, npatch=npatch)}.txt" + ) - # Add correlation object to class - if not hasattr(self, "cat_ggs"): - self.cat_ggs = {} - self.cat_ggs[ver] = gg + def _shear_columns(self, ver, compute_tomography): + """Positions, calibrated shears, weights and bin labels of a version. - self.print_done("Done 2PCF") + All arrays come from the same rows of the version's shear catalogue, as + read by ``self.results[ver]``. The bin labels are ``None`` unless + ``compute_tomography``. + """ + with self.results[ver].temporarily_read_data(): + dat = self.results[ver].dat_shear + g1, g2 = self._calibrated_g(ver) + return { + "ra": np.asarray(dat["RA"]), + "dec": np.asarray(dat["Dec"]), + "g1": np.asarray(g1), + "g2": np.asarray(g2), + "w": np.asarray(self._read_shear_cols(ver, "w_col")), + "tomo_bin": ( + np.asarray(dat[self.cc[ver]["shear"]["tomo_bin_col"]]) + if compute_tomography + else None + ), + } - return gg + def _patch_centers(self, cols, npatch): + """Seeded patch centres from the full catalogue of one version.""" + if int(npatch) <= 1: + return None + cat = treecorr.Catalog( + ra=cols["ra"], + dec=cols["dec"], + w=cols["w"], + ra_units=self.treecorr_config["ra_units"], + dec_units=self.treecorr_config["dec_units"], + ) + return jackknife_patch_centers(cat, int(npatch)) + + def _bin_catalog(self, cols, tomo_bin_id, npatch, patch_centers=None): + """TreeCorr catalogue of one tomographic bin (``"all"``: every row).""" + mask = slice(None) if tomo_bin_id == "all" else cols["tomo_bin"] == tomo_bin_id + return treecorr.Catalog( + ra=cols["ra"][mask], + dec=cols["dec"][mask], + g1=cols["g1"][mask], + g2=cols["g2"][mask], + w=cols["w"][mask], + ra_units=self.treecorr_config["ra_units"], + dec_units=self.treecorr_config["dec_units"], + npatch=npatch, + patch_centers=patch_centers, + ) - def plot_2pcf(self): - # Plot of n_pairs - plt.subplots(ncols=1, nrows=1) - for ver in self.versions: - self.calculate_2pcf(ver) - plt.plot( - self.cat_ggs[ver].meanr, - self.cat_ggs[ver].npairs, - label=ver, - ls=self.cc[ver]["ls"], - color=self.cc[ver]["colour"], - ) - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.ylabel(r"$n_{\rm pair}$") - plt.legend() - out_path = self._output_path("n_pair.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"n_pair plot saved to {out_path}") - - # Plot of xi_+ - plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) - for idx, ver in enumerate(self.versions): - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, fx=1.05, nx=len(ver)), - self.cat_ggs[ver].xip, - yerr=np.sqrt(self.cat_ggs[ver].varxip), - label=ver, - ls=self.cc[ver]["ls"], - color=self.cc[ver]["colour"], - ) - plt.xscale("log") - plt.yscale("log") - plt.legend() - plt.ticklabel_format(axis="y") - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.xlim([self.theta_min_plot, self.theta_max_plot]) - plt.ylabel(r"$\xi_+(\theta)$") - out_path = self._output_path("xi_p.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"xi_plus plot saved to {out_path}") - - # Plot of xi_- - plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) - for idx, ver in enumerate(self.versions): - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, fx=1.05, nx=len(ver)), - self.cat_ggs[ver].xim, - yerr=np.sqrt(self.cat_ggs[ver].varxim), - label=ver, - ls=self.cc[ver]["ls"], - color=self.cc[ver]["colour"], - ) - plt.xscale("log") - plt.yscale("log") - plt.legend() - plt.ticklabel_format(axis="y") - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.xlim([self.theta_min_plot, self.theta_max_plot]) - plt.ylabel(r"$\xi_-(\theta)$") - out_path = self._output_path("xi_m.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"xi_minus plot saved to {out_path}") - - # Plot of xi_+(theta) * theta - plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) - for idx, ver in enumerate(self.versions): - plt.errorbar( - self.cat_ggs[ver].meanr, - self.cat_ggs[ver].xip * self.cat_ggs[ver].meanr, - yerr=np.sqrt(self.cat_ggs[ver].varxip) * self.cat_ggs[ver].meanr, - label=ver, - ls=self.cc[ver]["ls"], - color=self.cc[ver]["colour"], - ) - plt.xscale("log") - plt.legend() - plt.ticklabel_format(axis="y") - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.xlim([self.theta_min_plot, self.theta_max_plot]) - plt.ylabel(r"$\theta \xi_+(\theta)$") - out_path = self._output_path("xi_p_theta.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"xi_plus_theta plot saved to {out_path}") - - # Plot of xi_- * theta - plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) - for idx, ver in enumerate(self.versions): - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xim * self.cat_ggs[ver].meanr, - yerr=np.sqrt(self.cat_ggs[ver].varxim) * self.cat_ggs[ver].meanr, - label=ver, - ls=self.cc[ver]["ls"], - color=self.cc[ver]["colour"], - ) - plt.xscale("log") - plt.legend() - plt.ticklabel_format(axis="y") - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.xlim([self.theta_min_plot, self.theta_max_plot]) - plt.ylabel(r"$\theta \xi_-(\theta)$") - out_path = self._output_path("xi_m_theta.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"xi_minus_theta plot saved to {out_path}") - - # Plot of xi_+ with and without xi_psf_sys - # but skip if xi_psf_sys is not calculated since that takes forever - if hasattr(self, "_xi_psf_sys"): - for idx, ver in enumerate(self.versions): - plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xip, - yerr=np.sqrt(self.cat_ggs[ver].varxim), - label=r"$\xi_+$", - ls="solid", - color="green", - ) - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.xi_psf_sys[ver]["mean"], - yerr=np.sqrt(self.xi_psf_sys[ver]["var"]), - label=r"$\xi^{\rm psf}_{+, {\rm sys}}$", - ls="dotted", - color="red", - ) - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xip + self.xi_psf_sys[ver]["mean"], - yerr=np.sqrt( - self.cat_ggs[ver].varxip + self.xi_psf_sys[ver]["var"] - ), - label=r"$\xi_+ + \xi^{\rm psf}_{+, {\rm sys}}$", - ls="dashdot", - color="magenta", - ) + def calculate_2pcf( + self, + npatch=None, + compute_tomography=False, + **treecorr_config, + ): + """ + Calculate the 2PCF ξ± for every catalog version in ``self.versions``. - plt.xscale("log") - plt.yscale("log") - plt.legend() - plt.ticklabel_format(axis="y") - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.xlim([self.theta_min_plot, self.theta_max_plot]) - plt.ylim(1e-8, 5e-4) - plt.ylabel(r"$\xi_+(\theta)$") - out_path = self._output_path(f"xi_p_xi_psf_sys_{ver}.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"xi_plus_xi_psf_sys {ver} plot saved to {out_path}") - - def plot_ratio_xi_sys_xi(self, threshold=0.1, offset=0.02): - - plt.subplots(ncols=1, nrows=1, figsize=(10, 7)) - - for idx, ver in enumerate(self.versions): - self.calculate_2pcf(ver) - xi_psf_sys = self.xi_psf_sys[ver] - gg = self.cat_ggs[ver] - - ratio = xi_psf_sys["mean"] / gg.xip - ratio_err = np.sqrt( - (np.sqrt(xi_psf_sys["var"]) / gg.xip) ** 2 - + (xi_psf_sys["mean"] * np.sqrt(gg.varxip) / gg.xip**2) ** 2 - ) + Parent function that iterates over ``self.versions`` and delegates the + per-version computation to :meth:`calculate_2pcf_version`. Results are stored + in ``self.cat_ggs`` keyed by version. - theta = gg.meanr - jittered_theta = theta * (1 + idx * offset) + Parameters: + npatch (int, optional): Number of patches to use; defaults to the + instance's `npatch` attribute. - plt.errorbar( - jittered_theta, - ratio, - yerr=ratio_err, - label=ver, - ls=self.cc[ver]["ls"], - color=self.cc[ver]["colour"], - fmt=self.cc[ver].get("marker", None), - capsize=5, + compute_tomography (bool, optional): Whether to compute tomographic + correlations. Defaults to False. + + **treecorr_config: Additional TreeCorr configuration parameters passed + through to each per-version call. + + Returns: + dict: ``self.cat_ggs``, mapping each version to its + ``{"tomo_bin_{b1}_tomo_bin_{b2}": treecorr.GGCorrelation}`` dict. + """ + self.cat_ggs = {} + for ver in self.versions: + self.cat_ggs[ver] = self.calculate_2pcf_version( + ver, + npatch=npatch, + compute_tomography=compute_tomography, + **treecorr_config, ) - plt.fill_between( - [self.theta_min_plot, self.theta_max_plot], - -threshold, - +threshold, - color="black", - alpha=0.1, - label=f"{threshold:.0%} threshold", - ) - plt.plot( - [self.theta_min_plot, self.theta_max_plot], - [threshold, threshold], - ls="dashed", - color="black", - ) - plt.plot( - [self.theta_min_plot, self.theta_max_plot], - [-threshold, -threshold], - ls="dashed", - color="black", - ) - plt.xscale("log") - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.ylabel(r"$\xi^{\rm psf}_{+, {\rm sys}} / \xi_+$") - plt.gca().yaxis.set_major_formatter(mticker.PercentFormatter(xmax=1)) - plt.legend() - plt.title("Ratio of PSF systematics to cosmic shear signal") - out_path = self._output_path("ratio_xi_sys_xi.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - print(f"Ratio of xi_psf_sys to xi plot saved to {out_path}") + return self.cat_ggs def calculate_aperture_mass_dispersion( - self, theta_min=0.3, theta_max=200, nbins=500, nbins_map=15, npatch=25 + self, + theta_min=0.3, + theta_max=200, + nbins=500, + nbins_map=15, + npatch=25, + compute_tomography=False, ): - self.print_start("Computing aperture-mass dispersion") - self._map2 = {} theta_map = np.geomspace(theta_min * 5, theta_max / 2, nbins_map) self._map2["theta_map"] = theta_map @@ -336,53 +233,48 @@ def calculate_aperture_mass_dispersion( treecorr_config = self._binning(theta_min, theta_max, nbins) for ver in self.versions: - self.print_magenta(ver) + if compute_tomography: + tomo_bin_ids, tomo_bin_pairs = self._get_tomo_bins(ver) + if tomo_bin_ids is None or tomo_bin_pairs is None: + raise ValueError( + f"Version {ver} does not have tomography information." + ) + self.print_magenta( + f"Computing MAP for {ver} with {len(tomo_bin_pairs)} bins." + ) + else: + self.print_magenta(f"Computing non-tomographic MAP for {ver}.") - gg = treecorr.GGCorrelation(treecorr_config) + tomo_bin_pairs = [("all", "all")] - out_fname = self._output_path(f"xi_for_map2_{ver}.txt") - if os.path.exists(out_fname): - self.print_green(f"Skipping xi for Map2, {out_fname} exists") - gg.read(out_fname) - else: - with self.results[ver].temporarily_read_data(): - g1, g2 = self._calibrated_g(ver) - cat_gal = treecorr.Catalog( - ra=self.results[ver].dat_shear["RA"], - dec=self.results[ver].dat_shear["Dec"], - g1=g1, - g2=g2, - w=self._read_shear_cols(ver, "w_col"), - ra_units=self.treecorr_config["ra_units"], - dec_units=self.treecorr_config["dec_units"], - npatch=npatch, - ) + self._map2.setdefault(ver, {}) + cols = self._shear_columns(ver, compute_tomography) + patch_centers = self._patch_centers(cols, npatch) - gg.process(cat_gal) - gg.write(out_fname) - del cat_gal - del g1 - del g2 + for bin1, bin2 in tomo_bin_pairs: + gg = treecorr.GGCorrelation(treecorr_config) - mapsq, mapsq_im, mxsq, mxsq_im, varmapsq = gg.calculateMapSq( - R=theta_map, - m2_uform="Schneider", - ) - out_fname_map2 = self._output_path(f"map2_{ver}.txt") - if os.path.exists(out_fname_map2): - self.print_green(f"Skipping Map2, {out_fname_map2} exists") - else: - print(f"Writing Map2 to output file {out_fname_map2} ") - gg.writeMapSq(out_fname_map2, R=theta_map, m2_uform="Schneider") - self._map2[ver] = { - "mapsq": mapsq, - "mapsq_im": mapsq_im, - "mxsq": mxsq, - "mxsq_im": mxsq_im, - "varmapsq": varmapsq, - } + cat_gal1 = self._bin_catalog(cols, bin1, npatch, patch_centers) + cat_gal2 = ( + self._bin_catalog(cols, bin2, npatch, patch_centers) + if bin1 != bin2 + else None + ) + + gg.process(cat_gal1, cat2=cat_gal2) - self.print_done("Done aperture-mass dispersion") + mapsq, mapsq_im, mxsq, mxsq_im, varmapsq = gg.calculateMapSq( + R=theta_map, + m2_uform="Schneider", + ) + self._map2[ver][f"tomo_bin_{bin1}_tomo_bin_{bin2}"] = { + "mapsq": mapsq, + "mapsq_im": mapsq_im, + "mxsq": mxsq, + "mxsq_im": mxsq_im, + "varmapsq": varmapsq, + } + self.print_done(f"Done aperture-mass dispersion for {ver}.") @property def map2(self): @@ -390,64 +282,502 @@ def map2(self): self.calculate_aperture_mass_dispersion() return self._map2 - def plot_aperture_mass_dispersion(self): - for mode in ["mapsq", "mapsq_im", "mxsq", "mxsq_im"]: - x = [self.map2["theta_map"] for ver in self.versions] - y = [self.map2[ver][mode] for ver in self.versions] - yerr = [np.sqrt(self.map2[ver]["varmapsq"]) for ver in self.versions] - labels = list(self.versions) - colors = [self.cc[ver]["colour"] for ver in self.versions] - linestyles = [self.cc[ver]["ls"] for ver in self.versions] - - xlabel = r"$\theta$ [arcmin]" - ylabel = "dispersion" - title = f"Aperture-mass dispersion {mode}" - out_path = self._output_path(f"{mode}.png") - cs_plots.plot_data_1d( - x, - y, - yerr, - title, - xlabel, - ylabel, - out_path=None, - labels=labels, - xlog=True, - xlim=[self.theta_min_plot, self.theta_max_plot], - ylim=[-2e-6, 5e-6], - colors=colors, - linestyles=linestyles, - shift_x=True, + def plot_2pcf( + self, tomography=False, offset=0.02, alpha=1.0, show=True, close=True + ): + """Plot ξ± of every version, one panel per bin pair. + + Measures, or reads back, the 2PCF with :meth:`calculate_2pcf` and draws + it with :meth:`plot_2pcf_tomography`, as ξ± (log-log) and as θ·ξ±. + Writes ``xi_pm_tomography_{tomography}.png`` and + ``xi_pm_theta_tomography_{tomography}.png`` under the output directory. + """ + self.calculate_2pcf(compute_tomography=tomography) + + for times_theta in (False, True): + prefix = r"$\theta\,$" if times_theta else "" + suffix = "_theta" if times_theta else "" + self.plot_2pcf_tomography( + self._xiplus_ximinus_sample_x_y_plot_function, + r"$\theta$ [arcmin]", + prefix + r"$\xi_+(\theta)$", + prefix + r"$\xi_-(\theta)$", + (0.05, 0.9) if times_theta else (0.8, 0.95), + extract_text_offset=times_theta, + add_index_version_to_kwargs=True, + x_scale="log", + y_scale="linear" if times_theta else "log", + tomography=tomography, + savefig=self._output_path(f"xi_pm{suffix}_tomography_{tomography}.png"), + show=show, + close=close, + offset=offset, + times_theta=times_theta, + alpha=alpha, ) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"linear-scale {mode} plot saved to {out_path}") - - for mode in ["mapsq", "mapsq_im", "mxsq", "mxsq_im"]: - x = [self.map2["theta_map"] for ver in self.versions] - y = [np.abs(self.map2[ver][mode]) for ver in self.versions] - yerr = [np.sqrt(self.map2[ver]["varmapsq"]) for ver in self.versions] - xlabel = r"$\theta$ [arcmin]" - ylabel = "dispersion" - title = f"Aperture-mass dispersion mode {mode}" - out_path = self._output_path(f"{mode}_log.png") - cs_plots.plot_data_1d( - x, - y, - yerr, - title, - xlabel, - ylabel, - out_path=None, - labels=labels, - xlog=True, - ylog=True, - xlim=[self.theta_min_plot, self.theta_max_plot], - ylim=[1e-8, 1e-5], - colors=colors, - linestyles=linestyles, - shift_x=True, + + def plot_ratio_xi_sys_xi( + self, tomography=False, threshold=0.1, offset=0.02, show=True, close=True + ): + """Plot ξ^{PSF, sys}_± / ξ± of every version, one panel per bin pair. + + The band marks ``±threshold``. ξ± comes from :meth:`calculate_2pcf` and + ξ^{PSF, sys} from the ``xi_psf_sys`` property, both on the instance's + ``treecorr_config`` binning. Writes ``ratio_xi_sys_xi.png`` (non- + tomographic) or ``ratio_xi_sys_xi_tomography.png`` under the output + directory. + """ + if tomography and not self.compute_tomography: + raise ValueError( + "plot_ratio_xi_sys_xi(tomography=True) needs the tomographic " + "xi_psf_sys; construct CosmologyValidation with " + "compute_tomography=True" + ) + self.calculate_2pcf(compute_tomography=tomography) + + y_label = r"$\xi^{{\rm PSF, sys}}_{0} / \xi_{0}$" + self.plot_2pcf_tomography( + self._ratio_xi_sys_xi_x_y_plot_function, + r"$\theta$ [arcmin]", + y_label.format("+"), + y_label.format("-"), + (0.8, 0.95), + extract_text_offset=False, + add_index_version_to_kwargs=True, + x_scale="log", + tomography=tomography, + savefig=self._output_path( + "ratio_xi_sys_xi_tomography.png" + if tomography + else "ratio_xi_sys_xi.png" + ), + show=show, + close=close, + offset=offset, + threshold=threshold, + ) + + def plot_2pcf_tomography( + self, + x_y_plot_function, + x_label, + y_label_plus, + y_label_minus, + tomo_bin_label_position, + extract_text_offset, + add_index_version_to_kwargs, + x_scale=None, + y_scale=None, + tomography=False, + versions=None, + colors=None, + savefig=None, + show=True, + close=True, + **kwargs, + ): + """ + Standard plot function for 2-point correlation functions with tomographic bins. + + Parameters + ---------- + x_y_plot_function : callable + Function to plot the x and y data. Should accept axes for `+' and `-' components, version, tomo_bin_indices, and kwargs. + x_label : str + Label for the x-axis. + y_label_plus : str + Label for the y-axis of the `+' component. + y_label_minus : str + Label for the y-axis of the `-' component. + tomo_bin_label_position : tuple + Position to place the tomographic bin labels in axes coordinates (x, y). + extract_text_offset : bool + If True, extract the y-axis offset text and include it in the y-label. + add_index_version_to_kwargs : bool + If True, add the index of the version to the kwargs for plotting. + x_scale : str, optional + Scale for the x-axis ('linear', 'log', etc.). If None, default scale is used. + y_scale : str, optional + Scale for the y-axis ('linear', 'log', etc.). If None, default scale is used. + tomography : bool, optional + If True, plot the tomographic bins. Default is False. + versions : list, optional + List of versions to plot. If None, all versions are plotted. + colors : list, optional + List of colors for each version. If None, default colors are used. + savefig : str, optional + If provided, save the figure to this file. + show : bool, optional + If True, show the figure. + close : bool, optional + If True, close the figure after saving or showing. + kwargs : dict + Additional keyword arguments to pass to the plotting function. + """ + versions = versions if versions is not None else self.versions + colors = ( + colors + if colors is not None + else [self.cc[ver]["colour"] for ver in versions] + ) + + # First get the max number of tomo bins among the versions. + tomo_bins = self._get_tomo_bins_for_versions(versions, tomography=tomography) + + max_key = max(tomo_bins, key=lambda k: len(tomo_bins[k]["ids"])) + n_tomo_bins_plot = len(tomo_bins[max_key]["ids"]) + reference_tomo_bin_pairs = tomo_bins[max_key]["pairs"] + + n_rows = n_tomo_bins_plot + n_cols = n_tomo_bins_plot + 2 + + # First start with the quantiles plot + fig, axs = plt.subplots( + n_rows, + n_cols, + figsize=(5 * n_cols, 5 * n_rows), + sharex=True, + sharey=True, + gridspec_kw={"wspace": 0, "hspace": 0}, + ) + + for idx, (ver, color) in enumerate(zip(versions, colors)): + tomo_bin_pairs = tomo_bins[ver]["pairs"] + + kwargs["color"] = color + if add_index_version_to_kwargs: + kwargs["idx"] = idx + kwargs["versions"] = versions + + for tomo_bin_a, tomo_bin_b in tomo_bin_pairs: + # Plot the nsamples last samples + ax_plus = self._get_ax_plus(axs, tomo_bin_a, tomo_bin_b) + ax_minus = self._get_ax_minus(axs, tomo_bin_a, tomo_bin_b) + + # Apply the x_y_plot_function to plot the data + x_y_plot_function( + ax_plus, ax_minus, ver, tomo_bin_a, tomo_bin_b, **kwargs + ) + + # Draw to extract the y-axis text offset + fig.canvas.draw() + + # Set the visibility to false where necessary + self._set_ax_visibility_to_false(axs, n_tomo_bins_plot) + + # Set the labels and scales for the plots + for tomo_bin_a, tomo_bin_b in reference_tomo_bin_pairs: + ax_plus = self._get_ax_plus(axs, tomo_bin_a, tomo_bin_b) + ax_minus = self._get_ax_minus(axs, tomo_bin_a, tomo_bin_b) + + ax_plus.tick_params( + axis="both", + which="both", + direction="in", + bottom=True, + top=False, + labelbottom=tomo_bin_b == 1 or tomo_bin_b == "all", + left=True, + right=False, + labelleft=tomo_bin_a == 1 or tomo_bin_a == "all", + ) + if x_scale is not None: + ax_plus.set_xscale(x_scale) + + ax_plus.text( + tomo_bin_label_position[0], + tomo_bin_label_position[1], + f"{tomo_bin_a}-{tomo_bin_b}", + transform=ax_plus.transAxes, + verticalalignment="top", + bbox=dict( + boxstyle="square", + facecolor="white", + edgecolor="black", + alpha=0.8, + ), + ) + ax_plus.axhline(0, color="k", linestyle="--") + + if y_scale is not None: + ax_plus.set_yscale(y_scale) + if tomo_bin_b == 1 or tomo_bin_b == "all": + ax_plus.set_xlabel(r"$\theta$ [arcmin]") + if tomo_bin_a == 1 or tomo_bin_a == "all": + text_offset = ( + ax_plus.yaxis.get_offset_text().get_text() + if extract_text_offset + else "" + ) + ax_plus.set_ylabel(y_label_plus + text_offset) + ax_plus.yaxis.get_offset_text().set_visible( + False + ) # Hide the offset text for the plus ax + + # Move the ticks to the right for the minus ax + ax_minus.yaxis.tick_right() + ax_minus.yaxis.set_label_position("right") + ax_minus.tick_params( + axis="both", + which="both", + direction="in", + bottom=True, + top=False, + labelbottom=tomo_bin_b == n_tomo_bins_plot or tomo_bin_b == "all", + left=False, + right=True, + labelleft=False, + labelright=tomo_bin_a == 1 or tomo_bin_a == "all", + ) + if x_scale is not None: + ax_minus.set_xscale(x_scale) + ax_minus.text( + tomo_bin_label_position[0], + tomo_bin_label_position[1], + f"{tomo_bin_a}-{tomo_bin_b}", + transform=ax_minus.transAxes, + verticalalignment="top", + bbox=dict( + boxstyle="square", + facecolor="white", + edgecolor="black", + alpha=0.8, + ), + ) + ax_minus.axhline(0, color="k", linestyle="--") + if y_scale is not None: + ax_minus.set_yscale(y_scale) + if tomo_bin_b == n_tomo_bins_plot or tomo_bin_b == "all": + ax_minus.set_xlabel(x_label) + if tomo_bin_a == 1 or tomo_bin_a == "all": + text_offset = ( + ax_minus.yaxis.get_offset_text().get_text() + if extract_text_offset + else "" + ) + ax_minus.set_ylabel(y_label_minus + text_offset) + ax_minus.yaxis.get_offset_text().set_visible( + False + ) # Hide the offset text for the minus ax + + # Build the legend + handles = [] + for ver, color in zip(versions, colors): + label = self.cc[ver]["label"] if "label" in self.cc[ver] else ver + handles.append(plt.Line2D([0], [0], color=color, lw=2, label=label)) + fig.legend( + handles=handles, + loc="upper center", + ncol=3, + frameon=False, + bbox_to_anchor=(0.5, 0.0), + ) + + if savefig is not None: + plt.savefig(savefig, dpi=300, bbox_inches="tight") + self.print_done(f"Plot saved to {os.path.abspath(savefig)}") + + if show: + plt.show() + + if close: + plt.close() + + def _xiplus_ximinus_sample_x_y_plot_function( + self, + ax_plus, + ax_minus, + version, + tomo_bin_a, + tomo_bin_b, + idx, + versions, + color, + offset, + times_theta, + alpha, + ): + """Plot the measured ξ± 2PCF for one version/tomographic-bin pair. + + Uses the jackknife covariance from TreeCorr to plot the error bars. This function is fed into :meth:`plot_2pcf_tomography` as the ``x_y_plot_function`` argument. + + Parameters + ---------- + ax_plus, ax_minus : matplotlib.axes.Axes + Axes for the ξ+ and ξ- components. + version : str + Catalog version to plot. + tomo_bin_a, tomo_bin_b : int or str + Tomographic bin pair (``"all"`` for the non-tomographic case). + idx : int + Index of ``version`` within ``versions`` (used for the x-jitter). + versions : list + Full list of versions being plotted. + color : str + Colour for this version. + offset : float + Fractional jitter applied to θ for readability. + times_theta : bool + If True, plot θ·ξ± rather than ξ±. + alpha : float + Opacity of the plotted points/error bars. + """ + # Get the measured 2PCF for this version and tomographic-bin pair. + gg = self.cat_ggs[version][f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}"] + + # Angular scales of the measurement. + theta = gg.meanr + + # Add the offset to the theta values for better visualisation. + jittered_theta = self._get_jittered_theta(theta, idx, len(versions), offset) + + scale = theta if times_theta else 1 + + y_plus = gg.xip * scale + y_minus = gg.xim * scale + yerr_plus = np.sqrt(gg.varxip) * scale + yerr_minus = np.sqrt(gg.varxim) * scale + + ax_plus.errorbar( + jittered_theta, + y_plus, + yerr=yerr_plus, + color=color, + alpha=alpha, + fmt="o", + markersize=3, + capsize=2, + ) + + ax_minus.errorbar( + jittered_theta, + y_minus, + yerr=yerr_minus, + color=color, + alpha=alpha, + fmt="o", + markersize=3, + capsize=2, + ) + + def _mapsq_mxsq_sample_x_y_plot_function( + self, + ax_plus, + ax_minus, + version, + tomo_bin_a, + tomo_bin_b, + idx, + versions, + color, + offset, + times_theta, + alpha, + ): + """Plot the aperture-mass dispersion ⟨M_ap²⟩ / ⟨M_ײ⟩ for one bin pair. + + Uses the jackknife covariance from TreeCorr to plot the error bars. This function is fed into :meth:`plot_2pcf_tomography` as the ``x_y_plot_function`` argument. The E-mode ⟨M_ap²⟩ is placed on the ``plus`` axis and the B-mode ⟨M_ײ⟩ on the ``minus`` axis. + + Parameters + ---------- + ax_plus, ax_minus : matplotlib.axes.Axes + Axes for the E-mode (⟨M_ap²⟩) and B-mode (⟨M_ײ⟩) components. + version : str + Catalog version to plot. + tomo_bin_a, tomo_bin_b : int or str + Tomographic bin pair (``"all"`` for the non-tomographic case). + idx : int + Index of ``version`` within ``versions`` (used for the x-jitter). + versions : list + Full list of versions being plotted. + color : str + Colour for this version. + offset : float + Fractional jitter applied to θ for readability. + times_theta : bool + If True, plot θ·⟨M²⟩ rather than ⟨M²⟩. + alpha : float + Opacity of the plotted points/error bars. + """ + # Get the aperture-mass dispersion for this version and bin pair. + map2 = self.map2[version][f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}"] + + # Angular scales of the measurement. + theta = self.map2["theta_map"] + + # Add the offset to the theta values for better visualisation. + jittered_theta = self._get_jittered_theta(theta, idx, len(versions), offset) + + scale = theta if times_theta else 1 + + y_plus = map2["mapsq"] * scale + y_minus = map2["mxsq"] * scale + # Both E- and B-mode share the same variance estimate. + yerr = np.sqrt(map2["varmapsq"]) * scale + + ax_plus.errorbar( + jittered_theta, + y_plus, + yerr=yerr, + color=color, + alpha=alpha, + fmt="o", + markersize=3, + capsize=2, + ) + + ax_minus.errorbar( + jittered_theta, + y_minus, + yerr=yerr, + color=color, + alpha=alpha, + fmt="o", + markersize=3, + capsize=2, + ) + + def _ratio_xi_sys_xi_x_y_plot_function( + self, + ax_plus, + ax_minus, + version, + tomo_bin_a, + tomo_bin_b, + idx, + versions, + color, + offset, + threshold, + ): + """Plot ξ^{PSF, sys}_± / ξ± for one version/tomographic-bin pair. + + Fed into :meth:`plot_2pcf_tomography` as the ``x_y_plot_function`` + argument. The error bar propagates the variances of both ξ^{PSF, sys} + and ξ±; the first version also draws the ``±threshold`` band. + """ + key = f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}" + gg = self.cat_ggs[version][key] + xi_psf_sys = self.xi_psf_sys[version][key] + + theta = self._get_jittered_theta(gg.meanr, idx, len(versions), offset) + + for ax, xi, var_xi, component in ( + (ax_plus, gg.xip, gg.varxip, "plus"), + (ax_minus, gg.xim, gg.varxim, "minus"), + ): + mean = xi_psf_sys[f"mean_{component}"] + var = xi_psf_sys[f"var_{component}"] + ratio = mean / xi + ratio_err = np.sqrt(var / xi**2 + mean**2 * var_xi / xi**4) + ax.errorbar( + theta, + ratio, + yerr=ratio_err, + color=color, + fmt=self.cc[version].get("marker", "o"), + markersize=3, + capsize=2, ) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"log-scale {mode} plot saved to {out_path}") + if idx == 0: + ax.axhspan(-threshold, threshold, color="black", alpha=0.1) diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index 8b7f57b7..b10bfe81 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -68,14 +68,16 @@ def xi_to_sacc( return s -def pseudo_cl_to_sacc(nz, metadata, ell_eff, cl_all, wsp, covariance=None): +def pseudo_cl_to_sacc(nz, metadata, ell_eff, cl_all, wsp, covariance=None, nside=None): """One pseudo-Cℓ part: EE/BB/EB with the shared bandpower window. ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); the - window comes from :func:`bandpower_window_from_workspace`. ``covariance``, - when given, is the dense ``[EE; BB; EB]``-ordered block matching insertion. + window comes from :func:`bandpower_window_from_workspace`. Supply ``nside`` + only for HEALPix-map spectra, whose measured spectrum includes ``pw²(ℓ)``; + catalogue-based spectra have no pixel window. ``covariance``, when given, + is the dense ``[EE; BB; EB]``-ordered block matching insertion. """ - window_ells, window_weights = bandpower_window_from_workspace(wsp) + window_ells, window_weights = bandpower_window_from_workspace(wsp, nside) s = sio.new_sacc(nz, metadata) sio.add_pseudo_cl( s, diff --git a/src/sp_validation/glass_mock.py b/src/sp_validation/glass_mock.py index 652fe713..b6347aed 100644 --- a/src/sp_validation/glass_mock.py +++ b/src/sp_validation/glass_mock.py @@ -75,12 +75,18 @@ class GlassMockConfig: As_init: float = 2.1e-9 # seed As before sigma8 rescaling kmax: float = 20.0 # --- galaxy population (downstream of map generation) --- - n_arcmin2: float = 6.0905 - sigma_e: float = 0.2684 - bias: float = 1.2 # constant linear galaxy bias b(z) + n_arcmin2: float | list = 6.0905 + sigma_e: float | list = 0.2684 + bias: float | list = 1.2 # constant linear galaxy bias b(z) phz_sigma_0: float = 0.03 nbins: int = 1 # number of tomographic bins ia_bias: float | None = None + # --- Runtime options --- + limber: bool = False + mask_path: str | None = None + nz_path: str | None = None + output_path: str | None = None + output_prefix: str | None = None @classmethod def from_planck18(cls, **overrides) -> "GlassMockConfig": @@ -100,6 +106,21 @@ def from_planck18(cls, **overrides) -> "GlassMockConfig": base.update(overrides) return cls(**base) + @classmethod + def from_yaml(cls, yaml_config, seed=42) -> "GlassMockConfig": + """ + Build a config for the GLASS mock generation from a YAML configuration file. + + Reads the dataclass fields from the YAML file. If absent, the field is kept at its default value. + """ + import yaml + + with open(yaml_config, "r") as f: + data = yaml.safe_load(f) + + overrides = {k: v for k, v in data.items() if k in cls.__dataclass_fields__} + return cls(**{**overrides, "seed": seed}) + @property def lmax(self) -> int: """``lmax`` is tied to ``nside`` in the production mocks.""" @@ -110,6 +131,34 @@ def Oc(self) -> float: """Cold dark matter density (CDM = matter - baryons), pre-neutrino.""" return self.Om - self.Ob + def check_consistency(self): + """Check that the configuration in terms of bins is internally consistent""" + redshift_distributions = np.loadtxt(self.nz_path) + if redshift_distributions.shape[1] - 1 != self.nbins: + raise ValueError( + f"The number of tomographic bins is inconsistent with the redshift distribution: {redshift_distributions.ndim - 1}." + ) + + if isinstance(self.n_arcmin2, list) and len(self.n_arcmin2) != self.nbins: + raise ValueError( + f"Number of elements in n_arcmin2 {len(self.n_arcmin2)} " + f"does not match number of bins {self.nbins}" + ) + + if isinstance(self.sigma_e, list) and len(self.sigma_e) != self.nbins: + raise ValueError( + f"Number of elements in sigma_e {len(self.sigma_e)} " + f"does not match number of bins {self.nbins}" + ) + + if isinstance(self.bias, list) and len(self.bias) != self.nbins: + raise ValueError( + f"Number of elements in bias {len(self.bias)} " + f"does not match number of bins {self.nbins}" + ) + + print("The number of tomographic bins is consistent across inputs...") + def build_camb_params(config: GlassMockConfig): """Build the CAMB parameters for a mock, with ``As`` rescaled to ``sigma8``. @@ -177,19 +226,21 @@ def build_shells(config: GlassMockConfig, pars): Returns the GLASS shell list (``(z, w, zeff)`` windows). Lazily imports GLASS; only callable where GLASS is installed. """ + import camb import glass - from cosmology import Cosmology + from cosmology.compat.camb import Cosmology - cosmo = Cosmology.from_camb(pars) + results = camb.get_background(pars) + cosmo = Cosmology(results) zb = glass.distance_grid(cosmo, 0.0, config.zmax, dx=config.dx) return glass.linear_windows(zb) -def matter_shell_cls(config: GlassMockConfig, pars, shells): +def matter_shell_cls(config: GlassMockConfig, pars, shells, limber=False): """Matter angular power spectra for the shells, from CAMB via GLASS.""" import glass.ext.camb - return glass.ext.camb.matter_cls(pars, config.lmax, shells) + return glass.ext.camb.matter_cls(pars, config.lmax, shells, limber=limber) def generate_matter_maps(config: GlassMockConfig, pars, shells, cls): @@ -221,9 +272,12 @@ def generate_matter_maps(config: GlassMockConfig, pars, shells, cls): def Cosmology_from_camb(pars): """Thin indirection so the convergence cosmology is built once, lazily.""" - from cosmology import Cosmology + import camb + from cosmology.compat.camb import Cosmology - return Cosmology.from_camb(pars) + results = camb.get_background(pars) + + return Cosmology(results) # --- mask / galaxy-sampling helpers (used by the generation runner) --------- @@ -322,6 +376,55 @@ def create_mask_from_catalogue(nside, path, output, ra_col="RA", dec_col="DEC"): hp.write_map(output, mask, dtype=np.float32, overwrite=True) +# --- shape noise and number density on mock catalogues ---------------------- +def shape_noise(e1, e2, w): + return np.sqrt( + 0.5 + * (np.sum(e1**2 * w**2) / np.sum(w**2) + np.sum(e2**2 * w**2) / np.sum(w**2)) + ) + + +def number_density(w, area): + return np.sum(w) ** 2 / np.sum(w**2) / area + + +def validate_shape_noise(cat, tomo=True, e1_col="e1", e2_col="e2", w_col="w"): + # First compute and print the non-tomographic shape noise + e1 = cat[e1_col] + e2 = cat[e2_col] + w = cat[w_col] + sigma_e = shape_noise(e1, e2, w) + print(f"Non-tomographic shape noise: {sigma_e}") + + if tomo: + # Now compute and print the tomographic shape noise + for b in range(1, cat["TOM_BIN_ID"].max() + 1): + e1_bin = e1[cat["TOM_BIN_ID"] == b] + e2_bin = e2[cat["TOM_BIN_ID"] == b] + w_bin = w[cat["TOM_BIN_ID"] == b] + sigma_e_bin = shape_noise(e1_bin, e2_bin, w_bin) + print(f"Tomographic shape noise (bin {b}): {sigma_e_bin}") + + +def validate_number_density(cat, mask, tomo=True, w_col="w"): + import healpy as hp + + # First compute and print the non-tomographic shape noise + w = cat[w_col] + nside = hp.npix2nside(mask.shape[0]) + area = ( + np.sum(mask) * hp.nside2pixarea(nside, degrees=True) * 60 * 60 + ) # area in arcmin^2 + n_gal = number_density(w, area) + print(f"Non-tomographic number density: {n_gal}") + + if tomo: + for b in range(1, cat["TOM_BIN_ID"].max() + 1): + w_bin = w[cat["TOM_BIN_ID"] == b] + n_gal_bin = number_density(w_bin, area) + print(f"Tomographic number density (bin {b}): {n_gal_bin}") + + # --- two-point statistics on mock catalogues -------------------------------- # Configuration-space treecorr defaults for the GLASS-mock 2PCF. @@ -372,16 +475,19 @@ def compute_two_point_xi(cat, config=None): return gg -def get_n_gal_map(nside, ra, dec): - """Galaxy-count HEALPix map plus the unique-pixel bookkeeping arrays. +def get_n_gal_map(nside, ra, dec, unique_pix=None, idx=None, idx_rep=None): + """Galaxy-count HEALPix map, shape ``(npix,)``. The unweighted twin of the pseudo-Cl ``get_n_gal_map`` primitive: delegates - to it with ``weights=None`` (counts). Imported lazily so this module keeps - resolving in CAMB-only environments without the harmonic stack. + to it with ``weights=None`` (counts), reusing the ``get_pixels`` bookkeeping + when given. Imported lazily so this module keeps resolving in CAMB-only + environments without the harmonic stack. """ from sp_validation.pseudo_cl import get_n_gal_map as _get_n_gal_map - return _get_n_gal_map(nside, ra, dec) + return _get_n_gal_map( + nside, ra, dec, unique_pix=unique_pix, idx=idx, idx_rep=idx_rep + ) def compute_two_point_cl(cat, nside=1024, lmin=8, n_bins=32): @@ -424,7 +530,8 @@ def compute_two_point_cl(cat, nside=1024, lmin=8, n_bins=32): cl_coupled = nmt.compute_coupled_cell(f_all, f_all) cl_all = wsp.decouple_cell(cl_coupled) - cl_coupled = np.concatenate([np.arange(1, lmax + 1)[np.newaxis, :], cl_coupled]) + # compute_coupled_cell runs over ell = 0..b_lmax, index = ell + cl_coupled = np.concatenate([np.arange(lmax)[np.newaxis, :], cl_coupled]) cl_all = np.concatenate([ell_eff[np.newaxis, ...], cl_all]) return cl_coupled, cl_all @@ -448,6 +555,8 @@ def compute_two_point_cl_map(cat, nside=1024, lmin=8, n_bins=32): import healpy as hp import pymaster as nmt + from sp_validation.pseudo_cl import get_pixels + e1 = cat["e1"] e2 = cat["e2"] ra = cat["ra"] @@ -459,7 +568,10 @@ def compute_two_point_cl_map(cat, nside=1024, lmin=8, n_bins=32): b, ell_eff, lmax, b_lmax = _mock_powspace_bin(nside, lmin, n_bins) factor = -1 - n_gal_map, unique_pix, _idx, idx_rep = get_n_gal_map(nside, ra, dec) + unique_pix, idx, idx_rep = get_pixels(ra, dec, nside) + n_gal_map = get_n_gal_map( + nside, ra, dec, unique_pix=unique_pix, idx=idx, idx_rep=idx_rep + ) mask = n_gal_map > 0 shear_map_e1 = np.zeros(hp.nside2npix(nside)) @@ -479,7 +591,8 @@ def compute_two_point_cl_map(cat, nside=1024, lmin=8, n_bins=32): cl_coupled = nmt.compute_coupled_cell(f_all, f_all) cl_all = wsp.decouple_cell(cl_coupled) - cl_coupled = np.concatenate([np.arange(1, lmax + 1)[np.newaxis, :], cl_coupled]) + # compute_coupled_cell runs over ell = 0..b_lmax, index = ell + cl_coupled = np.concatenate([np.arange(lmax)[np.newaxis, :], cl_coupled]) cl_all = np.concatenate([ell_eff[np.newaxis, ...], cl_all]) return cl_coupled, cl_all diff --git a/src/sp_validation/pseudo_cl.py b/src/sp_validation/pseudo_cl.py index 34cbc68d..f52ab1d1 100644 --- a/src/sp_validation/pseudo_cl.py +++ b/src/sp_validation/pseudo_cl.py @@ -16,11 +16,13 @@ import healpy as hp import numpy as np import pymaster as nmt +from cs_util.cosmo import get_theo_c_ell # Lowest multipole retained by the pseudo-Cl estimators. LMIN = 8 +# ---------------------- Binning utility functions ---------------------- def pseudo_cl_geometry(nside): """Return ``(lmin, lmax, b_lmax)`` for the pseudo-Cl estimator at ``nside``. @@ -87,7 +89,38 @@ def make_namaster_bin( return b -def get_n_gal_map(nside, ra, dec, weights=None): +# ---------------------- Map computation utility functions ---------------------- +def get_pixels(ra, dec, nside): + """ + Get the HEALPix pixel indices for given RA and Dec. + + Parameters + ---------- + ra : np.ndarray + Right ascension in degrees. + dec : np.ndarray + Declination in degrees. + nside : int + HEALPix nside parameter. + + Returns + ------- + unique_pix : np.ndarray + Sorted unique pixel indices. + idx : np.ndarray + First-occurrence indices into the input from ``np.unique``. + idx_rep : np.ndarray + Inverse map: pixel-group index for each input object. + """ + pixels = hp.ang2pix(nside, theta=np.radians(90 - dec), phi=np.radians(ra)) + + unique_pix, idx, idx_rep = np.unique(pixels, return_index=True, return_inverse=True) + return unique_pix, idx, idx_rep + + +def get_n_gal_map( + nside, ra, dec, weights=None, unique_pix=None, idx=None, idx_rep=None +): """Weighted galaxy number-density HEALPix map plus pixel bookkeeping. Bins ``(ra, dec)`` (degrees) onto an ``nside`` HEALPix grid. With @@ -98,21 +131,106 @@ def get_n_gal_map(nside, ra, dec, weights=None): ------- n_gal : np.ndarray Map of summed weights (or counts) per pixel, shape ``(npix,)``. - unique_pix : np.ndarray - Sorted unique occupied pixel indices. - idx : np.ndarray - First-occurrence indices into the input from ``np.unique``. - idx_rep : np.ndarray - Inverse map: pixel-group index for each input object. """ - theta = (90.0 - dec) * np.pi / 180.0 - phi = ra * np.pi / 180.0 - pix = hp.ang2pix(nside, theta, phi) + if unique_pix is None or idx is None or idx_rep is None: + unique_pix, idx, idx_rep = get_pixels(ra, dec, nside) - unique_pix, idx, idx_rep = np.unique(pix, return_index=True, return_inverse=True) n_gal = np.zeros(hp.nside2npix(nside)) n_gal[unique_pix] = np.bincount(idx_rep, weights=weights) - return n_gal, unique_pix, idx, idx_rep + return n_gal + + +def get_shear_map( + ra, dec, e1, e2, w, nside, unique_pix=None, idx=None, idx_rep=None, n_gal_map=None +): + """Weighted shear HEALPix maps plus pixel bookkeeping. + + Bins ``(ra, dec)`` (degrees) onto an ``nside`` HEALPix grid. The shear + components ``(e1, e2)`` are weighted by ``w`` and summed per pixel. If + ``unique_pix``, ``idx``, and ``idx_rep`` are provided, they are used to + avoid recomputing the pixel indices. + If ``n_gal_map`` is provided, it is used to normalize the shear maps by the galaxy density. + + Returns + ------- + e1_map : np.ndarray + Weighted sum of E1 per pixel, shape ``(npix,)``. + e2_map : np.ndarray + Weighted sum of E2 per pixel, shape ``(npix,)``. + """ + if unique_pix is None or idx is None or idx_rep is None: + unique_pix, idx, idx_rep = get_pixels(ra, dec, nside) + + if n_gal_map is None: + n_gal_map = get_n_gal_map( + nside, ra, dec, weights=w, unique_pix=unique_pix, idx=idx, idx_rep=idx_rep + ) + + npix = hp.nside2npix(nside) + e1_map = np.zeros(npix) + e2_map = np.zeros(npix) + + e1_map[unique_pix] = np.bincount(idx_rep, weights=e1 * w) + e2_map[unique_pix] = np.bincount(idx_rep, weights=e2 * w) + + non_zero = n_gal_map > 0 + e1_map[non_zero] /= n_gal_map[non_zero] + e2_map[non_zero] /= n_gal_map[non_zero] + + return e1_map, e2_map + + +def get_variance_map( + nside, ra, dec, e1, e2, w, unique_pix=None, idx=None, idx_rep=None +): + """Compute the variance map of the shear components. + + The variance is computed as the weighted variance of the shear components in each pixel. + + Returns + ------- + variance_map : np.ndarray + Variance map of the shear components, shape ``(npix,)``. + """ + if unique_pix is None or idx is None or idx_rep is None: + unique_pix, idx, idx_rep = get_pixels(ra, dec, nside) + + npix = hp.nside2npix(nside) + variance_map = np.zeros(npix) + + variance_map[unique_pix] = np.bincount( + idx_rep, weights=0.5 * (e1**2 + e2**2) * w**2 + ) + + return variance_map + + +def get_noise_bias_analytical( + ra, dec, e1, e2, w, lmax, nside=1024, unique_pix=None, idx=None, idx_rep=None +): + """ + Compute the analytical noise bias for shear power spectrum. + """ + variance_map = get_variance_map( + nside=nside, + ra=ra, + dec=dec, + e1=e1, + e2=e2, + w=w, + unique_pix=unique_pix, + idx=idx, + idx_rep=idx_rep, + ) + + noise_bias = hp.nside2pixarea(nside) * np.mean(variance_map) + + noise_bias_cl = np.zeros((4, lmax)) + + noise_bias_cl[0, :] = noise_bias # EE + noise_bias_cl[3, :] = noise_bias # BB + + return noise_bias_cl def apply_random_rotation(e1, e2, rng=None): @@ -140,13 +258,493 @@ def apply_random_rotation(e1, e2, rng=None): return e1_out, e2_out +def get_noise_realisation( + ra, + dec, + e1, + e2, + w, + nside, + unique_pix=None, + idx=None, + idx_rep=None, + n_gal_map=None, + rng=None, +): + """ + Generate a random noise realisation of the shear maps by applying a random rotation to the ellipticity components. + + Parameters + ---------- + ra, dec : np.ndarray + Right ascension and declination of the sources. + e1, e2 : np.ndarray + Ellipticity components. + w : np.ndarray + Weights of the sources. + nside : int + HEALPix resolution. + unique_pix, idx, idx_rep : np.ndarray, optional + Pixel indices and bookkeeping arrays. If not provided, they will be computed. + n_gal_map : np.ndarray, optional + Galaxy number density map. If not provided, it will be computed. + + Returns + ------- + noise_map_e1, noise_map_e2 : np.ndarray + Noise map for ellipticity components. + """ + # Apply random rotation to the ellipticity components + e1_rot, e2_rot = apply_random_rotation(e1, e2, rng=rng) + + # Compute the noise maps using the rotated ellipticity components + noise_map_e1, noise_map_e2 = get_shear_map( + ra=ra, + dec=dec, + e1=e1_rot, + e2=e2_rot, + w=w, + nside=nside, + unique_pix=unique_pix, + idx=idx, + idx_rep=idx_rep, + n_gal_map=n_gal_map, + ) + + return noise_map_e1, noise_map_e2 + + +def get_noise_bias_from_gaussian_real( + ra, + dec, + e1, + e2, + w, + nside, + nrandom_cell, + binning, + ell_step=10, + n_ell_bins=32, + power=0.5, + unique_pix=None, + idx=None, + idx_rep=None, + n_gal_map=None, + wsp=None, + seed=42, +): + """ + Compute the power spectrum of the noise bias from random realisations. + + Parameters + ---------- + ra, dec : np.ndarray + Right ascension and declination of the sources. + e1, e2 : np.ndarray + Ellipticity components. + w : np.ndarray + Weights of the sources. + nside : int + HEALPix resolution. + nrandom_cell : int + Number of random cells to use for the noise estimation. + binning, ell_step, n_ell_bins, power : str, int, int, float + Binning scheme and parameters. + unique_pix, idx, idx_rep : np.ndarray, optional + Pixel indices and bookkeeping arrays. If not provided, they will be computed. + n_gal_map : np.ndarray, optional + Galaxy number density map. If not provided, it will be computed. + + Returns + ------- + noise_bias_cl : np.ndarray + Power spectrum of the noise bias. + """ + lmin, lmax, b_lmax = pseudo_cl_geometry(nside) + + b = make_namaster_bin( + lmin, + lmax, + b_lmax, + binning, + ell_step=ell_step, + n_ell_bins=n_ell_bins, + power=power, + ) + + ell_eff = b.get_effective_ells() + noise_bias_cl = np.zeros((4, ell_eff.size)) + + rng = np.random.default_rng(seed) + + if unique_pix is None or idx is None or idx_rep is None: + unique_pix, idx, idx_rep = get_pixels(ra, dec, nside) + + if n_gal_map is None: + n_gal_map = get_n_gal_map( + nside, ra, dec, weights=w, unique_pix=unique_pix, idx=idx, idx_rep=idx_rep + ) + + if wsp is None: + _, _, wsp = get_field_and_workspace_from_map(b, mask_a=n_gal_map) + + for _ in range(nrandom_cell): + noise_map_e1, noise_map_e2 = get_noise_realisation( + ra, + dec, + e1, + e2, + w, + nside, + unique_pix=unique_pix, + idx=idx, + idx_rep=idx_rep, + n_gal_map=n_gal_map, + rng=rng, + ) + + noise_map = noise_map_e1 + 1j * noise_map_e2 + del noise_map_e1, noise_map_e2 + + _, cl_noise_, _ = get_pseudo_cls_map( + noise_map, + n_gal_map, + nside, + binning, + ell_step=ell_step, + n_ell_bins=n_ell_bins, + power=power, + wsp=wsp, + ) + + noise_bias_cl += cl_noise_ + + noise_bias_cl /= nrandom_cell + + return noise_bias_cl + + +def get_noise_bias( + ra, + dec, + e1, + e2, + w, + nside, + noise_bias_method, + binning, + *, + ell_step=10, + n_ell_bins=32, + power=0.5, + nrandom_cell=100, + seed=42, +): + """ + Compute the noise bias from object positions and ellipticities. + + Parameters + ---------- + ra, dec : np.ndarray + Right ascension and declination of the sources. + e1, e2 : np.ndarray + Ellipticity components. + w : np.ndarray + Weights of the sources. + nside : int + HEALPix resolution. + noise_bias_method : {'randoms', 'analytic'} + Method used to estimate the noise bias. + binning : {'linear', 'logspace', 'powspace'} + Binning scheme. + ell_step : int, optional + Bin width in ell for ``'linear'`` binning. + n_ell_bins : int, optional + Number of ell bins for ``'logspace'`` / ``'powspace'`` binning. + power : float, optional + Exponent for ``'powspace'`` binning. + nrandom_cell : int, optional + Number of random cells to use for the noise estimation. (only for the `randoms` method) + seed : int, optional + Random seed for reproducibility. (only for the `randoms` method) + + Returns + ------- + noise_bias_cl : np.ndarray + Power spectrum of the noise bias. + """ + if noise_bias_method not in ["randoms", "analytic"]: + raise ValueError("noise_bias_method must be 'randoms' or 'analytic'") + + lmin, lmax, b_lmax = pseudo_cl_geometry(nside) + + b = make_namaster_bin( + lmin, + lmax, + b_lmax, + binning, + ell_step=ell_step, + n_ell_bins=n_ell_bins, + power=power, + ) + + unique_pix, idx, idx_rep = get_pixels(ra, dec, nside) + + if noise_bias_method == "analytic": + noise_bias_cl = get_noise_bias_analytical( + ra, + dec, + e1, + e2, + w, + lmax, + nside, + unique_pix=unique_pix, + idx=idx, + idx_rep=idx_rep, + ) + + elif noise_bias_method == "randoms": + noise_bias_cl = get_noise_bias_from_gaussian_real( + ra, + dec, + e1, + e2, + w, + nside, + nrandom_cell, + binning, + ell_step=ell_step, + n_ell_bins=n_ell_bins, + power=power, + unique_pix=unique_pix, + idx=idx, + idx_rep=idx_rep, + seed=seed, + ) + + noise_bias_cl = b.unbin_cell(noise_bias_cl) + else: + raise ValueError( + f"Invalid noise bias method `{noise_bias_method}`. Must be 'analytic' or 'randoms'." + ) + + return noise_bias_cl + + +# ---------------------- Cl computation functions ---------------------- +def get_field_and_workspace_from_map( + b, + mask_a, + e1_map_a=None, + e2_map_a=None, + mask_b=None, + e1_map_b=None, + e2_map_b=None, + pol_factor=-1, + return_wsp=True, +): + """Compute a NaMaster field and workspace object from the input maps. + + If the shear maps are None, returns field objects but only the workspace objects is relevant and contains the mixing matrix. + If the second mask and shear maps (indexed b) are provided, the mixing matrix is computed between the two fields. + + Parameters + ---------- + b : nmt.NmtBin + NaMaster binning object. + mask_a : np.ndarray + Field mask for the first map. + e1_map_a : np.ndarray, optional + E1 map for the first field. + e2_map_a : np.ndarray, optional + E2 map for the first field. + mask_b : np.ndarray, optional + Field mask for the second map. + e1_map_b : np.ndarray, optional + E1 map for the second field. + e2_map_b : np.ndarray, optional + E2 map for the second field. + pol_factor : float, optional + Polarization factor to apply to the E2 map. + return_wsp : bool, optional + If True, return the NaMaster workspace object containing the mixing matrix. + + Returns + ------- + field_a : nmt.NmtField + NaMaster field object for the first map. + field_b : nmt.NmtField + NaMaster field object for the second map (if provided, same than the first map otherwise). + wsp : nmt.NmtWorkspace + NaMaster workspace object containing the mixing matrix. + + """ + nside = hp.npix2nside(len(mask_a)) + lmax = b.lmax + if e1_map_a is None or e2_map_a is None: + e1_map_a = np.zeros(hp.nside2npix(nside)) + e2_map_a = np.zeros(hp.nside2npix(nside)) + + # Create NaMaster field + field_a = nmt.NmtField( + mask=mask_a, maps=[e1_map_a, pol_factor * e2_map_a], lmax=lmax + ) + + if mask_b is not None: + if e1_map_b is None or e2_map_b is None: + e1_map_b = np.zeros(hp.nside2npix(nside)) + e2_map_b = np.zeros(hp.nside2npix(nside)) + + field_b = nmt.NmtField( + mask=mask_b, maps=[e1_map_b, pol_factor * e2_map_b], lmax=lmax + ) + else: + field_b = field_a + + if return_wsp: + # Create NaMaster workspace + wsp = nmt.NmtWorkspace.from_fields(field_a, field_b, b) + + return field_a, field_b, wsp + else: + return field_a, field_b, None + + +def get_field_and_workspace_from_catalog( + b, + ra_a, + dec_a, + e1_a, + e2_a, + w_a, + ra_b=None, + dec_b=None, + e1_b=None, + e2_b=None, + w_b=None, + pol_factor=-1, + return_wsp=True, + same_bin=False, +): + """Create a NaMaster field and workspace from the input catalog. + + If the second catalog is provided, the mixing matrix is computed between the two fields. + + Parameters + ---------- + b : nmt.NmtBin + NaMaster binning object. + ra_a : np.ndarray + Right ascension of sources in the first catalog. + dec_a : np.ndarray + Declination of sources in the first catalog. + e1_a : np.ndarray + E1 shear component of sources in the first catalog. + e2_a : np.ndarray + E2 shear component of sources in the first catalog. + w_a : np.ndarray + Weights of sources in the first catalog. + ra_b : np.ndarray, optional + Right ascension of sources in the second catalog. + dec_b : np.ndarray, optional + Declination of sources in the second catalog. + e1_b : np.ndarray, optional + E1 shear component of sources in the second catalog. + e2_b : np.ndarray, optional + E2 shear component of sources in the second catalog. + w_b : np.ndarray, optional + Weights of sources in the second catalog. + pol_factor : float, optional + Polarization factor to apply to the E2 component. + return_wsp : bool, optional + If True, return the NaMaster workspace object containing the mixing matrix. + + Returns + ------- + field_a : nmt.NmtFieldCatalog + NaMaster field object for the first catalog. + field_b : nmt.NmtFieldCatalog + NaMaster field object for the second catalog (if provided, same as the first catalog otherwise). + wsp : nmt.NmtWorkspace + NaMaster workspace object containing the mixing matrix. + + """ + lmax = b.lmax + # Get field for input catalog a + field_a = nmt.NmtFieldCatalog( + positions=[ra_a, dec_a], + weights=w_a, + field=[e1_a, pol_factor * e2_a], + lmax=lmax, + lmax_mask=lmax, + spin=2, + lonlat=True, + ) + + if ( + ra_b is not None + and dec_b is not None + and e1_b is not None + and e2_b is not None + and w_b is not None + and not same_bin + ): + field_b = nmt.NmtFieldCatalog( + positions=[ra_b, dec_b], + weights=w_b, + field=[e1_b, pol_factor * e2_b], + lmax=lmax, + lmax_mask=lmax, + spin=2, + lonlat=True, + ) + else: + field_b = field_a + + if return_wsp: + wsp = nmt.NmtWorkspace.from_fields(field_a, field_b, b) + return field_a, field_b, wsp + else: + return field_a, field_b, None + + +def compute_cl_from_field_and_workspace(field_a, field_b, wsp, b): + """Compute the angular power spectrum from the input NaMaster field and workspace + + Parameters + ---------- + field_a : nmt.NmtField + NaMaster field object for the first catalog. + field_b : nmt.NmtField + NaMaster field object for the second catalog. + wsp : nmt.NmtWorkspace + NaMaster workspace object containing the mixing matrix. + b : nmt.NmtBin + NaMaster binning object. + + Returns + ------- + cl_coupled : np.ndarray + Coupled angular power spectrum. + cl_decoupled : np.ndarray + Decoupled angular power spectrum. + """ + cl_coupled = nmt.compute_coupled_cell(field_a, field_b) + cl_decoupled = wsp.decouple_cell(cl_coupled) + + return cl_coupled, cl_decoupled + + def get_pseudo_cls_map( - shear_map, - mask, + shear_map_a, + mask_a, nside, binning, *, - pol_factor=True, + shear_map_b=None, + mask_b=None, + pol_factor=-1, wsp=None, ell_step=10, n_ell_bins=32, @@ -156,16 +754,20 @@ def get_pseudo_cls_map( Parameters ---------- - shear_map : np.ndarray + shear_map_a : np.ndarray Complex shear map (``e1 + 1j * e2``). - mask : np.ndarray + mask_a : np.ndarray Field mask (the galaxy number-density map). nside : int HEALPix resolution; fixes the harmonic geometry. binning : str Binning scheme passed to :func:`make_namaster_bin`. - pol_factor : bool, optional - If ``True`` flip the sign of the imaginary (e2) component. + shear_map_b : np.ndarray, optional + Complex shear map for the second field (``e1 + 1j * e2``). + mask_b : np.ndarray, optional + Field mask for the second field (the galaxy number-density map). + pol_factor : float, optional + Polarization factor to apply to the E2 component. wsp : nmt.NmtWorkspace, optional Reuse a coupling workspace; built from the field if ``None``. ell_step, n_ell_bins, power : optional @@ -180,6 +782,27 @@ def get_pseudo_cls_map( wsp : nmt.NmtWorkspace The coupling workspace (newly built or the one passed in). """ + # First do some assertion checks + if shear_map_b is not None: + assert mask_b is not None, "mask_b must be provided if shear_map_b is provided" + assert shear_map_a.shape == shear_map_b.shape, ( + "shear_map_a and shear_map_b must have the same shape" + ) + assert mask_a.shape == mask_b.shape, ( + "mask_a and mask_b must have the same shape" + ) + + if mask_b is not None: + assert shear_map_b is not None, ( + "shear_map_b must be provided if mask_b is provided" + ) + assert shear_map_a.shape == shear_map_b.shape, ( + "shear_map_a and shear_map_b must have the same shape" + ) + assert mask_a.shape == mask_b.shape, ( + "mask_a and mask_b must have the same shape" + ) + lmin, lmax, b_lmax = pseudo_cl_geometry(nside) b = make_namaster_bin( @@ -193,18 +816,36 @@ def get_pseudo_cls_map( ) ell_eff = b.get_effective_ells() - factor = -1 if pol_factor else 1 - - f_all = nmt.NmtField( - mask=mask, maps=[shear_map.real, factor * shear_map.imag], lmax=b_lmax - ) if wsp is None: - wsp = nmt.NmtWorkspace.from_fields(f_all, f_all, b) + field_a, field_b, wsp = get_field_and_workspace_from_map( + b, + mask_a, + e1_map_a=shear_map_a.real, + e2_map_a=shear_map_a.imag, + mask_b=mask_b, + e1_map_b=shear_map_b.real if shear_map_b is not None else None, + e2_map_b=shear_map_b.imag if shear_map_b is not None else None, + pol_factor=pol_factor, + return_wsp=True, + ) + else: + field_a, field_b, _ = get_field_and_workspace_from_map( + b, + mask_a, + e1_map_a=shear_map_a.real, + e2_map_a=shear_map_a.imag, + mask_b=mask_b, + e1_map_b=shear_map_b.real if shear_map_b is not None else None, + e2_map_b=shear_map_b.imag if shear_map_b is not None else None, + pol_factor=pol_factor, + return_wsp=False, + ) - cl_coupled = nmt.compute_coupled_cell(f_all, f_all) - cl_all = wsp.decouple_cell(cl_coupled) + cl_coupled, cl_decoupled = compute_cl_from_field_and_workspace( + field_a, field_b, wsp, b + ) - return ell_eff, cl_all, wsp + return ell_eff, cl_decoupled, wsp def get_pseudo_cls_catalog( @@ -213,7 +854,9 @@ def get_pseudo_cls_catalog( nside, binning, *, - pol_factor=True, + tomo_bin_a="all", + tomo_bin_b="all", + pol_factor=-1, wsp=None, ell_step=10, n_ell_bins=32, @@ -232,8 +875,10 @@ def get_pseudo_cls_catalog( HEALPix resolution; fixes the harmonic geometry. binning : str Binning scheme passed to :func:`make_namaster_bin`. - pol_factor : bool, optional - If ``True`` flip the sign of the e2 component. + tomo_bin_a, tomo_bin_b : str or int or None, optional + Tomographic bin IDs for the two fields. + pol_factor : int, optional + Polarization factor to apply to the E2 component. wsp : nmt.NmtWorkspace, optional Reuse a coupling workspace; built from the field if ``None``. ell_step, n_ell_bins, power : optional @@ -248,6 +893,11 @@ def get_pseudo_cls_catalog( wsp : nmt.NmtWorkspace The coupling workspace (newly built or the one passed in). """ + # First make some assertion checks reagarding the run mode + assert (tomo_bin_a == "all" and tomo_bin_b == "all") or ( + tomo_bin_a != "all" and tomo_bin_b != "all" + ), "Both tomo_bin_a and tomo_bin_b must be provided or both must be 'all'" + lmin, lmax, b_lmax = pseudo_cl_geometry(nside) b = make_namaster_bin( @@ -261,34 +911,156 @@ def get_pseudo_cls_catalog( ) ell_eff = b.get_effective_ells() - factor = -1 if pol_factor else 1 + is_tomography = tomo_bin_a != "all" and tomo_bin_b != "all" + if is_tomography: + assert params["tomo_bin_col"] is not None, ( + "The column of tomographic bin ids is not specified." + ) + mask_tomo_a = catalog[params["tomo_bin_col"]] == tomo_bin_a + mask_tomo_b = catalog[params["tomo_bin_col"]] == tomo_bin_b + catalog_a = catalog[mask_tomo_a] + catalog_b = catalog[mask_tomo_b] + same_bin = tomo_bin_a == tomo_bin_b + else: + catalog_a = catalog + catalog_b = catalog + same_bin = True - f_all = nmt.NmtFieldCatalog( - positions=[catalog[params["ra_col"]], catalog[params["dec_col"]]], - weights=catalog[params["w_col"]], - field=[catalog[params["e1_col"]], factor * catalog[params["e2_col"]]], - lmax=b_lmax, - lmax_mask=b_lmax, - spin=2, - lonlat=True, + if wsp is None: + field_a, field_b, wsp = get_field_and_workspace_from_catalog( + b, + ra_a=catalog_a[params["ra_col"]], + dec_a=catalog_a[params["dec_col"]], + e1_a=catalog_a[params["e1_col"]], + e2_a=catalog_a[params["e2_col"]], + w_a=catalog_a[params["w_col"]], + ra_b=catalog_b[params["ra_col"]], + dec_b=catalog_b[params["dec_col"]], + e1_b=catalog_b[params["e1_col"]], + e2_b=catalog_b[params["e2_col"]], + w_b=catalog_b[params["w_col"]], + pol_factor=pol_factor, + return_wsp=True, + same_bin=same_bin, + ) + else: + field_a, field_b, _ = get_field_and_workspace_from_catalog( + b, + ra_a=catalog_a[params["ra_col"]], + dec_a=catalog_a[params["dec_col"]], + e1_a=catalog_a[params["e1_col"]], + e2_a=catalog_a[params["e2_col"]], + w_a=catalog_a[params["w_col"]], + ra_b=catalog_b[params["ra_col"]], + dec_b=catalog_b[params["dec_col"]], + e1_b=catalog_b[params["e1_col"]], + e2_b=catalog_b[params["e2_col"]], + w_b=catalog_b[params["w_col"]], + pol_factor=pol_factor, + return_wsp=False, + same_bin=same_bin, + ) + + cl_coupled, cl_decoupled = compute_cl_from_field_and_workspace( + field_a, field_b, wsp, b ) - if wsp is None: - wsp = nmt.NmtWorkspace.from_fields(f_all, f_all, b) + return ell_eff, cl_decoupled, wsp - cl_coupled = nmt.compute_coupled_cell(f_all, f_all) - cl_all = wsp.decouple_cell(cl_coupled) - return ell_eff, cl_all, wsp +# ---------------------- Covariance computation functions ---------------------- +def get_fiducial_cl(z, dndz, lmax, cosmo, backend="camb"): + """ + Get the fiducial Cl's using the redshift distribution. + Cosmology is determined by the input cosmo object. + """ + ell = np.arange(1, lmax + 1) + + fiducial_cl = get_theo_c_ell(ell=ell, z=z, nz=dndz, backend=backend, cosmo=cosmo) + + return fiducial_cl + + +def get_pseudo_cl_iNKA_covariance( + input_cl_a1_b1, + input_cl_a1_b2, + input_cl_a2_b1, + input_cl_a2_b2, + field_a1, + field_a2, + field_b1, + field_b2, + wsp_a, + wsp_b, + b, +): + """Compute the iNKA covariance for pseudo-Cl. + + Parameters + ---------- + input_cl_a1_b1 : np.ndarray + Input Cl for field a1 and b1. + input_cl_a1_b2 : np.ndarray + Input Cl for field a1 and b2. + input_cl_a2_b1 : np.ndarray + Input Cl for field a2 and b1. + input_cl_a2_b2 : np.ndarray + Input Cl for field a2 and b2. + field_a1 : nmt.NmtField + NaMaster field object for the first catalog (a1). + field_a2 : nmt.NmtField + NaMaster field object for the second catalog (a2). + field_b1 : nmt.NmtField + NaMaster field object for the first catalog (b1). + field_b2 : nmt.NmtField + NaMaster field object for the second catalog (b2). + wsp_a : nmt.NmtWorkspace + NaMaster workspace object containing the mixing matrix for fields a. + wsp_b : nmt.NmtWorkspace + NaMaster workspace object containing the mixing matrix for fields b. + b : nmt.NmtBin + NaMaster binning object. + + Returns + ------- + cov_matrix : np.ndarray + Covariance matrix of the pseudo-Cl, shape ``(n_bins, n_bins)``. + """ + # Compute the coupling coefficients for the covariance + cw = nmt.NmtCovarianceWorkspace.from_fields(field_a1, field_a2, field_b1, field_b2) + + # Get actual number of ell bins from binning scheme + n_ell_actual = b.get_n_bands() + + # Compute the covariance using NaMaster's built-in function + cov_matrix = nmt.gaussian_covariance( + cw, + 2, + 2, + 2, + 2, + input_cl_a1_b1, + input_cl_a1_b2, + input_cl_a2_b1, + input_cl_a2_b2, + wsp_a, + wb=wsp_b, + ).reshape([n_ell_actual, 4, n_ell_actual, 4]) + + return cov_matrix # NaMaster spin-2 × spin-2 spectrum order: EE, EB, BE, BB. _NMT_EE = 0 -def bandpower_window_from_workspace(wsp): +def bandpower_window_from_workspace(wsp, nside=None): """Extract the bandpower window matrix ``W`` for a spin-2×spin-2 workspace. + For HEALPix-map spectra, supply ``nside`` to include the pixel window + ``pw²(ℓ)`` in the measured spectrum. Catalogue-based spectra have no pixel + window and must leave ``nside`` unset. + NaMaster's ``get_bandpower_windows()`` returns a four-index array ``(n_cl_out, n_bpw, n_cl_in, n_ell)`` describing how each output bandpower is built from the input multipoles across the EE/EB/BE/BB spectra. SACC's @@ -303,9 +1075,13 @@ def bandpower_window_from_workspace(wsp): ``compute_coupled_cell``. window_weights : np.ndarray ``W`` of shape ``(n_ell, n_bpw)`` — one column per bandpower, the layout - :func:`sp_validation.sacc_io.add_pseudo_cl` expects. + :func:`sp_validation.sacc_io.add_pseudo_cl` expects. If ``nside`` is + given, this includes ``pw²(ℓ)`` on the window's multipole grid. """ bpw = wsp.get_bandpower_windows() # (n_cl_out, n_bpw, n_cl_in, n_ell) diagonal = bpw[_NMT_EE, :, _NMT_EE, :] # (n_bpw, n_ell) window_ells = np.arange(diagonal.shape[1], dtype=float) + if nside is not None: + pixwin = hp.pixwin(nside, lmax=diagonal.shape[1] - 1) + diagonal = diagonal * pixwin**2 return window_ells, diagonal.T diff --git a/src/sp_validation/rho_tau.py b/src/sp_validation/rho_tau.py index b397b6de..e744fdf8 100644 --- a/src/sp_validation/rho_tau.py +++ b/src/sp_validation/rho_tau.py @@ -1,3 +1,4 @@ +import gc import os import time from pathlib import Path @@ -13,6 +14,7 @@ # SquareRootScale lives in sp_validation.plots; re-exported here so that # `from sp_validation.rho_tau import SquareRootScale` keeps working. from sp_validation.plots import SquareRootScale # noqa: F401 +from sp_validation.statistics import jackknife_patch_centers def _extract_xip(correlations): @@ -116,7 +118,7 @@ def __call__(self, block): return self._cache[block] -def get_params_rho_tau(cat, survey="other"): +def get_params_rho_tau(cat): """Rho/tau parameters for one catalogue-config entry ``cat``. The jackknife patch count is the entry's ``patch_number``; a missing key @@ -131,9 +133,8 @@ def get_params_rho_tau(cat, survey="other"): params["e2_star_col"] = cat["psf"]["e2_star_col"] params["PSF_size"] = cat["psf"]["PSF_size"] params["star_size"] = cat["psf"]["star_size"] - if survey != "DES": - params["PSF_flag"] = cat["psf"]["PSF_flag"] - params["star_flag"] = cat["psf"]["star_flag"] + params["PSF_flag"] = cat["psf"].get("PSF_flag") + params["star_flag"] = cat["psf"].get("star_flag") params["ra_units"] = "deg" params["dec_units"] = "deg" @@ -142,6 +143,7 @@ def get_params_rho_tau(cat, survey="other"): params["w_col"] = cat["shear"]["w_col"] params["e1_col"] = cat["shear"]["e1_col"] params["e2_col"] = cat["shear"]["e2_col"] + params["tomo_bin_col"] = cat["shear"].get("tomo_bin_col") params["R11"] = cat["shear"].get("R11") params["R22"] = cat["shear"].get("R22") @@ -153,43 +155,59 @@ def get_rho_tau_w_cov( version, treecorr_config, outdir, - base, + base_rho, + base_tau, method, + mask_star=None, + mask_gal=None, cov_rho=False, - npatch=None, + ncov=100, + **kwargs, ): """Compute rho/tau statistics and, if requested, their covariance.""" if method == "th": - nbin_ang, nbin_rad = 100, 200 + nbin_ang, nbin_rad = kwargs.get("nbin_ang", 100), kwargs.get("nbin_rad", 200) + compute_minus = kwargs.get("compute_minus", True) rho_stat_handler, tau_stat_handler = get_rho_tau( config, version, treecorr_config, outdir, - base, + base_rho, + base_tau, cov_rho=cov_rho, + mask_star=mask_star, + mask_gal=mask_gal, ) get_theory_cov( config, version, treecorr_config, outdir, - base, + base_tau, nbin_ang=nbin_ang, nbin_rad=nbin_rad, + compute_minus=compute_minus, + mask_star=mask_star, + mask_gal=mask_gal, ) return rho_stat_handler, tau_stat_handler elif method == "jk": + npatch = kwargs.get("npatch", 100) return get_jackknife_cov( config, version, treecorr_config, outdir, - base, + base_rho, + base_tau, npatch=npatch, + ncov=ncov, + mask_star=mask_star, + mask_gal=mask_gal, ) elif method == "sim": - tau_cov_path = Path(outdir) / f"cov_tau_{base}_th.npy" + tau_cov_path = Path(outdir) / f"cov_tau_{base_tau}_th.npy" if tau_cov_path.exists(): print(f"Found existing covariance at {tau_cov_path}") @@ -199,8 +217,11 @@ def get_rho_tau_w_cov( version, treecorr_config, outdir, - base, + base_rho, + base_tau, cov_rho=cov_rho, + mask_star=mask_star, + mask_gal=mask_gal, ) else: raise ValueError( @@ -215,9 +236,12 @@ def get_rho_tau( version, treecorr_config, outdir, - base, + base_rho, + base_tau, + mask_star=None, + mask_gal=None, cov_rho=False, - npatch=None, + force_run=False, ): """ Compute rho and tau statistics for a given version of the catalogue. @@ -232,16 +256,30 @@ def get_rho_tau( TreeCorr configuration (must include 'min_sep', 'max_sep', and 'nbins'). outdir : str Output directory. + base_rho : str + Base name for the rho output files. + base_tau : str + Base name for the tau output files. + mask_star : array-like, optional + Boolean selection over the rows of the ``psf`` entry, in the order + ``io.open_entry`` reads them. If None, no masking is applied. + mask_gal : array-like, optional + Boolean selection over the rows of the ``shear`` entry, in the order + ``io.open_entry`` reads them. If None, no masking is applied. + cov_rho : bool, optional + If True, compute the covariance of rho statistics. + force_run : bool, optional + If True, force the computation even if output files already exist. """ - params = get_params_rho_tau(config[version], survey=version) + params = get_params_rho_tau(config[version]) print("Compute Rho and Tau statistics for the version: ", version) start_time = time.time() outdir_path = Path(outdir) - rho_path = outdir_path / f"rho_stats_{base}.fits" - catalog_id = f"{base}_jk" if cov_rho else base + rho_path = outdir_path / f"rho_stats_{base_rho}.fits" + catalog_id = f"{base_rho}_jk" if cov_rho else base_rho cov_rho_path = outdir_path / f"cov_rho_{catalog_id}.npy" if cov_rho else None rho_stat_handler = RhoStat( @@ -251,12 +289,12 @@ def get_rho_tau( with _CatalogueLoader(config[version], params) as load: rho_stats_exists = rho_path.exists() cov_exists = True if not cov_rho else cov_rho_path.exists() - need_compute = (not rho_stats_exists) or (not cov_exists) + need_compute = (not rho_stats_exists) or (not cov_exists) or force_run if need_compute: rho_stat_handler.catalogs.set_params(params, outdir) rho_stat_handler.build_cat_to_compute_rho( - load("psf"), catalog_id=catalog_id + load("psf"), catalog_id=catalog_id, mask=mask_star ) rho_stat_handler.compute_rho_stats( @@ -273,7 +311,7 @@ def get_rho_tau( ) rho_stat_handler.load_rho_stats(rho_path.name) - tau_path = outdir_path / f"tau_stats_{base}.fits" + tau_path = outdir_path / f"tau_stats_{base_tau}.fits" tau_stat_handler = TauStat( catalogs=rho_stat_handler.catalogs, output=outdir, @@ -281,7 +319,7 @@ def get_rho_tau( verbose=True, ) - if tau_path.exists(): + if tau_path.exists() and not force_run: print( f"Skipping tau statistics computation, file {tau_path} already exists." ) @@ -292,12 +330,12 @@ def get_rho_tau( # Build the different catalogs if necessary if f"psf_{version}" not in tau_stat_handler.catalogs.catalogs_dict: tau_stat_handler.build_cat_to_compute_tau( - load("psf"), cat_type="psf", catalog_id=version + load("psf"), cat_type="psf", catalog_id=version, mask=mask_star ) # Build the catalog of galaxies. PSF was computed above tau_stat_handler.build_cat_to_compute_tau( - load("shear"), cat_type="gal", catalog_id=version + load("shear"), cat_type="gal", catalog_id=version, mask=mask_gal ) tau_stat_handler.compute_tau_stats(version, tau_path.name, var_method=None) @@ -314,17 +352,21 @@ def get_theory_cov( base, nbin_ang=100, nbin_rad=100, + compute_minus=True, + mask_star=None, + mask_gal=None, ): """ Compute an analytical estimate of the covariance matrix of rho and tau-statistics. + + ``CovTauTh`` takes the survey area and effective galaxy density from the + galaxy catalogue after ``mask_gal``. The masks select rows as in + ``get_rho_tau``. """ - params = get_params_rho_tau(config[version], survey=version) + params = get_params_rho_tau(config[version]) info = config[version] - A = info["cov_th"]["A"] * 60 * 60 - n_e = info["cov_th"]["n_e"] - n_psf = info["cov_th"]["n_psf"] target_cov = Path(outdir) / f"cov_tau_{base}_th.npy" @@ -341,21 +383,23 @@ def get_theory_cov( path_psf=load("psf"), hdu_psf=1, treecorr_config=treecorr_config, - A=A, - n_e=n_e, - n_psf=n_psf, params=params, + mask_star=mask_star, + mask_gal=mask_gal, ) elapsed = time.time() - start_time print(f"--- Rho/tau statistics for covariance computed in {elapsed:.2f}s ---") - cov = cov_tau_th.build_cov(nbin_ang=nbin_ang, nbin_rad=nbin_rad) + cov = cov_tau_th.build_cov( + nbin_ang=nbin_ang, nbin_rad=nbin_rad, compute_minus=compute_minus + ) print(f"--- Covariance matrix assembled in {time.time() - start_time:.2f}s ---") target_cov.parent.mkdir(parents=True, exist_ok=True) np.save(target_cov, cov) print("Saved covariance matrix of version: ", version) del cov_tau_th + gc.collect() return @@ -364,20 +408,24 @@ def get_jackknife_cov( version, treecorr_config, outdir, - base, + base_rho, + base_tau, npatch, ncov=100, + mask_star=None, + mask_gal=None, + force_run=False, ): """ Compute the covariance matrix of rho and tau-statistics using the jackknife method. Also compute rho and tau-statistics. """ + # TODO: Reorganise this to avoid recomputing unnecessarily the rho-stats multiple times. + rho_filename = f"rho_stats_{base_rho}.fits" + tau_filename = f"tau_stats_{base_tau}.fits" + tau_cov_path = Path(outdir) / f"cov_tau_{base_tau}_jk.npy" - rho_filename = f"rho_stats_{base}.fits" - tau_filename = f"tau_stats_{base}.fits" - tau_cov_path = Path(outdir) / f"cov_tau_{base}_jk.npy" - - if tau_cov_path.exists(): + if tau_cov_path.exists() and not force_run: print(f"Skipping covariance computation, file {tau_cov_path} already exists.") rho_stat_handler = RhoStat( output=outdir, treecorr_config=treecorr_config, verbose=False @@ -392,13 +440,13 @@ def get_jackknife_cov( rho_path = Path(outdir) / rho_filename tau_path = Path(outdir) / tau_filename - if rho_path.exists(): + if rho_path.exists() and not force_run: rho_stat_handler.load_rho_stats(rho_path.name) - if tau_path.exists(): + if tau_path.exists() and not force_run: tau_stat_handler.load_tau_stats(tau_path.name) return rho_stat_handler, tau_stat_handler - params = get_params_rho_tau(config[version], survey=version) + params = get_params_rho_tau(config[version]) rho_stat_handler = RhoStat( output=outdir, treecorr_config=treecorr_config, verbose=False @@ -415,52 +463,66 @@ def get_jackknife_cov( tau_stat_handler.catalogs.set_params(params, outdir) + # shear_psf_leakage keys its catalogues by catalog_id and writes each + # draw's covariance to cov_{rho,tau}_{catalog_id}.npy; base_tau names the + # draws so that the bins of a tomographic run never share them. + def catalog_id(i): + return f"{base_tau}{i}" + + def chunks(i): + return [ + os.path.join(outdir, f"cov_{kind}_{catalog_id(i)}.npy") + for kind in ("rho", "tau") + ] + with _CatalogueLoader(config[version], params) as load: for i in range(ncov): - tau_chunk = outdir + f"/cov_tau_{version}{i}.npy" - rho_chunk = outdir + f"/cov_rho_{version}{i}.npy" - if not (os.path.exists(tau_chunk) and os.path.exists(rho_chunk)): - print( - f"Computing rho-statistics for {version} (jackknife realisation {i + 1}/{ncov})" - ) + if not all(os.path.exists(chunk) for chunk in chunks(i)): + print(f"Computing rho-statistics for {version} (patch {i + 1}/{ncov})") - if f"psf_{version}{i}" not in rho_stat_handler.catalogs.catalogs_dict: + if ( + f"psf_{catalog_id(i)}" + not in rho_stat_handler.catalogs.catalogs_dict + ): # Build catalogues rho_stat_handler.build_cat_to_compute_rho( - load("psf"), catalog_id=version + str(i) + load("psf"), catalog_id=catalog_id(i), mask=mask_star ) tau_stat_handler.catalogs.catalogs_dict = ( rho_stat_handler.catalogs.catalogs_dict ) - # Build the catalog of galaxies. PSF was computed above - tau_stat_handler.build_cat_to_compute_tau( - load("shear"), - cat_type="gal", - catalog_id=version + str(i), + catalogs = rho_stat_handler.catalogs + centers = jackknife_patch_centers( + catalogs.catalogs_dict[f"psf_{catalog_id(i)}"], + catalogs._params["patch_number"], + seed=i, + init="kmeans++", + ) + # Fresh catalogues keep TreeCorr's cached patches consistent with + # this draw's shared star/galaxy layout. + stars = catalogs.read_shear_cat( + path_gal=None, path_psf=load("psf"), hdu=1 + ) + for cat_type in ("psf", "psf_error", "psf_size_error"): + catalogs.build_catalog( + cat=stars, + cat_type=cat_type, + key=f"{cat_type}_{catalog_id(i)}", + patch_centers=centers, + mask=mask_star, ) - - else: - print(f"Computing the patch centers for patch {i + 1}/{ncov}") - - npatch = rho_stat_handler.catalogs._params["patch_number"] - field = rho_stat_handler.catalogs.catalogs_dict[ - f"psf_{version}{i}" - ].getNField(max_top=int.bit_length(npatch) - 1, coords="spherical") - patch, centers = field.run_kmeans(npatch) - - # Update the patch centers of the catalogs - for key, cat in rho_stat_handler.catalogs.catalogs_dict.items(): - cat._centers = centers - field = cat.getNField( - max_top=int.bit_length(npatch) - 1, coords="spherical" - ) - cat._patch = field.kmeans_assign_patches(centers) + tau_stat_handler.build_cat_to_compute_tau( + load("shear"), + cat_type="gal", + catalog_id=catalog_id(i), + mask=mask_gal, + ) # Compute and save rho stats rho_stat_handler.compute_rho_stats( - version + str(i), + catalog_id(i), rho_filename, save_cov=True, func=_extract_xip, @@ -469,7 +531,7 @@ def get_jackknife_cov( # function to extract the tau_+ tau_stat_handler.compute_tau_stats( - version + str(i), + catalog_id(i), tau_filename, save_cov=True, func=_extract_xip, @@ -479,22 +541,21 @@ def get_jackknife_cov( # Update the keys in the dictionaries rho_dict = rho_stat_handler.catalogs.catalogs_dict tau_dict = tau_stat_handler.catalogs.catalogs_dict - rho_dict[f"psf_{version}{i + 1}"] = rho_dict.pop(f"psf_{version}{i}") - rho_dict[f"psf_error_{version}{i + 1}"] = rho_dict.pop( - f"psf_error_{version}{i}" - ) - rho_dict[f"psf_size_error_{version}{i + 1}"] = rho_dict.pop( - f"psf_size_error_{version}{i}" + for prefix in ("psf", "psf_error", "psf_size_error"): + rho_dict[f"{prefix}_{catalog_id(i + 1)}"] = rho_dict.pop( + f"{prefix}_{catalog_id(i)}" + ) + tau_dict[f"gal_{catalog_id(i + 1)}"] = tau_dict.pop( + f"gal_{catalog_id(i)}" ) - tau_dict[f"gal_{version}{i + 1}"] = tau_dict.pop(f"gal_{version}{i}") - cov_tau_loc = np.zeros_like(np.load(outdir + f"/cov_tau_{version}0.npy")) - cov_rho_loc = np.zeros_like(np.load(outdir + f"/cov_rho_{version}0.npy")) + cov_rho_loc, cov_tau_loc = (np.zeros_like(np.load(c)) for c in chunks(0)) for i in range(ncov): - cov_tau_loc += np.load(outdir + f"/cov_tau_{version}{i}.npy") - cov_rho_loc += np.load(outdir + f"/cov_rho_{version}{i}.npy") - os.remove(outdir + f"/cov_tau_{version}{i}.npy") - os.remove(outdir + f"/cov_rho_{version}{i}.npy") + rho_chunk, tau_chunk = chunks(i) + cov_rho_loc += np.load(rho_chunk) + cov_tau_loc += np.load(tau_chunk) + os.remove(rho_chunk) + os.remove(tau_chunk) cov_tau = cov_tau_loc / ncov cov_rho = cov_rho_loc / ncov @@ -502,7 +563,7 @@ def get_jackknife_cov( tau_cov_path.parent.mkdir(parents=True, exist_ok=True) np.save(tau_cov_path, cov_tau) - cov_rho_path = Path(outdir) / f"cov_rho_{base}_jk.npy" + cov_rho_path = Path(outdir) / f"cov_rho_{base_rho}_jk.npy" cov_rho_path.parent.mkdir(parents=True, exist_ok=True) np.save(cov_rho_path, cov_rho) @@ -511,11 +572,13 @@ def get_jackknife_cov( def get_samples( psf_fitter, - version, - base, + base_rho, + base_tau, cov_type="jk", apply_debias=None, sampler="emcee", + nsamples=10000, + nwalkers=124, ): """Return (alpha, beta, eta) samples using ``emcee`` or least squares. @@ -523,10 +586,10 @@ def get_samples( ---------- psf_fitter : PSFFitter PSF fitter instance that provides ``load_*`` helpers. - version : str - Catalog identifier whose rho/tau statistics are sampled. - base : str - Precomputed basename (e.g. ``SP_v1.4_minsep=…``) used for filenames. + base_rho : str + Precomputed basename (e.g. ``SP_v1.4_minsep=…``) used for rho-stat filenames. + base_tau : str + Precomputed basename (e.g. ``SP_v1.4_minsep=…``) used for tau-stat filenames. cov_type : str, optional Covariance label (``'jk'``, ``'th'``, or ``'sim'``). Defaults to ``'jk'``. apply_debias : int or None, optional @@ -534,22 +597,29 @@ def get_samples( sampler : str, optional ``'emcee'`` for MCMC sampling, ``'lsq'`` for least squares (default ``'emcee'``). + nsamples : int, optional + Number of samples to draw (default ``10000``). + nwalkers : int, optional + Number of walkers for the MCMC run (default ``124``). """ if sampler == "emcee": return get_samples_emcee( psf_fitter, - version, - base, + base_rho, + base_tau, cov_type=cov_type, apply_debias=apply_debias, + nsamples=nsamples, + nwalkers=nwalkers, ) elif sampler == "lsq": return get_samples_lsq( psf_fitter, - version, - base, + base_rho, + base_tau, cov_type=cov_type, apply_debias=apply_debias, + nsamples=nsamples, ) else: raise ValueError("Sampler must be either 'emcee' or 'lsq'.") @@ -557,8 +627,8 @@ def get_samples( def get_samples_emcee( psf_fitter, - version, - base, + base_rho, + base_tau, nwalkers=124, nsamples=10000, cov_type="jk", @@ -570,10 +640,10 @@ def get_samples_emcee( ---------- psf_fitter : PSFFitter PSF fitter instance managing rho/tau statistics and covariances. - version : str - Catalog identifier whose rho/tau statistics are sampled. - base : str - Precomputed basename for locating statistics/covariance files. + base_rho : str + Precomputed basename for locating rho statistics/covariance files. + base_tau : str + Precomputed basename for locating tau statistics/covariance files. nwalkers : int, optional Number of walkers for the MCMC run (default ``124``). nsamples : int, optional @@ -584,20 +654,21 @@ def get_samples_emcee( Jackknife patch count applied during debiasing. Disabled when ``None``. """ # Load rho and tau stats - psf_fitter.load_rho_stat(f"rho_stats_{base}.fits") - psf_fitter.load_tau_stat(f"tau_stats_{base}.fits") + psf_fitter.load_rho_stat(f"rho_stats_{base_rho}.fits") + psf_fitter.load_tau_stat(f"tau_stats_{base_tau}.fits") # Check if the path exists (use base for cache key to account for different TreeCorr configs) - sample_file_path = psf_fitter.get_sample_file_path(base) + base_sample = f"{base_tau}_sampler_emcee_cov_tau_type_{cov_type}" + sample_file_path = psf_fitter.get_sample_path(base_sample) if os.path.exists(sample_file_path): print(f"Skipping sampling; {sample_file_path} exists.") - flat_samples = psf_fitter.load_samples(base) - mcmc_result, q = psf_fitter.get_mcmc_from_samples(base) + flat_samples = psf_fitter.load_samples(base_sample) + mcmc_result, q = psf_fitter.get_mcmc_from_samples(flat_samples) print(mcmc_result) # Or run MCMC else: print("MCMC sampling") - cov_filename = f"cov_tau_{base}_{cov_type}.npy" + cov_filename = f"cov_tau_{base_tau}_{cov_type}.npy" psf_fitter.load_covariance(cov_filename, cov_type="tau") debias_npatch = apply_debias if (apply_debias is not None) else None @@ -606,16 +677,17 @@ def get_samples_emcee( nsamples=nsamples, npatch=debias_npatch, apply_debias=debias_npatch is not None, - savefig="mcmc_samples_" + version + ".png", + savefig="mcmc_samples_" + base_tau + ".png", ) - psf_fitter.save_samples(flat_samples, base) + psf_fitter.save_samples(flat_samples, base_sample) return flat_samples, mcmc_result, q def get_samples_lsq( psf_fitter, - version, - base, + base_rho, + base_tau, + nsamples=10000, apply_debias=None, cov_type="jk", ): @@ -625,36 +697,39 @@ def get_samples_lsq( ---------- psf_fitter : PSFFitter PSF fitter instance managing rho/tau statistics and covariances. - version : str - Catalog identifier whose rho/tau statistics are sampled. - base : str - Precomputed basename for locating statistics/covariance files. + base_rho : str + Precomputed basename for locating rho statistics/covariance files. + base_tau : str + Precomputed basename for locating tau statistics/covariance files. apply_debias : int or None, optional Jackknife patch count applied during debiasing. Disabled when ``None``. cov_type : str, optional Covariance label (defaults to ``'jk'``). """ # Load rho and tau stats - psf_fitter.load_rho_stat(f"rho_stats_{base}.fits") - psf_fitter.load_tau_stat(f"tau_stats_{base}.fits") + psf_fitter.load_rho_stat(f"rho_stats_{base_rho}.fits") + psf_fitter.load_tau_stat(f"tau_stats_{base_tau}.fits") + base_sample = f"{base_tau}_sampler_lsq_cov_tau_type_{cov_type}" # Check if the path exists (use base for cache key to account for different TreeCorr configs) - sample_file_path = psf_fitter.get_sample_path(base) + sample_file_path = psf_fitter.get_sample_path(base_sample) if os.path.exists(sample_file_path): print(f"Skipping sampling; {sample_file_path} exists.") - flat_samples = psf_fitter.load_samples(base) + flat_samples = psf_fitter.load_samples(base_sample) mcmc_result, q = psf_fitter.get_mcmc_from_samples(flat_samples) print(mcmc_result) # Or run MCMC else: print("Least square sampling") - tau_covariance = f"cov_tau_{base}_{cov_type}.npy" - rho_covariance = f"cov_rho_{base}_jk.npy" + tau_covariance = f"cov_tau_{base_tau}_{cov_type}.npy" + rho_covariance = f"cov_rho_{base_rho}_jk.npy" psf_fitter.load_covariance(tau_covariance, cov_type="tau") psf_fitter.load_covariance(rho_covariance, cov_type="rho") debias_npatch = apply_debias if (apply_debias is not None) else None flat_samples, mcmc_result, q = psf_fitter.get_least_squares_params_samples( - npatch=debias_npatch, apply_debias=(debias_npatch is not None) + npatch=debias_npatch, + apply_debias=(debias_npatch is not None), + n_samples=nsamples, ) - psf_fitter.save_samples(flat_samples, base) + psf_fitter.save_samples(flat_samples, base_sample) return flat_samples, mcmc_result, q diff --git a/src/sp_validation/statistics.py b/src/sp_validation/statistics.py index dc71f680..17e05219 100644 --- a/src/sp_validation/statistics.py +++ b/src/sp_validation/statistics.py @@ -3,15 +3,55 @@ :Name: statistics.py :Description: Cosmology-independent statistical helpers (jackknife resampling, - chi2/PTE, calibrated min-PTE across many null tests, - covariance<->correlation, OneCovariance reshaping). + jackknife patch centres, chi2/PTE, calibrated min-PTE across + many null tests, covariance<->correlation, OneCovariance reshaping). """ +import itertools from dataclasses import dataclass import numpy as np from scipy import stats +#: Depth of the ball-tree layers that seed the jackknife k-means. +PATCH_MIN_TOP = 6 + + +def jackknife_patch_centers(cat, npatch, seed=0, init="tree"): + """Seeded k-means jackknife patch centres for a TreeCorr catalogue. + + Seeding the k-means does not by itself fix the patches. TreeCorr starts the + k-means from the top layers of a ball tree whose depth, unless ``min_top`` + is given, grows with the OpenMP thread count (``Field._determine_top``), + and ``Catalog(npatch=..., rng=...)`` offers no way to set it. Pinning that + depth makes the centres a function of the catalogue's positions, weights + and ``seed`` alone, on every machine. + + Parameters + ---------- + cat : treecorr.Catalog + Catalogue with spherical (RA, Dec) positions. + npatch : int + Number of patches. + seed : int, optional + Seed for the k-means initialisation. + init : str, optional + TreeCorr initialisation method. Use ``kmeans++`` for draws that sample + different seeded starting centres rather than the same tree cells. + + Returns + ------- + numpy.ndarray + Patch centres, to pass as ``patch_centers`` to ``treecorr.Catalog``. + """ + field = cat.getNField( + min_top=PATCH_MIN_TOP, + max_top=int.bit_length(npatch) - 1, + coords="spherical", + ) + _, centers = field.run_kmeans(npatch, init=init, rng=np.random.default_rng(seed)) + return centers + def jackknif_weighted_average2( data, @@ -121,12 +161,62 @@ def cov_from_one_covariance(cov_one_cov, gaussian=True): Square covariance matrix. """ - n_bins = np.sqrt(cov_one_cov.shape[0]).astype(int) - cov = np.zeros((n_bins, n_bins)) - index_value = 10 if gaussian else 9 - for i in range(n_bins): - for j in range(n_bins): - cov[i, j] = cov_one_cov[i * n_bins + j, index_value] + # Get the ell_bins and tomo_bins for each covariance entry + ell1 = cov_one_cov[:, 1] + ell2 = cov_one_cov[:, 2] + tomoi = cov_one_cov[:, 5].astype(int) + tomoj = cov_one_cov[:, 6].astype(int) + tomok = cov_one_cov[:, 7].astype(int) + tomol = cov_one_cov[:, 8].astype(int) + + # Get the values to save in the covariance + cov_col = 10 if gaussian else 9 + values = cov_one_cov[:, cov_col] + + # Map the ell bins to and index + ell_bins = np.unique(ell1) + n_ell_bins = len(ell_bins) + ell_to_idx = {ell: idx for idx, ell in enumerate(ell_bins)} + + # Map the tomo bin pairs to an index + tomo_bin_ids = np.unique(tomoi) + tomo_bin_pairs = list(itertools.combinations_with_replacement(tomo_bin_ids, 2)) + n_spectra = len(tomo_bin_pairs) + pair_to_idx = {pair: idx for idx, pair in enumerate(tomo_bin_pairs)} + + # Initialize the covariance matrix + cov_size = n_ell_bins * n_spectra + cov = np.zeros((cov_size, cov_size)) + + # Get the tomo bin pair indices + # the ordering of OneCovariance is the same than itertools + a_idx = np.fromiter( + (pair_to_idx[(bin_i, bin_j)] for bin_i, bin_j in zip(tomoi, tomoj)), + dtype=int, + count=len(tomoi), + ) + b_idx = np.fromiter( + (pair_to_idx[(bin_k, bin_l)] for bin_k, bin_l in zip(tomok, tomol)), + dtype=int, + count=len(tomok), + ) + + # Get the ell bin indices + ell1_idx = np.fromiter( + (ell_to_idx[ell] for ell in ell1), dtype=int, count=len(ell1) + ) + ell2_idx = np.fromiter( + (ell_to_idx[ell] for ell in ell2), dtype=int, count=len(ell2) + ) + + # Get the row and col + row = a_idx * n_ell_bins + ell1_idx + col = b_idx * n_ell_bins + ell2_idx + + # Assign the values + cov[row, col] = values + cov[col, row] = values # Symmetrize + return cov diff --git a/src/sp_validation/tests/data/generate_test_cl_catalog_reference.py b/src/sp_validation/tests/data/generate_test_cl_catalog_reference.py new file mode 100644 index 00000000..dbed4a60 --- /dev/null +++ b/src/sp_validation/tests/data/generate_test_cl_catalog_reference.py @@ -0,0 +1,138 @@ +# %% +from pathlib import Path + +import IPython +import numpy as np +import yaml +from astropy.table import Table + +from sp_validation.cosmo_val import CosmologyValidation +from sp_validation.io import open_entry +from sp_validation.rho_tau import get_params_rho_tau + +ipython = IPython.get_ipython() + +# %% +NSIDE = 64 +SEED = 1234 +N_ELL_BINS = 8 + +rng = np.random.default_rng(SEED) +version = "TestCatalog" + +cat_dir = Path("./catalog") +nz_dir = Path("./nz") +output_dir = Path("./output_test") + +for d in (cat_dir, nz_dir, output_dir): + d.mkdir(exist_ok=True) + +n_gal = 5000 +ra = rng.uniform(10.0, 30.0, n_gal) +dec = rng.uniform(10.0, 30.0, n_gal) +e1 = rng.normal(0, 0.25, n_gal) +e2 = rng.normal(0, 0.25, n_gal) +w = rng.uniform(0.5, 1.0, n_gal) +tomo_bin_id = rng.integers(1, 3, n_gal) # Create a two bin catalogue +Table( + {"RA": ra, "Dec": dec, "e1": e1, "e2": e2, "w": w, "tomo_bin_id": tomo_bin_id} +).write(cat_dir / "shear.fits", overwrite=True) + +shear_cfg = { + "path": "shear.fits", + "w_col": "w", + "e1_col": "e1", + "e2_col": "e2", + "R": 1.0, + "e1_col_corrected": "e1", + "e2_col_corrected": "e2", + "ra_col": "RA", + "dec_col": "Dec", + "tomo_bin_col": "tomo_bin_id", +} + +psf_cfg = { + "path": "shear.fits", + "ra_col": "RA", + "dec_col": "Dec", + "e1_PSF_col": "e1", + "e2_PSF_col": "e2", + "e1_star_col": "e1", + "e2_star_col": "e2", + "PSF_size": "w", + "star_size": "w", + "PSF_flag": "w", + "star_flag": "w", +} +config_data = { + "nz": {"subdir": str(nz_dir), "dndz": {"path": "dndz_{pipeline}_A.txt"}}, + "paths": {"output": str(output_dir)}, + version: { + "subdir": str(cat_dir), + "pipeline": "SP", + "shear": shear_cfg, + "star": {**psf_cfg}, + "psf": psf_cfg, + "patch_number": 150, + }, +} + +config_path = Path("./config.yaml") +config_path.write_text(yaml.dump(config_data, sort_keys=False)) + +# %% +cv = CosmologyValidation( + versions=[version], + catalog_config=config_path, + output_dir=str(output_dir), + nside=NSIDE, + binning="powspace", + power=0.5, + n_ell_bins=N_ELL_BINS, + pol_factor=-1, +) +cv._test_version = version + +ver = cv._test_version +params = get_params_rho_tau(cv.cc[ver]) +cat_gal = open_entry(cv.cc[ver]["shear"]) + +cat_gal_tomo_bin_1 = cat_gal[cat_gal[params["tomo_bin_col"]] == 1] +cat_gal_tomo_bin_2 = cat_gal[cat_gal[params["tomo_bin_col"]] == 2] +n_gal_map_a = cv.get_n_gal_map(params, NSIDE, cat_gal_tomo_bin_1) +shear_map_a_e1, shear_map_a_e2 = cv.get_shear_map(params, NSIDE, cat_gal_tomo_bin_1) +n_gal_map_b = cv.get_n_gal_map(params, NSIDE, cat_gal_tomo_bin_2) +shear_map_b_e1, shear_map_b_e2 = cv.get_shear_map(params, NSIDE, cat_gal_tomo_bin_2) + +shear_map_a = shear_map_a_e1 + 1j * shear_map_a_e2 +shear_map_b = shear_map_b_e1 + 1j * shear_map_b_e2 + + +# %% +ell_eff, cl_all, wsp = cv.get_pseudo_cls_map( + shear_map_a, n_gal_map_a, shear_map_b=shear_map_b, mask_b=n_gal_map_b +) + +# %% +# Get the catalogue output +ver = cv._test_version +cv._pseudo_cls = { + ver: { + "tomo_bin_1_tomo_bin_1": {}, + "tomo_bin_1_tomo_bin_2": {}, + "tomo_bin_2_tomo_bin_2": {}, + } +} +out_path = cv._output_path(f"pseudo_cl_cat_{ver}.fits") +tomo_bin_ids, tomo_bin_pairs = cv._get_tomo_bins(ver) +for tomo_bin_a, tomo_bin_b in tomo_bin_pairs: + out_path = cv._output_path(f"pseudo_cl_cat_{ver}_{tomo_bin_a}_{tomo_bin_b}.fits") + cv.calculate_pseudo_cl_catalog( + ver, out_path, tomo_bin_a=tomo_bin_a, tomo_bin_b=tomo_bin_b + ) + + +# %% +np.savez("./test_cl_catalog", cv._pseudo_cls[ver]) +# %% +test_result = np.load("./test_cl_catalog.npz", allow_pickle=True) diff --git a/src/sp_validation/tests/data/test_cl_catalog.npz b/src/sp_validation/tests/data/test_cl_catalog.npz new file mode 100644 index 00000000..fdba323b Binary files /dev/null and b/src/sp_validation/tests/data/test_cl_catalog.npz differ diff --git a/src/sp_validation/tests/regression/test_lsq_sample_cache_ignores_cov_type.py b/src/sp_validation/tests/regression/test_lsq_sample_cache_ignores_cov_type.py new file mode 100644 index 00000000..88e778df --- /dev/null +++ b/src/sp_validation/tests/regression/test_lsq_sample_cache_ignores_cov_type.py @@ -0,0 +1,80 @@ +"""Least-squares PSF-leakage samples must follow the requested tau covariance.""" + +import inspect + +import numpy as np +import pytest +from astropy.table import Table +from shear_psf_leakage.rho_tau_stat import PSFErrorFit, RhoStat, TauStat + +from sp_validation.rho_tau import get_samples_lsq + +CFG = dict(min_sep=1.0, max_sep=30.0, nbins=1, sep_units="arcmin") +SIGMA_TH = 1e-3 +SIGMA_JK = 1e-1 + + +def _write_fixture(out): + # One theta bin: rho_0 = rho_1 = rho_3 = 1, other rho and all tau = 0. + # Each of alpha/beta/eta has independent unit response to a tau component. + rd = {"theta": [10.0]} + for i in range(6): + rd[f"rho_{i}_p"] = [1.0 if i in (0, 1, 3) else 0.0] + td = {"theta": [10.0]} + for i in (0, 2, 5): + td[f"tau_{i}_p"] = [0.0] + Table(rd).write(out / "rho_stats_fit.fits") + Table(td).write(out / "tau_stats_fit.fits") + np.save(out / "cov_tau_fit_th.npy", np.eye(3) * SIGMA_TH**2) + np.save(out / "cov_tau_fit_jk.npy", np.eye(3) * SIGMA_JK**2) + np.save(out / "cov_rho_fit_jk.npy", np.zeros((6, 6))) + + +def _run(fitter, cov_type): + np.random.seed(23) + if "base_tau" in inspect.signature(get_samples_lsq).parameters: + # Tomography branch takes separate rho/tau basenames and a draw count. + return get_samples_lsq(fitter, "fit", "fit", cov_type=cov_type, nsamples=256) + return get_samples_lsq(fitter, "test", "fit", cov_type=cov_type) + + +@pytest.fixture +def restore_random_state(): + state = np.random.get_state() + yield + np.random.set_state(state) + + +def test_lsq_posterior_width_follows_cov_type_not_cached_th( + tmp_path, monkeypatch, restore_random_state +): + """A jk request after a th request must use jk covariance, not cached th draws. + + This one-bin unit-response model has tau = 0 and diagonal covariance + sigma_tau^2 I, so each parameter's posterior std is sigma_tau by + construction: 1e-3 for th and 1e-1 for jk. Using the same output directory + must not let a basename-only cache silently retain the previous method's + alpha/beta/eta errors and xi_psf_sys uncertainty bands. + """ + _write_fixture(tmp_path) + rho = RhoStat(output=str(tmp_path), treecorr_config=CFG) + tau = TauStat(output=str(tmp_path), treecorr_config=CFG) + fitter = PSFErrorFit(rho, tau, str(tmp_path)) + original = fitter.get_least_squares_params_samples + + def small_sample(**kwargs): + # Only reduce Monte Carlo work; leave loading and cache logic untouched. + kwargs.update(n_samples=256, verbose=False) + return original(**kwargs) + + monkeypatch.setattr(fitter, "get_least_squares_params_samples", small_sample) + s_th, _, _ = _run(fitter, "th") + std_th = s_th.std(axis=0) + assert np.allclose(std_th, SIGMA_TH, rtol=0.3), std_th + + s_jk, _, _ = _run(fitter, "jk") + std_jk = s_jk.std(axis=0) + assert np.allclose(std_jk, SIGMA_JK, rtol=0.3), ( + f"jk after th returned posterior std {std_jk}; expected ~{SIGMA_JK}, " + f"not cached th width ~{SIGMA_TH}" + ) diff --git a/src/sp_validation/tests/regression/test_pseudo_cl_covariance_producer_crashes.py b/src/sp_validation/tests/regression/test_pseudo_cl_covariance_producer_crashes.py new file mode 100644 index 00000000..f9502cf3 --- /dev/null +++ b/src/sp_validation/tests/regression/test_pseudo_cl_covariance_producer_crashes.py @@ -0,0 +1,132 @@ +"""Pseudo-Cl covariance producers must run against their branch's public API.""" + +import ast +import inspect +from pathlib import Path + +import numpy as np +import pyccl as ccl +import pytest + +import sp_validation +from sp_validation.cosmo_val import CosmologyValidation + + +def _scripts(): + root = Path(__file__).resolve().parents[4] + code_root = Path(sp_validation.__file__).resolve().parents[2] + if code_root != root: + root = code_root + return root / "workflow/scripts" + + +def _cv_method_calls(script): + """Return name, positional count and keyword names of each cv method call.""" + calls = [] + for node in ast.walk(ast.parse(script.read_text())): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "cv" + ): + calls.append( + (node.func.attr, len(node.args), [k.arg for k in node.keywords]) + ) + return calls + + +class _ReachedCatalogue(Exception): + """The real fiducial-spectrum step succeeded and reached catalogue loading.""" + + +def test_pseudo_cl_cov_does_not_multiply_fiducial_dictionary(tmp_path, monkeypatch): + """The covariance must get past its real fiducial-C_ell/pixel-window step. + + Fiducial theory returns a dictionary keyed by tomographic pair, so the + producer must apply the HEALPix window to each spectrum, not the dictionary. + The catalogue loader raises a sentinel: reaching it is correct by + construction, because theory must finish before data loading starts. + Synthetic n(z) and nside=16 keep this small; no theory result is patched. + """ + mod = pytest.importorskip( + "sp_validation.cosmo_val.pseudo_cl", + reason="tomography branch reorganized the pseudo-Cl covariance module", + ) + z = np.linspace(0.01, 2.0, 100) + nz = np.exp(-0.5 * ((z - 0.7) / 0.2) ** 2) + zpath = tmp_path / "nz.txt" + np.savetxt(zpath, np.column_stack([z, nz])) + + def open_entry(*args, **kwargs): + raise _ReachedCatalogue + + def noop(*args, **kwargs): + pass + + monkeypatch.setattr(mod, "get_params_rho_tau", lambda *args, **kwargs: {}) + monkeypatch.setattr(mod, "open_entry", open_entry) + + class Tiny(mod.PseudoClMixin): + pass + + stub = Tiny() + stub.__dict__.update( + nside=16, + cell_method="map", + force_run=True, + versions=["v"], + cc={"v": {"shear": {"redshift_path": str(zpath)}}}, + cosmo=ccl.CosmologyVanillaLCDM(), + noise_bias_method="analytic", + _output_path=lambda name: str(tmp_path / name), + _output_path_pseudo_cl_cov=lambda *args, **kwargs: str(tmp_path / "cov.fits"), + print_start=noop, + print_magenta=noop, + print_cyan=noop, + print_done=noop, + get_namaster_bin=noop, + ) + with pytest.raises(_ReachedCatalogue): + stub.calculate_pseudo_cl_inka_cov(compute_tomography=False) + + +def test_generate_pseudo_cl_cov_does_not_call_removed_methods(): + """Every cv method called by the covariance producer must exist. + + Existence is required before Python can produce any covariance. + """ + script = _scripts() / "generate_pseudo_cl_cov.py" + missing = [ + name + for name, _, _ in _cv_method_calls(script) + if not hasattr(CosmologyValidation, name) + ] + assert not missing, ( + f"{script.name} calls missing CosmologyValidation methods: {missing}" + ) + + +def test_cv_pseudo_cl_does_not_omit_required_plot_arguments(): + """The workflow's plot call must bind to its actual plotting API. + + The plot-only producer ingests saved spectra and covariances, then calls + plot_pseudo_cl_spectrum once per polarization rather than a cv method. + """ + from sp_validation.cosmo_val.pseudo_cl import plot_pseudo_cl_spectrum + + script = _scripts() / "cv_plot_pseudo_cl.py" + sig = inspect.signature(plot_pseudo_cl_spectrum) + calls = [ + node + for node in ast.walk(ast.parse(script.read_text())) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "plot_pseudo_cl_spectrum" + ] + assert calls, "cv_plot_pseudo_cl.py must call plot_pseudo_cl_spectrum" + for call in calls: + sig.bind( + *([None] * len(call.args)), + **{key.arg: None for key in call.keywords}, + ) diff --git a/src/sp_validation/tests/regression/test_rho_tau_ignores_hsm_flags.py b/src/sp_validation/tests/regression/test_rho_tau_ignores_hsm_flags.py new file mode 100644 index 00000000..4562c81a --- /dev/null +++ b/src/sp_validation/tests/regression/test_rho_tau_ignores_hsm_flags.py @@ -0,0 +1,166 @@ +"""Rho/tau must exclude stars whose configured HSM PSF or star fit failed. + +Clean stars are perfectly modelled, making rho_1 and tau_2 zero by construction. +""" + +import inspect + +import numpy as np +from astropy.io import fits + +from sp_validation.cosmo_val.psf_systematics import PSFSystematicsMixin + +VERSION = "SP_synthetic" +N_GOOD = 2000 +N_FLAGGED = 100 +N_GAL = 3000 + + +def _write_catalogues(tmp_path): + rng = np.random.default_rng(1) + n = N_GOOD + N_FLAGGED + ra = rng.uniform(10.0, 12.0, n) + dec = rng.uniform(0.0, 2.0, n) + e1_psf = rng.normal(0.0, 0.02, n) + e2_psf = rng.normal(0.0, 0.02, n) + t_psf = rng.uniform(0.4, 0.6, n) + + # Good stars: q = e_star - e_PSF = 0 and T_star = T_PSF exactly. + e1_star = e1_psf.copy() + e2_star = e2_psf.copy() + t_star = t_psf.copy() + flag_psf = np.zeros(n, dtype=np.int16) + flag_star = np.zeros(n, dtype=np.int16) + + # Failed fits contribute q = (-0.05, 0) if either flag is ignored. + bad = slice(N_GOOD, n) + e1_psf[bad] = 0.05 + e2_psf[bad] = 0.0 + e1_star[bad] = 0.0 + e2_star[bad] = 0.0 + t_star[bad] = 0.069 + flag_star[N_GOOD : N_GOOD + N_FLAGGED // 2] = 1 + flag_psf[N_GOOD + N_FLAGGED // 2 : n] = 1 + + psf_path = tmp_path / "psf.fits" + fits.BinTableHDU.from_columns( + [ + fits.Column("RA", "D", array=ra), + fits.Column("DEC", "D", array=dec), + fits.Column("HSM_G1_PSF", "D", array=e1_psf), + fits.Column("HSM_G2_PSF", "D", array=e2_psf), + fits.Column("HSM_G1_STAR", "D", array=e1_star), + fits.Column("HSM_G2_STAR", "D", array=e2_star), + fits.Column("HSM_T_PSF", "D", array=t_psf), + fits.Column("HSM_T_STAR", "D", array=t_star), + fits.Column("HSM_FLAG_PSF", "I", array=flag_psf), + fits.Column("HSM_FLAG_STAR", "I", array=flag_star), + ] + ).writeto(psf_path) + + shear_path = tmp_path / "shear.fits" + fits.BinTableHDU.from_columns( + [ + fits.Column("RA", "D", array=rng.uniform(10.0, 12.0, N_GAL)), + fits.Column("Dec", "D", array=rng.uniform(0.0, 2.0, N_GAL)), + fits.Column("e1", "D", array=rng.normal(0.0, 0.3, N_GAL)), + fits.Column("e2", "D", array=rng.normal(0.0, 0.3, N_GAL)), + fits.Column("w_des", "D", array=np.ones(N_GAL)), + ] + ).writeto(shear_path) + return psf_path, shear_path + + +class _Validation(PSFSystematicsMixin): + """Minimal host carrying the configuration read by the real entry point.""" + + def __init__(self, cc): + self.cc = cc + self.versions = [VERSION] + self.treecorr_config = { + "ra_units": "deg", + "dec_units": "deg", + "sep_units": "arcmin", + "min_sep": 2.0, + "max_sep": 100.0, + "nbins": 5, + "num_threads": 2, + } + self.cov_estimate_method = "sim" + self.compute_cov_rho = False + self.npatch = 4 + + def basename(self, version, tomo_bin_a="all", **kwargs): + return f"{version}_{tomo_bin_a}" + + def _get_galaxy_mask(self, ver, tomo_bin_id): + return np.ones(N_GAL, dtype=bool) + + def rho_tau_to_sacc_part(self, *args, **kwargs): + pass + + def print_start(self, *args, **kwargs): + pass + + print_done = print_cyan = print_magenta = print_start + + +def test_flagged_stars_excluded_from_rho_1_and_tau_2(tmp_path): + """Both configured HSM flags must exclude failed fits from rho/tau. + + Every unflagged star has e_star == e_PSF exactly, so q is identically zero: + rho_1 = and tau_2 = must be zero in every bin. + Half of the bad stars have only HSM_FLAG_STAR set, and half have only + HSM_FLAG_PSF set; their q = (-0.05, 0) catches ignoring either flag. + This merges the two audit reproductions and exercises the pipeline's + calculate_rho_tau_stats entry point, not a separately masked reference. + """ + psf_path, shear_path = _write_catalogues(tmp_path) + out = tmp_path / "out" + out.mkdir() + cc = { + "paths": {"output": str(out)}, + VERSION: { + "patch_number": 4, + "cov_th": {"A": 4.0, "n_e": 1.0, "n_psf": 1.0, "sigma_e": 0.3}, + "psf": { + "path": str(psf_path), + "hdu": 1, + "ra_col": "RA", + "dec_col": "DEC", + "e1_PSF_col": "HSM_G1_PSF", + "e2_PSF_col": "HSM_G2_PSF", + "e1_star_col": "HSM_G1_STAR", + "e2_star_col": "HSM_G2_STAR", + "PSF_size": "HSM_T_PSF", + "star_size": "HSM_T_STAR", + "PSF_flag": "HSM_FLAG_PSF", + "star_flag": "HSM_FLAG_STAR", + }, + "shear": { + "path": str(shear_path), + "ra_col": "RA", + "dec_col": "Dec", + "w_col": "w_des", + "e1_col": "e1", + "e2_col": "e2", + }, + }, + } + rho_tau_dir = out / "rho_tau_stats" + rho_tau_dir.mkdir() + # The sim method only needs an existing covariance; don't run an estimator. + np.save(rho_tau_dir / f"cov_tau_{VERSION}_all_th.npy", np.eye(3)) + + val = _Validation(cc) + kwargs = ( + {"tomography": False} + if "tomography" in inspect.signature(val.calculate_rho_tau_stats).parameters + else {} + ) + val.calculate_rho_tau_stats(**kwargs) + + rho_1 = np.asarray(val._rho_stat_handler.rho_stats["rho_1_p"]) + tau_2 = np.asarray(val._tau_stat_handler.tau_stats["tau_2_p"]) + np.testing.assert_allclose(rho_1, 0, atol=1e-12, rtol=0) + np.testing.assert_allclose(tau_2, 0, atol=1e-12, rtol=0) diff --git a/src/sp_validation/tests/regression/test_tomo_covariance_rules_call_external_checkout.py b/src/sp_validation/tests/regression/test_tomo_covariance_rules_call_external_checkout.py new file mode 100644 index 00000000..c5079874 --- /dev/null +++ b/src/sp_validation/tests/regression/test_tomo_covariance_rules_call_external_checkout.py @@ -0,0 +1,239 @@ +"""Exercise real covariance rules without catalogues or CosmoCov. + +These tests pass on develop: it uses checkout-local scripts and launch-relative +outputs. They guard against the external paths introduced on the tomography +branch. Snakemake >=8 is optional; all generated workflow files stay in tmp_path. +""" + +import json +import os +import shlex +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest + +import sp_validation + + +def _run_snakemake(case, *args): + env = dict(os.environ) + env.update( + PYTHONDONTWRITEBYTECODE="1", + XDG_CACHE_HOME=str(case.directory / "cache"), + MPLCONFIGDIR=str(case.directory / "mplconfig"), + TMPDIR=str(case.directory / "tmp"), + # Preserve the dependency location if pytest loaded Snakemake via its + # pythonpath setting rather than the interpreter's site-packages. + PYTHONPATH=str(case.snakemake_root) + os.pathsep + env.get("PYTHONPATH", ""), + ) + return subprocess.run( + [ + sys.executable, + "-m", + "snakemake", + "--snakefile", + str(case.snakefile), + "--cores", + "1", + "--profile", + "none", + "--workflow-profile", + "none", + "--nolock", + "--scheduler", + "greedy", + *args, + ], + cwd=case.directory, + env=env, + text=True, + capture_output=True, + timeout=60, + ) + + +@pytest.fixture +def covariance_rules(tmp_path): + snakemake = pytest.importorskip( + "snakemake", + minversion="8", + reason="Real covariance rules require Snakemake >=8", + ) + # Use the imported package's __file__, not cwd or a hard-coded checkout. + # This also lets PYTHONPATH select tomography while keeping this test file. + root = Path(sp_validation.__file__).resolve().parents[2] + source = root / "workflow/rules/covariance.smk" + if not source.is_file(): + pytest.skip("Checkout does not contain workflow/rules/covariance.smk") + (tmp_path / "tmp").mkdir() + snakefile = tmp_path / "Snakefile" + case = SimpleNamespace( + root=root, + directory=tmp_path, + snakefile=snakefile, + snakemake_root=Path(snakemake.__file__).resolve().parents[1], + ) + # Only globals normally supplied by the workflow are synthetic. Snakemake + # parses the unmodified rules, resolves entry points and executes the job. + snakefile.write_text( + f""" +from pathlib import Path +from snakemake.io import apply_wildcards +import json +COSMO_INFERENCE = Path({str(tmp_path / "cosmo_inference")!r}) +COSMO_VAL = Path({str(tmp_path / "cosmo_val")!r}) +COSMOLOGY_PARAMS = str(COSMO_INFERENCE / "cosmology.json") +GLASS_MOCK_DIR = str(Path({str(tmp_path / "glass_mock")!r})) +GLASS_MOCK_SUITE = "glass_mock_v1.4.6" +PLANCK18 = dict.fromkeys( + ["Omega_m", "Omega_v", "sigma_8", "n_s", "h", "Omega_b"], 0.0 +) +BLOCK_PAIRS = [] +DEFAULT_MASK_SUFFIX = "" +FIDUCIAL = dict.fromkeys([ + "mock_version", "version", "blind", "min_sep", "max_sep", "nbins", + "min_sep_int", "max_sep_int", "nbins_int", +], "synthetic") +config = {{ + "tools": {{"cosmocov_executable": "unused"}}, + "glass_mocks": {{"seed_range": [1, 1]}}, +}} +def fiducial_binning_suffix(): + return "_synthetic" +def covariance_path(*args, **kwargs): + return "unused-fiducial-input" +def covariance_dir(*args, **kwargs): + return "unused-covariance-directory" +def redshift_path(*args, **kwargs): + return "unused-redshift-input" +def build_redshift_path(*args, **kwargs): + return "unused-redshift-input" +def cv_cov_tau(*args, **kwargs): + return "unused-tau-covariance" +def cv_tau_stats(*args, **kwargs): + return "unused-tau-statistics" +wildcard_constraints: + mask_suffix="(?:_masked)?" +include: {str(source)!r} +manifest = {{}} +for name in [ + "covariance_process", "generate_glass_mock_rhotau_samples", + "covariance_glass_mock", +]: + registered = workflow.get_rule(name) + manifest[name] = {{ + "script": registered.script, "shellcmd": registered.shellcmd, + "basedir": str(registered.basedir), + "output": list(map(str, registered.output)), + }} +synthetic_wildcards = dict( + version="synthetic", blind="A", gaussian="ng", min_sep="1", max_sep="2", + nbins="1", mask_suffix="", +) +manifest["covariance_process"]["synthetic_matrix"] = apply_wildcards( + workflow.get_rule("covariance_process").output[0], synthetic_wildcards +) +Path("rules.json").write_text(json.dumps(manifest, indent=2)) +""" + ) + # Upper-triangle dump: columns 8 and 9 contain Gaussian and non-Gaussian + # terms. Their sum is the positive-definite matrix [[3, .25], [.25, 6]]. + raw = np.array( + [ + [0, 0, 0, 0, 0, 0, 0, 0, 2, 1], + [0, 1, 0, 0, 0, 0, 0, 0, 0.25, 0], + [1, 1, 0, 0, 0, 0, 0, 0, 4, 2], + ] + ) + parsed = _run_snakemake(case, "--list-rules") + assert parsed.returncode == 0, parsed.stdout + parsed.stderr + case.manifest = json.loads((tmp_path / "rules.json").read_text()) + case.matrix = Path(case.manifest["covariance_process"]["synthetic_matrix"]) + raw_path = Path(str(case.matrix).replace("_processed.txt", ".txt")) + raw_path.parent.mkdir(parents=True) + np.savetxt(raw_path, raw) + return case + + +@pytest.mark.slow +def test_covariance_postprocessing_runs_from_launched_checkout(covariance_rules): + """Process a synthetic CosmoCov dump without relying on another checkout. + + The three input rows give G+NG = [[3, .25], [.25, 6]], reflecting the supplied + off-diagonal element, and G = [[2, .25], [.25, 4]]. Snakemake executes the + actual rule with existing raw input, so no survey data or expensive + covariance calculation is needed. A missing external entry point is a + workflow defect. Develop passes because its processor is checkout-local; + this guards against the tomography merge reintroducing an external script. + """ + case = covariance_rules + result = _run_snakemake(case, "--printshellcmds", str(case.matrix)) + log = result.stdout + result.stderr + diagnosis = next( + (line for line in log.splitlines() if "can't open file" in line), log[-2500:] + ) + assert result.returncode == 0, ( + f"covariance_process returned {result.returncode}; expected exit 0 and " + f"matrix [[3.0, 0.25], [0.25, 6.0]]; {diagnosis}" + ) + np.testing.assert_allclose( + np.loadtxt(case.matrix), [[3, 0.25], [0.25, 6]], rtol=0, atol=0 + ) + gaussian = Path(str(case.matrix).replace("_processed.txt", "_processed_g.txt")) + np.testing.assert_allclose( + np.loadtxt(gaussian), [[2, 0.25], [0.25, 4]], rtol=0, atol=0 + ) + + +def test_tau_sampling_selects_script_in_launched_checkout(covariance_rules): + """The real tau-sampling rule must select code from the launched checkout. + + Identical script contents today cannot guarantee this: edits to an unrelated + checkout must not change a fixed commit's mock samples. Inspect Snakemake's + registered entry point instead of inferring provenance from random values. + Develop passes with its local script directive; the tomography merge must + not restore an external entry point. No survey inputs are read. + """ + case = covariance_rules + rule = case.manifest["generate_glass_mock_rhotau_samples"] + if rule["script"]: + selected = (Path(rule["basedir"]) / rule["script"]).resolve() + else: + tokens = shlex.split(rule["shellcmd"]) + assert tokens[0] == "python", f"Unexpected tau command: {rule['shellcmd']}" + selected = (case.directory / tokens[1]).resolve() + expected = ( + case.root / "workflow/scripts/generate_glass_mock_rhotau_samples.py" + ).resolve() + assert selected == expected, ( + f"Tau sampling selects {selected}; expected launched-checkout script {expected}" + ) + assert selected.is_file(), f"Selected tau script is missing: {selected}" + + +def test_glass_covariance_outputs_stay_in_launch_directory(covariance_rules): + """All seven GLASS outputs must live in the launch directory's results tree. + + Resolve the real rule's output objects relative to the synthetic launch + directory, the correct base for portable products and independent launches. + No GLASS catalogues are read or external products written. Develop passes + with relative outputs; this guards against the tomography merge restoring + absolute paths that could overwrite another analysis. + """ + case = covariance_rules + expected_dir = case.directory / "results/covariance/glass_mock_v1.4.6" + outputs = case.manifest["covariance_glass_mock"]["output"] + assert len(outputs) == 7 + wrong = [] + for output in outputs: + actual = (case.directory / output).resolve() + if actual.parent != expected_dir: + wrong.append(str(actual)) + assert not wrong, ( + f"GLASS covariance outputs escape launch directory: {wrong}; " + f"expected directory {expected_dir}" + ) diff --git a/src/sp_validation/tests/regression/test_tomo_drops_min_top_pin.py b/src/sp_validation/tests/regression/test_tomo_drops_min_top_pin.py new file mode 100644 index 00000000..09cea030 --- /dev/null +++ b/src/sp_validation/tests/regression/test_tomo_drops_min_top_pin.py @@ -0,0 +1,101 @@ +"""ξ± from CosmologyValidation must not depend on the host's CPU count.""" + +import multiprocessing + +import numpy as np +import pytest +import treecorr +import yaml + +from sp_validation.cosmo_val import CosmologyValidation + + +@pytest.fixture(autouse=True) +def _restore_treecorr_threads(): + previous = treecorr.get_omp_threads() + try: + yield + finally: + treecorr.set_omp_threads(previous) + + +def _cosmology_validation(tmp_path): + """Minimal CosmologyValidation: only the constructor runs, no catalogue is read.""" + cfg = { + "paths": {"output": str(tmp_path / "out")}, + "nz": {"subdir": str(tmp_path), "dndz": {"path": "nz.txt"}}, + "synthetic": {"subdir": str(tmp_path), "shear": {"path": "cat.fits"}}, + } + cfg_path = tmp_path / "cat_config.yaml" + cfg_path.write_text(yaml.safe_dump(cfg)) + return CosmologyValidation( + ["synthetic"], + catalog_config=str(cfg_path), + npatch=8, + theta_min=15.0, + theta_max=70.0, + nbins=6, + ) + + +def _synthetic_catalog(**kwargs): + """4000 galaxies over ~4x4 deg with a spatially correlated shear field.""" + rng = np.random.default_rng(1234) + n = 4000 + ra = rng.uniform(150.0, 154.0, n) + dec = rng.uniform(0.0, 4.0, n) + g1 = 0.02 * np.sin(np.radians(ra) * 90) + 0.25 * rng.standard_normal(n) + g2 = 0.02 * np.cos(np.radians(dec) * 90) + 0.25 * rng.standard_normal(n) + return treecorr.Catalog( + ra=ra, dec=dec, g1=g1, g2=g2, ra_units="deg", dec_units="deg", **kwargs + ) + + +def _xi_on_node(cv, n_cpu, patch_centers, monkeypatch): + """ξ± as calculate_2pcf_version builds it (instance npatch, jackknife, fixed + patch centres, fresh Catalog), on a node with n_cpu CPUs.""" + monkeypatch.setattr(multiprocessing, "cpu_count", lambda: n_cpu) + # Simulate the thread count seen by Field's default min_top calculation, + # without actually launching 48 threads inside a two-core test allocation. + # Leave min_top itself untouched: the missing pin must remain observable. + monkeypatch.setattr(treecorr.field, "get_omp_threads", multiprocessing.cpu_count) + config = {**cv._binning(), "var_method": "jackknife"} + gg = treecorr.GGCorrelation(config) + gg.process(_synthetic_catalog(patch_centers=patch_centers), num_threads=2) + return gg + + +def test_xi_pm_identical_on_16_and_48_cpu_nodes(tmp_path, monkeypatch): + """Protects the machine independence of ξ±. TreeCorr picks the depth of its root + cells (min_top) as max(3, ceil(log2 n_threads)) when the config leaves it unset, + and the thread count defaults to the node's CPU count; min_top decides which pairs + bin_slop approximates, so ξ± changes between a 16-CPU node + (min_top=4) and a 48-CPU node (min_top=6) once the catalogue is split into jackknife + patches. The same catalogue, patch centres and CosmologyValidation settings must + give the same ξ± on either node, to far below the jackknife σ (1e-6σ), which holds + once the config pins min_top (and num_threads). This passes on develop, + whose pin must survive the tomography merge. Only Field's view of the host + thread count is simulated; the actual correlation runs on two cores. + """ + cv = _cosmology_validation(tmp_path) + patch_file = str(tmp_path / "patches.dat") + treecorr.set_omp_threads(2) + _synthetic_catalog( + npatch=cv.npatch, rng=np.random.default_rng(99) + ).write_patch_centers(patch_file) + gg16 = _xi_on_node(cv, 16, patch_file, monkeypatch) + gg48 = _xi_on_node(cv, 48, patch_file, monkeypatch) + sigma = np.sqrt(np.concatenate([gg48.varxip, gg48.varxim])) + shift = ( + np.abs( + np.concatenate([gg16.xip, gg16.xim]) - np.concatenate([gg48.xip, gg48.xim]) + ) + / sigma + ) + + assert shift.max() < 1e-6, ( + f"ξ± moves by {shift.max():.3g}σ between a 16- and a 48-CPU node " + f"(xip {shift[: cv.nbins].max():.3g}σ, xim {shift[cv.nbins :].max():.3g}σ); " + f"treecorr_config min_top={cv.treecorr_config.get('min_top')}, " + f"num_threads={cv.treecorr_config.get('num_threads')}" + ) diff --git a/src/sp_validation/tests/regression/test_tomo_gaussian_sims_seed_filename_mismatch.py b/src/sp_validation/tests/regression/test_tomo_gaussian_sims_seed_filename_mismatch.py new file mode 100644 index 00000000..4bfd96f7 --- /dev/null +++ b/src/sp_validation/tests/regression/test_tomo_gaussian_sims_seed_filename_mismatch.py @@ -0,0 +1,100 @@ +"""Gaussian-sim covariance must aggregate the spectra its seeded workers wrote. + +Develop lacks this tomography-branch script, so these tests skip there. +""" + +import importlib.util +import inspect +from pathlib import Path + +import numpy as np +import pytest + +import sp_validation + +SCRIPT_REL = "scripts/cosmo_val/run_cl_gaussian_sims.py" +VERSION = "vtest" +TOMO = False +SEED = 7 +N_SIMS = 3 +N_ELL = 4 + + +@pytest.fixture +def gaussian_sims(): + # Resolve from the imported package, so PYTHONPATH can select another branch + # while this regression file stays in the original checkout. + root = Path(sp_validation.__file__).resolve().parents[2] + script = root / SCRIPT_REL + if not script.is_file(): + pytest.skip(f"{SCRIPT_REL} is only available on the tomography branch") + pytest.importorskip("mpi4py", reason="Gaussian-sim script requires mpi4py") + spec = importlib.util.spec_from_file_location("run_cl_gaussian_sims", script) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + if not hasattr(module, "get_covariance_from_simulated_spectra"): + pytest.skip("Covariance aggregator is only available on the tomography branch") + return module + + +def _cl_dict(rng): + # NaMaster-decoupled spin-2 x spin-2: (EE, EB, BE, BB) x n_ell. + return {"W1xW1": rng.normal(size=(4, N_ELL))} + + +def _writer_name(out_dir, sim_id): + # Naming used by run_one_simulation when it saves a realisation. + return out_dir / (f"cl_sample_{sim_id}_{VERSION}_tomography_{TOMO}_seed_{SEED}.npz") + + +def _stale_name(out_dir, sim_id): + # Pre-seed naming: what an older run left behind in the same directory. + return out_dir / f"cl_sample_{sim_id}_{VERSION}_tomography_{TOMO}.npz" + + +def _call(module, out_dir): + fn = module.get_covariance_from_simulated_spectra + kwargs = {} + if "seed" in inspect.signature(fn).parameters: + kwargs["seed"] = SEED + return fn(N_SIMS, VERSION, TOMO, [1], "EE", str(out_dir), **kwargs) + + +def test_covariance_reads_seed_suffixed_spectra_in_fresh_dir(tmp_path, gaussian_sims): + """A seed-S run must aggregate exactly its cl_sample_*_seed_S.npz files. + + Nothing else exists in this fresh directory, so aggregation must succeed + and equal np.cov of the seed-S EE realisations, computed independently here. + No HEALPix simulation, catalogue or MPI work is needed. + """ + rng = np.random.default_rng(0) + seeded = [] + for i in range(N_SIMS): + cl = _cl_dict(rng) + np.savez(_writer_name(tmp_path, i), cl_decoupled=cl) + seeded.append(cl["W1xW1"][0]) + cov = _call(gaussian_sims, tmp_path) + np.testing.assert_allclose(cov, np.cov(np.array(seeded), rowvar=False)) + + +def test_covariance_ignores_stale_unsuffixed_spectra(tmp_path, gaussian_sims): + """A seed-S covariance must not silently use stale unsuffixed spectra. + + Both sets exist, but their distinct amplitudes make their sample covariances + different. The right answer is np.cov of this run's seeded EE realisations, + not the pre-seed files left by an older run. + """ + rng = np.random.default_rng(1) + seeded, stale = [], [] + for i in range(N_SIMS): + cl = _cl_dict(rng) + np.savez(_writer_name(tmp_path, i), cl_decoupled=cl) + seeded.append(cl["W1xW1"][0]) + old = {"W1xW1": 100.0 * rng.normal(size=(4, N_ELL))} + np.savez(_stale_name(tmp_path, i), cl_decoupled=old) + stale.append(old["W1xW1"][0]) + want = np.cov(np.array(seeded), rowvar=False) + old_cov = np.cov(np.array(stale), rowvar=False) + assert not np.allclose(want, old_cov), "Fixture must distinguish the two runs" + cov = _call(gaussian_sims, tmp_path) + np.testing.assert_allclose(cov, want) diff --git a/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py b/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py new file mode 100644 index 00000000..4d7be5db --- /dev/null +++ b/src/sp_validation/tests/regression/test_tomo_inka_merge_crosspol_transpose.py @@ -0,0 +1,96 @@ +"""Tomographic iNKA merging must transpose the (Q,P) block, not (P,Q). + +Develop lacks this tomography-branch merger, so the test skips there. +""" + +import itertools + +import numpy as np +import pytest + +pseudo_cl = pytest.importorskip( + "sp_validation.cosmo_val.pseudo_cl", + reason="Tomographic iNKA merger module is only on the tomography branch", +) +PseudoClMixin = getattr(pseudo_cl, "PseudoClMixin", None) + +POLS = ["EE", "EB", "BE", "BB"] + + +def test_tomo_inka_merge_lower_triangle_uses_transposed_qp_block(tmp_path): + """The merged tomographic iNKA covariance must equal the joint covariance. + + Draw a symmetric positive-definite covariance C over the data vector indexed + (polarisation P, spectrum pair s, ell). The correct COVAR_P_Q is therefore + C[P, :, :, Q, :, :] by construction. Write only upper-triangle spectrum-pair + blocks (a <= b), as the pipeline does, in its [ell, pol, ell, pol] layout + using the real _save_iNKA_covariance method. + + The lower block (b, a) of COVAR_P_Q is Cov(C_b^P, C_a^Q), which equals + block_{Q,P}(a, b).T. Using block_{P,Q}(a, b).T is only right when P == Q. + This protects all 16 polarisation blocks, including EE_BB and EB_BE, and + the symmetry COVAR_P_Q == COVAR_Q_P.T. It consolidates both audit reproductions + of the same incorrect cross-polarisation transpose. + """ + if PseudoClMixin is None or not hasattr(PseudoClMixin, "_merge_iNKA_covariance"): + pytest.skip("Tomographic iNKA merger is only on the tomography branch") + + n_bins_src = [1, 2] + pairs = list(itertools.combinations_with_replacement(n_bins_src, 2)) + n_s, n_ell, n_pol = len(pairs), 3, 4 + + rng = np.random.default_rng(7) + dim = n_pol * n_s * n_ell + draws = rng.standard_normal((dim, dim)) + covariance = (draws @ draws.T / dim + np.eye(dim)).reshape( + n_pol, n_s, n_ell, n_pol, n_s, n_ell + ) + + class Tiny(PseudoClMixin): + def _get_tomo_bins(self, ver): + return n_bins_src, pairs + + def _output_path_pseudo_cl_cov(self, ver, method, tomography): + return str(tmp_path / "merged.fits") + + def _output_path_iNKA_block_cov(self, ver, tomo_bin_quad): + return str(tmp_path / ("_".join(map(str, tomo_bin_quad)) + ".fits")) + + instance = Tiny() + instance._pseudo_cls = { + "v": {"tomo_bin_all_tomo_bin_all": {"pseudo_cl": {"ELL": np.arange(n_ell)}}} + } + for ia, ib in itertools.combinations_with_replacement(range(n_s), 2): + # [ell_a, pol_a, ell_b, pol_b] + block = np.transpose(covariance[:, ia, :, :, ib, :], (1, 0, 3, 2)) + quad = (*pairs[ia], *pairs[ib]) + with instance._save_iNKA_covariance( + block, instance._output_path_iNKA_block_cov("v", quad) + ): + pass + + with instance._merge_iNKA_covariance("v", True) as merged: + errors = {} + for i, pa in enumerate(POLS): + for j, pb in enumerate(POLS): + expected = covariance[i, :, :, j, :, :].reshape( + n_s * n_ell, n_s * n_ell + ) + got = merged[f"COVAR_{pa}_{pb}"].data + errors[f"{pa}_{pb}"] = np.max(np.abs(got - expected)) + bad = {k: f"{v:.3g}" for k, v in errors.items() if v > 1e-12} + asym = np.max(np.abs(merged["COVAR_EE_BB"].data - merged["COVAR_BB_EE"].data.T)) + assert not bad and asym < 1e-12, ( + f"Merged blocks differ from the joint covariance: {bad}; " + f"max|COVAR_EE_BB - COVAR_BB_EE.T| = {asym:.3g}" + ) + joint = np.block( + [[merged[f"COVAR_{pa}_{pb}"].data for pb in POLS] for pa in POLS] + ) + np.testing.assert_allclose(joint, joint.T, rtol=0, atol=1e-12) + eigenvalues = np.linalg.eigvalsh(joint) + assert eigenvalues.min() > 0 + print( + f"iNKA joint covariance: symmetry error={np.max(np.abs(joint - joint.T)):.3g}, " + f"min eigenvalue={eigenvalues.min():.12g}" + ) diff --git a/src/sp_validation/tests/regression/test_tomo_joint_pure_b_hartlap_dimension.py b/src/sp_validation/tests/regression/test_tomo_joint_pure_b_hartlap_dimension.py new file mode 100644 index 00000000..177e9837 --- /dev/null +++ b/src/sp_validation/tests/regression/test_tomo_joint_pure_b_hartlap_dimension.py @@ -0,0 +1,39 @@ +"""Joint xi+ B / xi- B statistics must debias the full joint inverse covariance.""" + +from types import SimpleNamespace + +import numpy as np +import pytest +from scipy import stats + +from sp_validation.b_modes import calculate_eb_statistics + + +@pytest.mark.parametrize("start, stop", [(0, 4), (0, 3), (1, 3)]) +def test_joint_b_mode_chi2_hartlap_uses_joint_dimension_2n(start, stop): + """The combined B-mode PTE must apply Hartlap with p = 2n, not p = n. + + An inverse sample covariance from N patches over p entries requires + (N - p - 2)/(N - 1). For identity covariance the raw joint chi2 is the + sum of squared xi+ B and xi- B over the cut, so the PTE is known by hand. + Full and interior cuts check that p is twice the *selected* bin count. + Develop already uses p = 2n; this unmarked test protects that correct + behaviour against the tomography branch's single-block p = n regression. + """ + nbins, npatch = 4, 50 + results = { + # Both API contracts: develop reads theta/npatch; tomography reads gg. + "theta": np.geomspace(1.0, 100.0, nbins), + "npatch": npatch, + "gg": SimpleNamespace(nbins=nbins, npatch1=npatch, npatch2=npatch), + "cov": np.eye(6 * nbins), + "xip_B": np.linspace(0.5, 2.0, nbins), + "xim_B": np.linspace(-1.5, 1.0, nbins), + } + pte = calculate_eb_statistics(results)["pte_matrices"]["combined"] + n = stop - start + raw = np.sum(results["xip_B"][start:stop] ** 2) + np.sum( + results["xim_B"][start:stop] ** 2 + ) + expected = stats.chi2.sf((npatch - 2 * n - 2) / (npatch - 1) * raw, 2 * n) + assert pte[start, stop - 1] == pytest.approx(expected, rel=1e-10) diff --git a/src/sp_validation/tests/regression/test_tomo_mc_covariance_uniform_rebinning.py b/src/sp_validation/tests/regression/test_tomo_mc_covariance_uniform_rebinning.py new file mode 100644 index 00000000..aafed418 --- /dev/null +++ b/src/sp_validation/tests/regression/test_tomo_mc_covariance_uniform_rebinning.py @@ -0,0 +1,105 @@ +"""Pure-E/B MC covariance must describe the data's pair-weighted xi estimator.""" + +import inspect +import sys +import types +from types import SimpleNamespace + +import numpy as np +import pytest + +from sp_validation import b_modes as bm + +# Reporting bins are exact unions of eight fine log bins, so no partial-bin +# interpolation enters the independently constructed pair-weighted reference. +EDGES_INT = np.geomspace(1.0, 100.0, 41) +THETA_INT = np.sqrt(EDGES_INT[:-1] * EDGES_INT[1:]) +REPORT_IDX = np.array([4, 12, 20, 28, 36]) +EDGES_REP = EDGES_INT[REPORT_IDX] +WEIGHT_INT = THETA_INT**2 # annular area, hence pair counts, grows as theta^2 + + +def _pair_weighted_binning(): + matrix = np.zeros((len(EDGES_REP) - 1, len(THETA_INT))) + for i, (lo, hi) in enumerate(zip(REPORT_IDX[:-1], REPORT_IDX[1:])): + matrix[i, lo:hi] = WEIGHT_INT[lo:hi] / WEIGHT_INT[lo:hi].sum() + return matrix + + +def test_mc_reporting_xi_is_pair_weighted_mean_of_fine_bins(monkeypatch, tmp_path): + """Each MC reporting xi must be sum_j w_j xi_j / sum_j w_j, like TreeCorr. + + Reporting-bin xi feeds the local term of the pure-E/B transform. A uniform + average of fine bins would give a covariance for a different estimator + from the pair-weighted data. Exact unions of fine bins make the reference + independent of the package's binning implementation. + Develop's explicit operator already uses pair weights, so this test is + unmarked: it protects against the tomography branch's uniform MC rebinning. + The transform and theory stubs only expose its inputs, not fix the binning. + """ + reference = _pair_weighted_binning() + rng = np.random.default_rng(0) + if "cov_path_int" in inspect.signature(bm.calculate_pure_eb_correlation).parameters: + # Tomography API: observe reporting and fine xi passed to the transform. + n_int = len(THETA_INT) + gg = SimpleNamespace( + meanr=np.sqrt(EDGES_REP[:-1] * EDGES_REP[1:]), + left_edges=EDGES_REP[:-1], + right_edges=EDGES_REP[1:], + xip=np.zeros(4), + xim=np.zeros(4), + ) + gg_int = SimpleNamespace( + meanr=THETA_INT, + left_edges=EDGES_INT[:-1], + right_edges=EDGES_INT[1:], + xip=np.zeros(n_int), + xim=np.zeros(n_int), + weight=WEIGHT_INT, + ) + cov_path = tmp_path / "cov_int.txt" + np.savetxt(cov_path, np.eye(2 * n_int)) + calls = [] + + def observe_pure_eb(theta, xip, xim, theta_int, xip_int, xim_int, **kwargs): + calls.append((np.array([xip, xim]), np.array([xip_int, xim_int]))) + return tuple(np.zeros(len(theta)) for _ in range(6)) + + # Replace only the transform dependency, avoiding its eager JIT import. + # The package's actual MC sampling and rebinning still run unchanged. + module = types.ModuleType("cosmo_numba.B_modes.schneider2022") + module.get_pure_EB_modes = observe_pure_eb + monkeypatch.setitem(sys.modules, "cosmo_numba.B_modes.schneider2022", module) + monkeypatch.setattr( + bm, "get_theo_xi", lambda **kwargs: (np.zeros(n_int), np.zeros(n_int)) + ) + # Keep the package's MC sampling but avoid changing global random state. + monkeypatch.setattr(np.random, "multivariate_normal", rng.multivariate_normal) + with pytest.warns(UserWarning, match="covariance matrix is not positive"): + bm.calculate_pure_eb_correlation( + gg, + gg_int, + cov_path_int=str(cov_path), + cosmo_cov=object(), + n_samples=3, + z_dist=np.ones((5, 2)), + ) + mc = calls[1:] # First call transforms data, not an MC sample. + assert len(mc) == 3 + got = np.array([reporting for reporting, fine in mc]) + want = np.array([fine @ reference.T for reporting, fine in mc]) + else: + # Develop API: rebinning is the explicit linear operator. + matrix, edges = bm._reporting_binning(WEIGHT_INT, EDGES_INT, EDGES_REP) + np.testing.assert_allclose(edges, EDGES_REP) + fine = rng.standard_normal((3, len(THETA_INT))) + got = fine @ matrix.T + want = fine @ reference.T + + np.testing.assert_allclose( + got, + want, + rtol=0, + atol=1e-12, + err_msg="MC reporting xi is not the pair-weighted mean of fine bins", + ) diff --git a/src/sp_validation/tests/regression/test_tomo_plot_cosebis_ignores_tomography.py b/src/sp_validation/tests/regression/test_tomo_plot_cosebis_ignores_tomography.py new file mode 100644 index 00000000..63de87b7 --- /dev/null +++ b/src/sp_validation/tests/regression/test_tomo_plot_cosebis_ignores_tomography.py @@ -0,0 +1,141 @@ +"""plot_cosebis(compute_tomography=True) must produce per-bin-pair COSEBIs.""" + +import inspect +from contextlib import nullcontext +from types import SimpleNamespace + +import numpy as np +import pytest +import yaml +from astropy.io import fits + +cosebis_mod = pytest.importorskip( + "sp_validation.cosmo_val.cosebis", + reason="COSEBIs module required by the tomography branch", +) +from sp_validation.cosmo_val import CosmologyValidation # noqa: E402 + +VER = "SP_toy" +NBINS = 60 + + +def _make_cv(tmp_path): + rng = np.random.default_rng(11) + n = 600 + t = np.zeros( + n, + dtype=[ + ("RA", "f8"), + ("Dec", "f8"), + ("e1", "f8"), + ("e2", "f8"), + ("w", "f8"), + ("tomo", "i4"), + ], + ) + t["RA"] = 10 + rng.uniform(-0.5, 0.5, n) + t["Dec"] = rng.uniform(-0.5, 0.5, n) + t["tomo"] = np.repeat([1, 2], n // 2) + # Opposite coherent e1 in the two bins: the merged catalogue nearly cancels, + # so the all-galaxy COSEBIs differ strongly from any single bin pair. + t["e1"] = np.where(t["tomo"] == 1, 0.1, -0.1) + rng.normal(0, 0.01, n) + t["e2"] = rng.normal(0, 0.01, n) + t["w"] = 1.0 + cat = tmp_path / "toy.fits" + fits.BinTableHDU(t).writeto(cat) + out = tmp_path / "out" + out.mkdir() + cfg = { + VER: { + "subdir": str(tmp_path), + "shear": { + "path": str(cat), + "e1_col": "e1", + "e2_col": "e2", + "w_col": "w", + "R": 1.0, + "tomo_bin_col": "tomo", + }, + }, + "nz": {"subdir": str(tmp_path)}, + "paths": {"output": str(out)}, + } + cfg_path = tmp_path / "cfg.yaml" + cfg_path.write_text(yaml.safe_dump(cfg)) + cv = CosmologyValidation( + [VER], + catalog_config=str(cfg_path), + output_dir=str(out), + compute_tomography=True, + npatch=4, + ) + cv.treecorr_config["num_threads"] = 2 + # Hand the table straight in, bypassing the leakage-correction reader. + cv._results = { + VER: SimpleNamespace(dat_shear=t, temporarily_read_data=lambda: nullcontext()) + } + cv._c1 = {VER: 0.0} + cv._c2 = {VER: 0.0} + # Each bin pair takes its own jackknife covariance; a single cov_path cannot + # serve every pair. + params = dict( + version=VER, + min_sep_int=1.0, + max_sep_int=30.0, + nbins_int=NBINS, + npatch=4, + nmodes=1, + ) + return cv, params + + +@pytest.mark.slow +def test_plot_cosebis_compute_tomography_true_stores_bin_pair_results( + tmp_path, monkeypatch +): + """plot_cosebis exposes a ``compute_tomography`` switch and saves/stores the + COSEBIs data vector and PTEs it computes. With ``compute_tomography=True`` on + a two-bin catalogue the stored results must be the three tomographic bin + pairs (1,1), (1,2), (2,2), identical to what ``calculate_cosebis(..., + compute_tomography=True)`` returns for the same inputs; the merged + all-galaxy E_n is the wrong answer. The fixture gives the two bins opposite + coherent e1 so the merged catalogue's E_1 differs from every bin pair's by + a large factor, making the substitution unmistakable.""" + if ( + "compute_tomography" + not in inspect.signature(CosmologyValidation.plot_cosebis).parameters + ): + pytest.skip("Tomography branch adds plot_cosebis(compute_tomography=...)") + # The first COSEBIs call can compile numba kernels; keep their cache local. + monkeypatch.setenv("NUMBA_CACHE_DIR", str(tmp_path / "numba-cache")) + # Plot rendering is irrelevant; keep genuine computation and result storage. + monkeypatch.setattr(cosebis_mod, "plot_cosebis_modes", lambda *a, **k: None) + monkeypatch.setattr( + cosebis_mod, "plot_cosebis_covariance_matrix", lambda *a, **k: None + ) + cv, params = _make_cv(tmp_path) + + expected = cv.calculate_cosebis(**params, compute_tomography=True) + assert set(expected) == { + "tomo_bin_1_tomo_bin_1", + "tomo_bin_1_tomo_bin_2", + "tomo_bin_2_tomo_bin_2", + } + + cv.plot_cosebis(**params, compute_tomography=True) + stored = cv._cosebis_results[VER] + + stored_keys = sorted(stored) if isinstance(stored, dict) else None + assert isinstance(stored, dict) and set(expected) <= set(stored), ( + "plot_cosebis(compute_tomography=True) stored non-tomographic COSEBIs: " + f"keys={stored_keys}, En={np.asarray(stored.get('En')).tolist()}; " + "expected bin pairs with En=" + f"{ {k: v['En'].tolist() for k, v in expected.items()} }" + ) + for k, v in expected.items(): + np.testing.assert_allclose(stored[k]["En"], v["En"], rtol=1e-10) + products = list((tmp_path / "out").glob(f"*_{k}_data.npz")) + assert len(products) == 1 + with np.load(products[0]) as product: + np.testing.assert_allclose(product["En"], v["En"], rtol=1e-10) + np.testing.assert_allclose(product["Bn"], v["Bn"], rtol=1e-10) diff --git a/src/sp_validation/tests/regression/test_tomo_pure_eb_mapping_keyerror.py b/src/sp_validation/tests/regression/test_tomo_pure_eb_mapping_keyerror.py new file mode 100644 index 00000000..2e6ce4bc --- /dev/null +++ b/src/sp_validation/tests/regression/test_tomo_pure_eb_mapping_keyerror.py @@ -0,0 +1,127 @@ +"""plot_pure_eb must accept the bin-keyed results from calculate_pure_eb.""" + +import inspect + +import numpy as np +import pytest +import treecorr + +pure_eb = pytest.importorskip( + "sp_validation.cosmo_val.pure_eb", + reason="PureEBMixin is required for the tomography branch's bin-keyed API", +) +if not hasattr(pure_eb, "PureEBMixin"): + pytest.skip( + "PureEBMixin is missing (tomography branch API)", allow_module_level=True + ) +PureEBMixin = pure_eb.PureEBMixin +BIN_KEY = "tomo_bin_all_tomo_bin_all" + + +def _synthetic_gg(min_sep, max_sep, nbins, npatch): + rng = np.random.default_rng(1) + n = 8000 + cat = treecorr.Catalog( + ra=rng.uniform(150.0, 152.0, n), + dec=rng.uniform(1.0, 3.0, n), + g1=rng.normal(0, 0.2, n), + g2=rng.normal(0, 0.2, n), + ra_units="deg", + dec_units="deg", + npatch=npatch, + rng=rng, + ) + gg = treecorr.GGCorrelation( + min_sep=min_sep, + max_sep=max_sep, + nbins=nbins, + var_method="jackknife", + sep_units="arcmin", + bin_slop=0.0, + ) + gg.process(cat, num_threads=2) + return gg + + +class _TomoCV(PureEBMixin): + """Minimal host supplying the tomography branch's documented 2PCF mapping.""" + + versions = ["v"] + npatch = 32 + treecorr_config = {"min_sep": 2.0, "max_sep": 60.0, "nbins": 3} + + def __init__(self, output): + self.cc = {"paths": {"output": str(output)}} + self._pure_eb_results = {} + + def print_start(self, *args, **kwargs): + pass + + def _binning(self, min_sep=None, max_sep=None, nbins=None): + return { + "min_sep": min_sep or self.treecorr_config["min_sep"], + "max_sep": max_sep or self.treecorr_config["max_sep"], + "nbins": nbins or self.treecorr_config["nbins"], + } + + def calculate_2pcf_version(self, version, npatch, compute_tomography, **config): + return { + BIN_KEY: _synthetic_gg( + config["min_sep"], config["max_sep"], config["nbins"], npatch + ) + } + + +@pytest.mark.slow +def test_plot_pure_eb_passes_one_bin_result_to_eb_statistics(tmp_path, monkeypatch): + """The statistics step needs a per-bin result, not the outer bin mapping. + + On the tomography branch calculate_pure_eb returns {bin_key: result}, even + for the single (all, all) bin. plot_pure_eb must pass that inner result + (carrying gg, cov and xip_B) to calculate_eb_statistics. A finite combined + B-mode PTE is the expected outcome of this random-shear TreeCorr fixture. + The real calculation and statistics run; only the 2PCF source and figure + writers are substituted. Develop's single-result API is unaffected and + skips because it lacks the tomography branch's compute_tomography argument. + The real transform's eager JIT and jackknife resampling take over 20 seconds. + """ + if ( + "compute_tomography" + not in inspect.signature(PureEBMixin.calculate_pure_eb).parameters + ): + pytest.skip("Bin-keyed calculate_pure_eb API exists only on tomography branch") + + for name in ( + "plot_integration_vs_reporting", + "plot_pure_eb_correlations", + "plot_pte_2d_heatmaps", + "plot_eb_covariance_matrix", + "save_pure_eb_results", + ): + if hasattr(pure_eb, name): + monkeypatch.setattr(pure_eb, name, lambda *args, **kwargs: None) + + seen = [] + real_statistics = pure_eb.calculate_eb_statistics + + def observe_statistics(results, **kwargs): + seen.append(sorted(results)[:6]) + return real_statistics(results, **kwargs) + + monkeypatch.setattr(pure_eb, "calculate_eb_statistics", observe_statistics) + cv = _TomoCV(tmp_path) + try: + cv.plot_pure_eb( + versions=["v"], + min_sep_int=0.5, + max_sep_int=200.0, + nbins_int=60, + npatch=32, + ) + except KeyError as error: + pytest.fail( + f"Statistics received keys {seen} rather than one per-bin result: {error}" + ) + assert set(cv._pure_eb_results["v"]) == {BIN_KEY} + pte = cv._pure_eb_results["v"][BIN_KEY]["pte_matrices"]["combined"] + assert np.isfinite(pte).any(), "No finite combined B-mode PTE produced" diff --git a/src/sp_validation/tests/regression/test_tomo_rho_tau_driver_external_checkout.py b/src/sp_validation/tests/regression/test_tomo_rho_tau_driver_external_checkout.py new file mode 100644 index 00000000..8343238f --- /dev/null +++ b/src/sp_validation/tests/regression/test_tomo_rho_tau_driver_external_checkout.py @@ -0,0 +1,101 @@ +"""The rho_tau_stats driver must use the checkout that launched the workflow.""" + +import io +import os +import runpy +import sys +import types +from pathlib import Path + +import pytest + +import sp_validation +import sp_validation.cosmo_val as cosmo_val_mod + + +class _Stop(Exception): + """Stop before any catalogue reads or output writes.""" + + +def test_rho_tau_driver_reads_cat_config_from_launched_checkout(monkeypatch, tmp_path): + """Resolve the catalogue config and output inside the launched checkout. + + The constructor stub records cwd and arguments, then stops before reading + survey data. Defaults './cat_config.yaml' and './output' resolve from cwd. + The expected checkout contains the imported sp_validation code; ordinarily + that is the repository containing this test, but a branch comparison can + import a different checkout while retaining this test file. + + Develop passes explicit paths from rule params and does not chdir, so this + test is unmarked: it guards against the tomography branch's regression of + changing cwd to an unrelated, hard-coded checkout and using defaults. + """ + checkout = Path(__file__).resolve().parents[4] + code_checkout = Path(sp_validation.__file__).resolve().parents[2] + if code_checkout != checkout: + checkout = code_checkout + driver = checkout / "workflow" / "scripts" / "run_rho_tau.py" + if not driver.is_file(): + pytest.skip("rho/tau workflow driver from the tomography branch is absent") + + seen = {} + + def fake_cv(*args, **kwargs): + seen["cwd"] = Path.cwd() + seen["kwargs"] = kwargs + raise _Stop + + monkeypatch.setattr(cosmo_val_mod, "CosmologyValidation", fake_cv) + fake_snk = types.SimpleNamespace( + params={ + "ver": "SP_v1.4.6.3", + "min_sep": "1.0", + "max_sep": "250.0", + "nbins": "20", + "npatch": "20", + "cat_config": str(checkout / "cosmo_val" / "cat_config.yaml"), + "output_dir": str(checkout / "cosmo_val" / "output"), + }, + output={}, + input={}, + ) + snk_script = types.ModuleType("snakemake.script") + snk_script.snakemake = fake_snk + monkeypatch.setitem(sys.modules, "snakemake.script", snk_script) + try: + import IPython + except ImportError: + pytest.skip("tomography branch driver requires IPython") + monkeypatch.setattr(IPython, "get_ipython", lambda: None) + # Stream unbuffering is unrelated to path resolution. Avoid closing pytest's + # captured descriptors when either driver generation calls os.fdopen. + monkeypatch.setattr(os, "fdopen", lambda *args, **kwargs: io.StringIO()) + # Register the original streams for restoration after the driver's assignment. + monkeypatch.setattr(sys, "stdout", sys.stdout) + monkeypatch.setattr(sys, "stderr", sys.stderr) + monkeypatch.syspath_prepend(str(driver.parent)) + monkeypatch.chdir(tmp_path) + + try: + with pytest.raises(_Stop): + runpy.run_path(str(driver), init_globals={"snakemake": fake_snk}) + except FileNotFoundError as exc: + # The tomography driver may chdir to a survey checkout absent on CI. + missing = Path(exc.filename) if exc.filename else None + if ( + missing + and missing.is_absolute() + and missing.parts[1] in {"n17data", "n09data", "n23data1", "home"} + ): + pytest.skip( + f"tomography branch external checkout is unavailable: {missing}" + ) + raise + + kw = seen["kwargs"] + cat_cfg = (seen["cwd"] / kw.get("catalog_config", "./cat_config.yaml")).resolve() + out_dir = (seen["cwd"] / (kw.get("output_dir") or "./output")).resolve() + assert cat_cfg.is_relative_to(checkout) and out_dir.is_relative_to(checkout), ( + f"driver used catalog_config={cat_cfg}, output_dir={out_dir}, " + f"cwd={seen['cwd']}; expected paths inside launched checkout {checkout}" + ) diff --git a/src/sp_validation/tests/regression/test_tomo_tau_centering_before_bin_mask.py b/src/sp_validation/tests/regression/test_tomo_tau_centering_before_bin_mask.py new file mode 100644 index 00000000..b789438c --- /dev/null +++ b/src/sp_validation/tests/regression/test_tomo_tau_centering_before_bin_mask.py @@ -0,0 +1,140 @@ +"""Tomographic tau statistics must centre galaxy ellipticities within each bin. + +The tomography branch passes a bin mask to TauStat. The mean subtracted must +be the selected bin's weighted mean, as in CovTauTh's per-bin covariance. +The shared dependency exposes this mask API on develop as well. +""" + +import numpy as np +import pytest + +rts = pytest.importorskip("shear_psf_leakage.rho_tau_stat") + +# e1 is constant in each bin; correct centring gives g1 == 0. The PSF has +# e2 == 0, so galaxy e2 noise cannot contribute to tau_0+ = . +E1_BIN = {1: 0.10, 2: 0.30} +E1_PSF = 0.10 + + +def _params(): + return { + "e1_col": "e1", + "e2_col": "e2", + "w_col": "w", + "ra_col": "RA", + "dec_col": "Dec", + "ra_PSF_col": "RA", + "dec_PSF_col": "Dec", + "e1_PSF_col": "E1_PSF", + "e2_PSF_col": "E2_PSF", + "e1_star_col": "E1_STAR", + "e2_star_col": "E2_STAR", + "PSF_size": "T_PSF", + "star_size": "T_STAR", + "patch_number": 2, + "patch_seed": 1, + "ra_units": "deg", + "dec_units": "deg", + } + + +def _galaxies(n=4000, seed=0): + rng = np.random.default_rng(seed) + gal = np.zeros( + n, + dtype=[ + ("RA", "f8"), + ("Dec", "f8"), + ("e1", "f8"), + ("e2", "f8"), + ("w", "f8"), + ("bin", "i4"), + ], + ) + gal["RA"] = rng.uniform(10, 11, n) + gal["Dec"] = rng.uniform(0, 1, n) + gal["bin"] = np.where(np.arange(n) % 2 == 0, 1, 2) + gal["e1"] = np.where(gal["bin"] == 1, E1_BIN[1], E1_BIN[2]) + gal["e2"] = rng.normal(0, 0.01, n) + gal["w"] = 1.0 + return gal + + +def _stars(n=2000, seed=1): + rng = np.random.default_rng(seed) + st = np.zeros( + n, + dtype=[ + ("RA", "f8"), + ("Dec", "f8"), + ("E1_PSF", "f8"), + ("E2_PSF", "f8"), + ("E1_STAR", "f8"), + ("E2_STAR", "f8"), + ("T_PSF", "f8"), + ("T_STAR", "f8"), + ], + ) + st["RA"] = rng.uniform(10, 11, n) + st["Dec"] = rng.uniform(0, 1, n) + st["E1_PSF"] = st["E1_STAR"] = E1_PSF + rng.normal(0, 0.01, n) + st["E2_STAR"] = rng.normal(0, 0.01, n) # Non-degenerate tau_2/tau_5 fields. + st["T_PSF"] = 1.0 + st["T_STAR"] = 1.0 + rng.normal(0, 0.01, n) + return st + + +@pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason="CosmoStat/shear_psf_leakage#52: centring precedes bin masking", +) +def test_masked_galaxy_catalog_is_centred_on_bin_mean(): + """A bin's TreeCorr galaxy catalogue must have zero weighted mean e1. + + Bin 2 has constant e1 = 0.3 and the full catalogue mean is 0.2. + Subtracting the bin mean gives zero by construction; centring before + masking instead leaves g1 = +0.1 on every bin-2 galaxy. + """ + gal = _galaxies() + cats = rts.Catalogs(params=_params()) + cats.build_catalog(gal, "gal", "gal_b2", mask=gal["bin"] == 2, npatch=2) + tc = cats.get_cat("gal_b2") + mean_g1 = np.average(tc.g1, weights=tc.w) + assert mean_g1 == pytest.approx(0.0, abs=1e-12), ( + f"bin-2 mean g1 = {mean_g1:.6f}; expected 0 after removing bin mean " + f"{E1_BIN[2]}, not the full-catalogue mean" + ) + + +@pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason="CosmoStat/shear_psf_leakage#52: full-catalogue mean biases bin tau", +) +def test_tomographic_tau0_plus_vanishes_for_bin_constant_ellipticity(tmp_path): + """TauStat must give tau_0+ = 0 for a bin with constant galaxy e1. + + Per-bin centring makes g1 identically zero and the PSF has p2 = 0, + so vanishes at every scale up to TreeCorr round-off. + Full-catalogue centring leaves (0.3 - 0.2) * 0.1 = 0.01 instead. + """ + gal, st = _galaxies(), _stars() + cfg = { + "ra_units": "deg", + "dec_units": "deg", + "sep_units": "arcmin", + "min_sep": 1.0, + "max_sep": 30.0, + "nbins": 5, + } + tau = rts.TauStat(params=_params(), output=str(tmp_path), treecorr_config=cfg) + tau.build_cat_to_compute_tau(st, cat_type="psf", catalog_id="v") + tau.build_cat_to_compute_tau( + gal, cat_type="gal", catalog_id="v", mask=gal["bin"] == 2 + ) + tau.compute_tau_stats("v", "tau_v.fits", var_method=None) + tau0p = np.asarray(tau.tau_stats["tau_0_p"]) + assert np.allclose(tau0p, 0.0, atol=1e-6), ( + f"bin-2 tau_0+ = {np.round(tau0p, 6).tolist()}; expected zero at all scales" + ) diff --git a/src/sp_validation/tests/test_column_schema.py b/src/sp_validation/tests/test_column_schema.py index 850de5b6..d4ad4cfd 100644 --- a/src/sp_validation/tests/test_column_schema.py +++ b/src/sp_validation/tests/test_column_schema.py @@ -61,6 +61,11 @@ "C22", "SNR", "T", + # NaMaster field/workspace dicts keyed by bin; cosmology parameter dicts + "W{}", + "H0", + # GLASS mock catalogues, which are not ShapePipe products + "TOM_BIN_ID", } #: An f-string template needs this many literal characters to count as diff --git a/src/sp_validation/tests/test_cosebis_tomography.py b/src/sp_validation/tests/test_cosebis_tomography.py new file mode 100644 index 00000000..d58c6696 --- /dev/null +++ b/src/sp_validation/tests/test_cosebis_tomography.py @@ -0,0 +1,30 @@ +"""Covariance selection at the tomographic COSEBIs boundary.""" + +from types import SimpleNamespace + +import pytest + +from sp_validation.cosmo_val.cosebis import CosebisMixin + + +@pytest.mark.parametrize( + "options", + [ + {}, + {"scale_cuts": [(1, 10)]}, + {"evaluate_all_scale_cuts": True}, + ], +) +def test_tomographic_cosebis_rejects_single_covariance(options): + calls = [] + cv = SimpleNamespace( + npatch=7, + print_start=lambda *args: None, + _binning=lambda *args: {}, + calculate_2pcf_version=lambda *args, **kwargs: calls.append(kwargs) or {}, + ) + with pytest.raises(ValueError, match="cov_path.*single.*tomographic bin pair"): + CosebisMixin.calculate_cosebis( + cv, "v", cov_path="xi_cov.txt", compute_tomography=True, **options + ) + assert calls == [] diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index c08c055e..7029cdcd 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -283,6 +283,7 @@ def _write_synthetic_catalogs( seed=1234, coherent_shear=False, with_psf=False, + with_tomography=False, ): """Write small deterministic FITS catalogs + dndz, return a config dict. @@ -300,6 +301,8 @@ def _write_synthetic_catalogs( with_psf : bool If True, add a ``psf`` config block (rho/tau / pseudo-Cl read it via ``get_params_rho_tau``). + with_tomography : bool + If True, add two tomographic bins to the shear catalogue. """ from astropy.table import Table @@ -325,9 +328,10 @@ def _write_synthetic_catalogs( w = rng.uniform(0.5, 1.0, n_gal) shear_path = cat_dir / "shear.fits" - Table({"RA": ra, "Dec": dec, "e1": e1, "e2": e2, "w": w}).write( - shear_path, overwrite=True - ) + shear_data = {"RA": ra, "Dec": dec, "e1": e1, "e2": e2, "w": w} + if with_tomography: + shear_data["tomo_bin_id"] = rng.integers(1, 3, n_gal) + Table(shear_data).write(shear_path, overwrite=True) star_path = cat_dir / "star.fits" Table( @@ -362,6 +366,8 @@ def _write_synthetic_catalogs( "e1_col_corrected": "e1", "e2_col_corrected": "e2", } + if with_tomography: + shear_cfg["tomo_bin_col"] = "tomo_bin_id" star_cfg = { "path": "star.fits", "ra_col": "RA", @@ -372,6 +378,7 @@ def _write_synthetic_catalogs( version_cfg = { "subdir": str(cat_dir), "pipeline": "SP", + "colour": "tab:blue", "shear": shear_cfg, "star": star_cfg, } @@ -429,13 +436,14 @@ def test_calculate_2pcf_runs_on_synthetic_catalog(self, tmp_path): **params, ) - gg = cv.calculate_2pcf(version) + ggs = cv.calculate_2pcf_version(version) # treecorr GGCorrelation with xi+/- on the configured angular grid - assert gg.xip.shape == (nbins,) - assert gg.xim.shape == (nbins,) - assert np.all(np.isfinite(gg.xip)) - assert np.all(np.isfinite(gg.xim)) + # LG:Hardcoded to be non-tomographic for now + assert ggs["tomo_bin_all_tomo_bin_all"].xip.shape == (nbins,) + assert ggs["tomo_bin_all_tomo_bin_all"].xim.shape == (nbins,) + assert np.all(np.isfinite(ggs["tomo_bin_all_tomo_bin_all"].xip)) + assert np.all(np.isfinite(ggs["tomo_bin_all_tomo_bin_all"].xim)) # The additive-bias subtraction in the pipeline must have run. assert version in cv.c1 and version in cv.c2 @@ -483,66 +491,130 @@ def test_xi_part_carries_the_covariance_the_measurement_estimated( ) def test_a_patched_xi_dump_reads_back(self, tmp_path): - """calculate_2pcf reads back the text dump a patched measurement wrote. + """calculate_2pcf_version reads back the dump a patched measurement wrote. The ξ± figure rules re-enter calculate_2pcf on the reporting grid, which has patches, and are handed the dump rule xi wrote (to its precision). """ params, version = self._write_synthetic_catalogs(tmp_path) binning = dict(npatch=4, min_sep=5.0, max_sep=100.0, nbins=6) - measured = CosmologyValidation(versions=[version], **params).calculate_2pcf( - version, **binning - ) - read = CosmologyValidation(versions=[version], **params).calculate_2pcf( + pair = "tomo_bin_all_tomo_bin_all" + measured = CosmologyValidation( + versions=[version], **params + ).calculate_2pcf_version(version, **binning)[pair] + dumps = list(Path(params["output_dir"]).glob(f"xi_{version}_*.txt")) + assert len(dumps) == 1 + read = CosmologyValidation(versions=[version], **params).calculate_2pcf_version( version, **binning - ) + )[pair] for column in ("meanr", "npairs", "xip", "xim", "varxip", "varxim"): np.testing.assert_allclose( getattr(read, column), getattr(measured, column), rtol=1e-4 ) - def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): - """calculate_2pcf's ξ± is the same on 4 and on 48 TreeCorr threads. - - Production binning (default bin_slop/angle_slop), both runs on the - jackknife patches the first one writes, each from a fresh Catalog; they - must agree to far below the jackknife σ. - """ + def test_calculate_2pcf_is_reproducible_across_machines( + self, tmp_path, monkeypatch + ): + """Fresh calculate_2pcf_version runs share patches and ξ± across CPU counts.""" import treecorr + import treecorr.field + + patches = [] + process = treecorr.GGCorrelation.process + + def recording_process(gg, cat, *args, **kwargs): + patches.append(np.array(cat.patch)) + return process(gg, cat, *args, **kwargs) + + monkeypatch.setattr(treecorr.GGCorrelation, "process", recording_process) + + xi, var, counts = {}, {}, {} + for tree, n_cpu in (("a", 4), ("b", 16)): + monkeypatch.setattr(treecorr.field, "get_omp_threads", lambda n=n_cpu: n) + run_dir = tmp_path / tree + run_dir.mkdir() + params, version = self._write_synthetic_catalogs( + run_dir, + n_gal=4000, + ra_range=(0.0, 60.0), + dec_range=(-10.0, 30.0), + coherent_shear=True, + ) + gg = CosmologyValidation( + versions=[version], + npatch=100, + theta_min=15.0, + theta_max=70.0, + nbins=6, + **params, + ).calculate_2pcf_version(version, num_threads=n_cpu)[ + "tomo_bin_all_tomo_bin_all" + ] + xi[tree] = np.concatenate([gg.xip, gg.xim]) + var[tree] = np.concatenate([gg.varxip, gg.varxim]) + counts[tree] = {key: result.npairs for key, result in gg.results.items()} + + np.testing.assert_array_equal(patches[0], patches[1]) + assert counts["a"].keys() == counts["b"].keys() + for key in counts["a"]: + np.testing.assert_array_equal(counts["a"][key], counts["b"][key]) + np.testing.assert_allclose(xi["a"], xi["b"], rtol=0, atol=1e-12) + np.testing.assert_allclose(var["a"], var["b"], rtol=1e-10) + + def test_cross_tomographic_pair_shares_patch_centres(self, tmp_path, monkeypatch): + """Both catalogues in a cross-bin pair use the full-sample centres.""" + import treecorr + + from sp_validation.statistics import jackknife_patch_centers params, version = self._write_synthetic_catalogs( - tmp_path, n_gal=4000, coherent_shear=True + tmp_path, n_gal=400, with_tomography=True ) - cv = CosmologyValidation( - versions=[version], - npatch=8, - theta_min=15.0, - theta_max=70.0, - nbins=6, - **params, + cv = CosmologyValidation(versions=[version], npatch=4, **params) + cols = cv._shear_columns(version, compute_tomography=True) + full_catalog = treecorr.Catalog( + ra=cols["ra"], + dec=cols["dec"], + w=cols["w"], + ra_units=cv.treecorr_config["ra_units"], + dec_units=cv.treecorr_config["dec_units"], + ) + expected_centers = jackknife_patch_centers(full_catalog, 4) + catalogs = [] + make_catalog = cv._bin_catalog + + def recording_bin_catalog(cols, bin_id, npatch, patch_centers=None): + catalog = make_catalog(cols, bin_id, npatch, patch_centers) + catalogs.append((bin_id, patch_centers, np.array(catalog._centers))) + return catalog + + monkeypatch.setattr(cv, "_bin_catalog", recording_bin_catalog) + cv.calculate_2pcf_version(version, npatch=4, compute_tomography=True) + + assert len(catalogs) == 4 + cross_pair_catalogs = catalogs[1:3] + assert [entry[0] for entry in cross_pair_catalogs] == [1, 2] + np.testing.assert_array_equal( + cross_pair_catalogs[0][1], cross_pair_catalogs[1][1] + ) + np.testing.assert_array_equal( + cross_pair_catalogs[0][2], cross_pair_catalogs[1][2] + ) + np.testing.assert_allclose( + cross_pair_catalogs[0][2], expected_centers, rtol=0, atol=1e-14 + ) + np.testing.assert_allclose( + cross_pair_catalogs[1][2], expected_centers, rtol=0, atol=1e-14 ) - - xi = {} - for n_threads in (4, 48): - # calculate_2pcf reads back an existing text dump instead of measuring. - for dump in Path(params["output_dir"]).glob(f"{version}_xi_*.txt"): - dump.unlink() - gg = cv.calculate_2pcf(version, num_threads=n_threads) - assert treecorr.get_omp_threads() == n_threads # the count took effect - xi[n_threads] = np.concatenate([gg.xip, gg.xim]) - sigma = np.sqrt(np.concatenate([gg.varxip, gg.varxim])) - - shift = np.max(np.abs(xi[48] - xi[4]) / sigma) - assert shift < 1e-6, f"ξ± moves by {shift:.3g}σ between 4 and 48 threads" def test_treecorr_runs_on_the_cpus_the_process_holds(self, tmp_path): """By default TreeCorr takes the process's CPU affinity, not the node's count.""" import treecorr params, version = self._write_synthetic_catalogs(tmp_path) - CosmologyValidation(versions=[version], npatch=1, **params).calculate_2pcf( - version - ) + CosmologyValidation( + versions=[version], npatch=1, **params + ).calculate_2pcf_version(version) assert treecorr.get_omp_threads() == len(os.sched_getaffinity(0)) def test_calculate_scale_dependent_leakage_runs_on_synthetic_catalog( @@ -616,13 +688,10 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path, pure_eb_xi) ) cv.treecorr_config.update(bin_slop=0, angle_slop=0) - results = cv.calculate_pure_eb( - version, - npatch=npatch, - min_sep_int=1.0, - max_sep_int=300.0, - nbins_int=600, - ) + integration = dict(min_sep_int=1.0, max_sep_int=300.0, nbins_int=600) + results = cv.calculate_pure_eb(version, npatch=npatch, **integration)[ + "tomo_bin_all_tomo_bin_all" + ] measured = { key: results[key] @@ -651,7 +720,17 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path, pure_eb_xi) cov = np.asarray(results["cov"]) assert cov.shape == (6 * nbins, 6 * nbins) assert results["npatch"] == npatch - gg_int = cv.cat_ggs[version] + # The integration-grid ξ± again, measured on the jackknife patches the + # first measurement wrote (its columns-only dump carries no patches). + for dump in Path(params["output_dir"]).glob(f"xi_{version}_*.txt"): + dump.unlink() + gg_int = cv.calculate_2pcf_version( + version, + npatch=npatch, + min_sep=integration["min_sep_int"], + max_sep=integration["max_sep_int"], + nbins=integration["nbins_int"], + )["tomo_bin_all_tomo_bin_all"] jackknife_of_modes = treecorr.estimate_multi_cov( [gg_int], "jackknife", @@ -661,3 +740,130 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path, pure_eb_xi) np.testing.assert_allclose( cov, jackknife_of_modes, rtol=0, atol=1e-10 * np.abs(cov).max() ) + + def test_plot_2pcf_writes_the_non_tomographic_figures(self, tmp_path): + """plot_2pcf draws the ("all", "all") ξ± through plot_2pcf_tomography.""" + params, version = self._write_synthetic_catalogs(tmp_path) + cv = CosmologyValidation( + versions=[version], + npatch=1, + theta_min=5.0, + theta_max=100.0, + nbins=6, + **params, + ) + cv.plot_2pcf(show=False) + + out = Path(params["output_dir"]) + for name in ("xi_pm_tomography_False", "xi_pm_theta_tomography_False"): + assert (out / f"{name}.png").is_file(), name + + def test_plot_ratio_xi_sys_xi_writes_the_declared_figure(self, tmp_path): + """plot_ratio_xi_sys_xi divides the pair's ξ^{PSF, sys}± by its ξ±. + + ξ^{PSF, sys} is set on the instance, so the test needs no ρ/τ fit; the + figure lands where the cv_ratio_xi_sys_xi rule declares it. + """ + params, version = self._write_synthetic_catalogs(tmp_path) + nbins = 6 + cv = CosmologyValidation( + versions=[version], + npatch=1, + theta_min=5.0, + theta_max=100.0, + nbins=nbins, + **params, + ) + sys = np.full(nbins, 1e-6) + cv._xi_psf_sys = { + version: { + "tomo_bin_all_tomo_bin_all": { + "mean_plus": sys, + "var_plus": sys**2, + "mean_minus": sys, + "var_minus": sys**2, + } + } + } + cv.plot_ratio_xi_sys_xi(show=False) + + assert (Path(params["output_dir"]) / "ratio_xi_sys_xi.png").is_file() + with pytest.raises(ValueError, match="compute_tomography"): + cv.plot_ratio_xi_sys_xi(tomography=True, show=False) + + def test_summarize_bmodes_reads_the_all_pair(self, tmp_path): + """Each statistic's ("all", "all") result reaches the summary. + + A result whose shape the summary cannot read raises rather than + silently leaving its column empty. + """ + from types import SimpleNamespace + + from sp_validation.statistics import chi2_and_pte + + params, version = self._write_synthetic_catalogs(tmp_path) + cv = CosmologyValidation(versions=[version], **params) + pair = "tomo_bin_all_tomo_bin_all" + + edges = np.geomspace(1.0, 100.0, 6) + cv._pure_eb_results[version] = { + pair: { + "left_edges": edges[:-1], + "right_edges": edges[1:], + "pte_matrices": { + stat: np.full((5, 5), pte) + for stat, pte in (("xip_B", 0.1), ("xim_B", 0.2), ("combined", 0.3)) + }, + "npatch": 8, + } + } + cv._cosebis_results[version] = {pair: {"pte_B": 0.4}} + cl_bb = np.array([1.0, -0.5, 0.25]) + cov_bb = np.diag([2.0, 1.0, 0.5]) + cv._pseudo_cls = { + version: { + pair: { + "pseudo_cl": {"BB": cl_bb}, + "cov": {"COVAR_BB_BB": SimpleNamespace(data=cov_bb)}, + } + } + } + + row = cv.summarize_bmodes(fiducial_scale_cut=(1.0, 100.0))[version] + + assert row == pytest.approx( + { + "xip_B": 0.1, + "xim_B": 0.2, + "combined": 0.3, + "COSEBIS": 0.4, + "C_l_BB": chi2_and_pte(cl_bb, cov_bb)[2], + } + ) + + del cv._pure_eb_results[version][pair]["pte_matrices"] + with pytest.raises(KeyError): + cv.summarize_bmodes(fiducial_scale_cut=(1.0, 100.0)) + + def test_sacc_nz_is_the_summed_tomographic_nz(self, tmp_path): + """A multi-column n(z) file gives the SACC parts its whole-survey sum.""" + params, version = self._write_synthetic_catalogs(tmp_path) + cv = CosmologyValidation(versions=[version], **params) + + z = np.linspace(0.0, 2.0, 11) + nz_bins = np.stack([np.exp(-((z - mu) ** 2)) for mu in (0.5, 1.0, 1.5)]) + nz_path = tmp_path / "nz_tomo.txt" + np.savetxt(nz_path, np.column_stack([z, *nz_bins])) + cv.cc[version]["shear"]["redshift_path"] = str(nz_path) + + [(bin_id, (z_read, nz_read))] = cv.sacc_nz(version).items() + assert bin_id == 0 + np.testing.assert_allclose(z_read, z) + np.testing.assert_allclose(nz_read, nz_bins.sum(axis=0)) + + @pytest.mark.parametrize("pol_factor", [True, False, 0, 2]) + def test_pol_factor_must_be_plus_or_minus_one(self, tmp_path, pol_factor): + """pol_factor is a ±1 e2 multiplier; a bool would pass `in (-1, 1)`.""" + params, version = self._write_synthetic_catalogs(tmp_path) + with pytest.raises(ValueError, match="pol_factor"): + CosmologyValidation(versions=[version], pol_factor=pol_factor, **params) diff --git a/src/sp_validation/tests/test_cv_init_params.py b/src/sp_validation/tests/test_cv_init_params.py index ffc87d8b..5d1a045d 100644 --- a/src/sp_validation/tests/test_cv_init_params.py +++ b/src/sp_validation/tests/test_cv_init_params.py @@ -1,14 +1,16 @@ -"""Guard: the workflow sets every CosmologyValidation constructor default. +"""Guard: the workflow builds ``cv`` and names its products as the class does. ``workflow/common.py::cv_init_params`` builds the constructor kwargs for every cosmo_val rule. A keyword it does not forward falls back to the constructor default without any trace in the run config, so each keyword with a default is -either forwarded or exempted below with its reason. +either forwarded or exempted below with its reason. ``cv_basename`` names the +rule outputs, so it must spell ``CosmologyValidation.basename`` exactly. """ import importlib.util import inspect from pathlib import Path +from types import SimpleNamespace import yaml @@ -16,7 +18,15 @@ REPO = Path(__file__).resolve().parents[3] -EXEMPT = {} +EXEMPT = { + # The workflow is the non-tomographic DAG: every rule script names the bin + # pair it computes, and the flag only switches the lazy properties on to + # tomographic products no rule declares. + "compute_tomography": "rule scripts pass tomography per call", + # Snakemake owns staleness: it removes a job's declared outputs before + # running it, so the methods' skip-if-exists only ever reuses intermediates. + "force_run": "Snakemake decides what reruns", +} def _load_common(): @@ -49,3 +59,14 @@ def test_every_default_is_forwarded_or_exempt(): def test_exemptions_are_live_keywords(): stale = set(EXEMPT) - _defaulted_keywords() assert not stale, f"EXEMPT names keywords the constructor lacks: {sorted(stale)}" + + +def test_cv_basename_is_the_class_basename(): + fiducial = {"min_sep": 1.0, "max_sep": 250.0, "nbins": 20, "npatch": 100} + cv = SimpleNamespace( + treecorr_config={k: fiducial[k] for k in ("min_sep", "max_sep", "nbins")}, + npatch=fiducial["npatch"], + ) + assert _load_common().cv_basename( + "SP_v1.4.6.3", fiducial + ) == CosmologyValidation.basename(cv, "SP_v1.4.6.3") diff --git a/src/sp_validation/tests/test_glass_mock.py b/src/sp_validation/tests/test_glass_mock.py index 233d8aa1..54014403 100644 --- a/src/sp_validation/tests/test_glass_mock.py +++ b/src/sp_validation/tests/test_glass_mock.py @@ -133,16 +133,13 @@ def test_config_change_breaks_reference(): @pytest.mark.skipif(not HAVE_GLASS, reason="GLASS not installed in this image") @pytest.mark.xfail( reason=( - "glass_mock map path is incompatible with the installed glass/cosmology " - "API: cosmology.Cosmology.from_camb returns a CambCosmology lacking " - "comoving_distance, which glass.distance_grid / MultiPlaneConvergence " - "require. The map path was never exercised before GLASS was added to the " - "image. Fix = pin a compatible glass+cosmology pair (or adapt the API " - "calls) and verify in the fresh image; then drop this xfail. " - "See fiber shapepipe/sp_validation glass-cosmology-api-pin." + "glass_mock builds its cosmology with cosmology.compat.camb, which an image " + "built before the lock carried glass 2026.2 + cosmology-compat-camb 0.2.0 " + "lacks. With those locked pins on the path the test passes; drop this " + "xfail once it XPASSes in the image built from the current uv.lock." ), strict=False, - raises=AttributeError, + raises=ModuleNotFoundError, ) def test_matter_maps_are_seed_deterministic(): """Same config + seed → bit-identical matter/lensing maps. diff --git a/src/sp_validation/tests/test_grammar.py b/src/sp_validation/tests/test_grammar.py index 08ed6742..a855e73b 100644 --- a/src/sp_validation/tests/test_grammar.py +++ b/src/sp_validation/tests/test_grammar.py @@ -273,7 +273,7 @@ def test_get_rho_tau_identical_for_v1_and_v2_psf_catalogues(tmp_path): "max_sep": 600, "nbins": 4, } - get_rho_tau(config, label, treecorr_config, str(outdir), label) + get_rho_tau(config, label, treecorr_config, str(outdir), label, label) stats[label] = [ fits.getdata(outdir / f"{kind}_stats_{label}.fits") for kind in ("rho", "tau") @@ -285,6 +285,52 @@ def test_get_rho_tau_identical_for_v1_and_v2_psf_catalogues(tmp_path): np.testing.assert_allclose(table_v1[name], table_v2[name], rtol=1e-10) +def test_jackknife_cov_reads_the_draws_it_writes(tmp_path): + """The jackknife ρ/τ covariance averages the per-draw files the library saves. + + shear_psf_leakage writes each draw to cov_{rho,tau}_{catalog_id}.npy; the + reader must look for those names, and leave only the averaged products. + """ + from sp_validation.rho_tau import get_jackknife_cov + + _, v2 = _twins() + rng = np.random.default_rng(5) + shear = np.empty(N, dtype=[(n, "f8") for n in ("RA", "Dec", "e1", "e2", "w")]) + shear["RA"], shear["Dec"] = v2["RA"], v2["DEC"] + shear["e1"], shear["e2"] = rng.normal(0, 0.3, (2, N)) + shear["w"] = 1.0 + fits.BinTableHDU(shear).writeto(tmp_path / "shear.fits") + fits.BinTableHDU(np.array(v2[list(PSF_BLOCK.values())])).writeto( + tmp_path / "psf.fits" + ) + config = { + "v": { + "patch_number": 2, + "psf": PSF_BLOCK | {"path": str(tmp_path / "psf.fits"), "hdu": 1}, + "shear": SHEAR_BLOCK | {"path": str(tmp_path / "shear.fits")}, + } + } + treecorr_config = { + "ra_units": "deg", + "dec_units": "deg", + "sep_units": "arcmin", + "min_sep": 10, + "max_sep": 600, + "nbins": 4, + } + outdir = tmp_path / "out" + outdir.mkdir() + get_jackknife_cov( + config, "v", treecorr_config, str(outdir), "v", "v_bin_1", npatch=2, ncov=2 + ) + + cov_tau = np.load(outdir / "cov_tau_v_bin_1_jk.npy") + cov_rho = np.load(outdir / "cov_rho_v_jk.npy") + assert cov_tau.shape == (3 * 4, 3 * 4) + assert cov_rho.shape == (6 * 4, 6 * 4) + assert not list(outdir.glob("cov_*[0-9].npy")) + + # -- mask-bit columns: a rule family of their own --------------------------- diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index ca6bbb1f..e3dae689 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -51,6 +51,7 @@ """ import os +from pathlib import Path import numpy as np import numpy.testing as npt @@ -59,7 +60,12 @@ from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation +from sp_validation.cosmo_val.pseudo_cl import _apply_pixel_window_to_fiducial_cl from sp_validation.cosmo_val.sacc_writers import BIN as SACC_BIN +from sp_validation.pseudo_cl import ( + apply_random_rotation, + bandpower_window_from_workspace, +) from sp_validation.rho_tau import get_params_rho_tau # These tests need the full harmonic-space stack (pymaster/NaMaster + healpy), @@ -75,6 +81,8 @@ SEED = 1234 N_ELL_BINS = 8 +REFERENCE_TOMO = Path(__file__).parent / "data" / "test_cl_catalog.npz" + # --------------------------------------------------------------------------- # Synthetic-catalog fixture (deterministic; no cluster data) @@ -103,9 +111,10 @@ def _write_synthetic_config(tmp_path): e1 = rng.normal(0, 0.25, n_gal) e2 = rng.normal(0, 0.25, n_gal) w = rng.uniform(0.5, 1.0, n_gal) - Table({"RA": ra, "Dec": dec, "e1": e1, "e2": e2, "w": w}).write( - cat_dir / "shear.fits", overwrite=True - ) + tomo_bin_id = rng.integers(1, 3, n_gal) # Create a two bin catalogue + Table( + {"RA": ra, "Dec": dec, "e1": e1, "e2": e2, "w": w, "tomo_bin_id": tomo_bin_id} + ).write(cat_dir / "shear.fits", overwrite=True) z_edges = np.linspace(0.05, 3.0, 31) dndz = np.exp(-(((z_edges - 0.7) / 0.3) ** 2)) @@ -123,6 +132,7 @@ def _write_synthetic_config(tmp_path): "e2_col_corrected": "e2", "ra_col": "RA", "dec_col": "Dec", + "tomo_bin_col": "tomo_bin_id", } # Minimal psf block so get_params_rho_tau() succeeds; the pseudo-Cl # primitives only read the shear-side keys, but the production call path @@ -169,7 +179,7 @@ def cv(tmp_path): binning="powspace", power=0.5, n_ell_bins=N_ELL_BINS, - pol_factor=True, + pol_factor=-1, ) cv._test_version = version return cv @@ -179,11 +189,16 @@ def cv(tmp_path): def cat_and_params(cv): """(catalog ndarray, params dict) for the synthetic version.""" ver = cv._test_version - params = get_params_rho_tau(cv.cc[ver], survey=ver) + params = get_params_rho_tau(cv.cc[ver]) cat_gal = fits.getdata(cv.cc[ver]["shear"]["path"]) return cat_gal, params +def test_reference_exists(): + """Guard the guard: a missing reference must fail loudly, not skip.""" + assert REFERENCE_TOMO.exists(), f"committed reference missing: {REFERENCE_TOMO}" + + # Tolerances ---------------------------------------------------------------- # Bitwise-stable primitives (binning math, n_gal map, map-based pseudo-Cl). RTOL_DET = 1e-9 @@ -269,18 +284,38 @@ def test_unknown_binning_raises(self, cv): cv.get_namaster_bin(LMIN, LMAX, B_LMAX) +# ========================================================================== +# get_pixels -- fetch the pixels from ra, dec, and nside (HEALPix) +# ========================================================================== +def test_get_pixels(cv, cat_and_params): + """Pin the HEALPix pixelization of the synthetic catalog.""" + cat_gal, params = cat_and_params + unique_pix, idx, idx_rep = cv.get_pixels(params, NSIDE, cat_gal) + + # Structural invariants of the HEALPix occupancy map. + assert unique_pix.size == 485 + assert idx_rep.size == 5000 # one entry per galaxy + + # Pinned scalar summaries: total weight is conserved (sum of weights), + # peak occupancy, and the pixel index bookkeeping. + npt.assert_array_equal(unique_pix[:5], [12167, 12168, 12169, 12170, 12171]) + assert int(unique_pix.sum()) == 7849989 + + # =========================================================================== # get_n_gal_map -- weighted galaxy number-density map # =========================================================================== def test_get_n_gal_map(cv, cat_and_params): cat_gal, params = cat_and_params - n_gal, unique_pix, idx, idx_rep = cv.get_n_gal_map(params, NSIDE, cat_gal) + + unique_pix, idx, idx_rep = cv.get_pixels(params, NSIDE, cat_gal) + n_gal = cv.get_n_gal_map( + params, NSIDE, cat_gal, unique_pix=unique_pix, idx=None, idx_rep=idx_rep + ) # Structural invariants of the HEALPix occupancy map. assert n_gal.shape == (healpy.nside2npix(NSIDE),) assert int(np.count_nonzero(n_gal)) == 485 - assert unique_pix.size == 485 - assert idx_rep.size == 5000 # one entry per galaxy # The map is supported exactly on the occupied pixels. npt.assert_array_equal(np.nonzero(n_gal)[0], np.sort(unique_pix)) @@ -288,8 +323,6 @@ def test_get_n_gal_map(cv, cat_and_params): # peak occupancy, and the pixel index bookkeeping. npt.assert_allclose(n_gal.sum(), 3760.657282591494, rtol=RTOL_DET) npt.assert_allclose(n_gal.max(), 16.063410433711447, rtol=RTOL_DET) - npt.assert_array_equal(unique_pix[:5], [12167, 12168, 12169, 12170, 12171]) - assert int(unique_pix.sum()) == 7849989 npt.assert_allclose( n_gal[unique_pix][:5], np.array( @@ -311,7 +344,10 @@ def test_get_n_gal_map(cv, cat_and_params): # =========================================================================== def _build_shear_map(cv, cat_gal, params): """Replicate calculate_pseudo_cl_map's weighted shear-map construction.""" - n_gal, unique_pix, _idx, idx_rep = cv.get_n_gal_map(params, NSIDE, cat_gal) + unique_pix, _idx, idx_rep = cv.get_pixels(params, NSIDE, cat_gal) + n_gal = cv.get_n_gal_map( + params, NSIDE, cat_gal, unique_pix=unique_pix, idx=_idx, idx_rep=idx_rep + ) w = cat_gal[params["w_col"]] e1 = cat_gal[params["e1_col"]] e2 = cat_gal[params["e2_col"]] @@ -325,10 +361,32 @@ def _build_shear_map(cv, cat_gal, params): return m1 + 1j * m2, n_gal +def test_build_shear_map(cv, cat_and_params): + """Pin the weighted shear map construction from the synthetic catalog.""" + cat_gal, params = cat_and_params + unique_pix, _idx, idx_rep = cv.get_pixels(params, NSIDE, cat_gal) + n_gal = cv.get_n_gal_map( + params, NSIDE, cat_gal, unique_pix=unique_pix, idx=_idx, idx_rep=idx_rep + ) + m1, m2 = cv.get_shear_map( + params, NSIDE, cat_gal, unique_pix=unique_pix, idx=_idx, idx_rep=idx_rep + ) + + m = m1 + 1j * m2 + + m_replicate, n_gal_replicate = _build_shear_map(cv, cat_gal, params) + + npt.assert_allclose(m, m_replicate, rtol=RTOL_DET, atol=ATOL_DET) + npt.assert_allclose(n_gal, n_gal_replicate, rtol=RTOL_DET, atol=ATOL_DET) + + def test_get_pseudo_cls_map(cv, cat_and_params): cat_gal, params = cat_and_params shear_map, n_gal = _build_shear_map(cv, cat_gal, params) ell_eff, cl_all, wsp = cv.get_pseudo_cls_map(shear_map, n_gal) + ell_eff_2, cl_all_2, wsp_2 = cv.get_pseudo_cls_map( + shear_map, n_gal, shear_map_b=shear_map, mask_b=n_gal + ) assert cl_all.shape == (4, N_ELL_BINS) npt.assert_allclose( @@ -391,13 +449,106 @@ def test_get_pseudo_cls_map(cv, cat_and_params): # BE (index 2) is the transpose-symmetric partner of EB for an auto-spectrum. npt.assert_allclose(cl_all[2], cl_all[1], rtol=RTOL_DET, atol=1e-18) + # Assert that running the same map against itself and against itself as a second map gives the same result. + npt.assert_allclose(cl_all, cl_all_2, rtol=RTOL_DET, atol=ATOL_DET) + npt.assert_allclose(ell_eff, ell_eff_2, rtol=RTOL_DET, atol=ATOL_DET) + + +def test_get_pseudo_cls_map_with_tomo(cv, cat_and_params): + cat_gal, params = cat_and_params + cat_gal_tomo1 = cat_gal[cat_gal[params["tomo_bin_col"]] == 1] + cat_gal_tomo2 = cat_gal[cat_gal[params["tomo_bin_col"]] == 2] + shear_map_1, n_gal_1 = _build_shear_map(cv, cat_gal_tomo1, params) + shear_map_2, n_gal_2 = _build_shear_map(cv, cat_gal_tomo2, params) + ell_eff, cl_all, wsp = cv.get_pseudo_cls_map( + shear_map_1, n_gal_1, shear_map_b=shear_map_2, mask_b=n_gal_2 + ) + + assert cl_all.shape == (4, N_ELL_BINS) + npt.assert_allclose( + ell_eff, + np.array([11.5, 20.0, 30.5, 43.5, 58.5, 75.5, 95.0, 116.5]), + rtol=RTOL_DET, + atol=ATOL_DET, + ) + npt.assert_allclose( + cl_all[0], # EE + np.array( + [ + -8.648947250571888e-06, + 4.386882223005456e-06, + -2.3193526251107696e-06, + 1.0651397314115168e-06, + -2.722591256637468e-07, + -3.862270431501105e-07, + -1.40420405782524e-06, + 2.1531429566476774e-07, + ] + ), + rtol=RTOL_DET, + atol=ATOL_DET, + ) + npt.assert_allclose( + cl_all[1], # EB + np.array( + [ + -2.8032134416262087e-07, + -1.271298363785575e-06, + 1.7112863041724342e-08, + -2.899403854283714e-07, + 1.360217254538336e-06, + 7.682614612656511e-08, + 6.620365648170535e-07, + -3.164479179586416e-07, + ] + ), + rtol=RTOL_DET, + atol=ATOL_DET, + ) + npt.assert_allclose( + cl_all[2], # BE + np.array( + [ + -3.83934537153214e-06, + 6.218772676611559e-06, + -5.542920986484232e-06, + 8.810792449470091e-07, + -3.2973293732012423e-07, + -1.3155110757359602e-08, + -1.420039280183438e-07, + 3.09464700816748e-07, + ] + ), + rtol=RTOL_DET, + atol=ATOL_DET, + ) + npt.assert_allclose( + cl_all[3], # BB + np.array( + [ + 9.387700364892905e-06, + -2.9045253374872504e-06, + -1.7006849005134948e-06, + -3.6869001010423035e-07, + 4.4986878855942184e-07, + 6.021221048891767e-07, + 8.413198154662088e-08, + -5.782957119801217e-07, + ] + ), + rtol=RTOL_DET, + atol=ATOL_DET, + ) + # =========================================================================== # get_pseudo_cls_catalog -- NmtFieldCatalog path (drifts ~2e-12) # =========================================================================== def test_get_pseudo_cls_catalog(cv, cat_and_params): cat_gal, params = cat_and_params - ell_eff, cl_all, wsp = cv.get_pseudo_cls_catalog(catalog=cat_gal, params=params) + ell_eff, cl_all, wsp = cv.get_pseudo_cls_catalog( + catalog=cat_gal, params=params, tomo_bin_a="all", tomo_bin_b="all" + ) assert cl_all.shape == (4, N_ELL_BINS) # Effective ells share the binning math with the map path: bitwise-stable. @@ -462,6 +613,16 @@ def test_get_pseudo_cls_catalog(cv, cat_and_params): npt.assert_allclose(cl_all[2], cl_all[1], rtol=RTOL_CAT, atol=ATOL_CAT) +def test_get_pseudo_cls_catalog_defaults_to_the_whole_catalogue(cv, cat_and_params): + """Called without bins, the wrapper measures the ("all", "all") pair.""" + cat_gal, params = cat_and_params + _, cl_default, _ = cv.get_pseudo_cls_catalog(catalog=cat_gal, params=params) + _, cl_all, _ = cv.get_pseudo_cls_catalog( + catalog=cat_gal, params=params, tomo_bin_a="all", tomo_bin_b="all" + ) + npt.assert_allclose(cl_default, cl_all, rtol=RTOL_CAT, atol=ATOL_CAT) + + # =========================================================================== # apply_random_rotation -- invariant + reproducibility # =========================================================================== @@ -475,7 +636,7 @@ def test_apply_random_rotation_preserves_magnitude(cv, cat_and_params): e1 = np.asarray(cat_gal[params["e1_col"]], dtype=np.float64) e2 = np.asarray(cat_gal[params["e2_col"]], dtype=np.float64) - e1_rot, e2_rot = cv.apply_random_rotation(e1, e2) + e1_rot, e2_rot = apply_random_rotation(e1, e2) assert e1_rot.shape == e1.shape assert e2_rot.shape == e2.shape @@ -495,37 +656,56 @@ def test_apply_random_rotation_reproducible_with_seed(cv, cat_and_params): e1 = np.asarray(cat_gal[params["e1_col"]], dtype=np.float64) e2 = np.asarray(cat_gal[params["e2_col"]], dtype=np.float64) - a1, a2 = cv.apply_random_rotation(e1, e2, np.random.default_rng(42)) - b1, b2 = cv.apply_random_rotation(e1, e2, np.random.default_rng(42)) + a1, a2 = apply_random_rotation(e1, e2, np.random.default_rng(42)) + b1, b2 = apply_random_rotation(e1, e2, np.random.default_rng(42)) npt.assert_array_equal(a1, b1) npt.assert_array_equal(a2, b2) - c1, _ = cv.apply_random_rotation(e1, e2, np.random.default_rng(7)) + c1, _ = apply_random_rotation(e1, e2, np.random.default_rng(7)) assert not np.allclose(a1, c1) - d1, _ = cv.apply_random_rotation(e1, e2) - f1, _ = cv.apply_random_rotation(e1, e2) + d1, _ = apply_random_rotation(e1, e2) + f1, _ = apply_random_rotation(e1, e2) assert not np.allclose(d1, f1) # =========================================================================== # calculate_pseudo_cl_catalog -- deterministic end-to-end catalog path # =========================================================================== -def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): - """End-to-end catalog path: SACC round-trip of ell + EE/EB/BB. - - The catalog method has no random noise debiasing, so it is reproducible to - the same ~2e-12 catalog-path float noise; we pin the round-tripped spectra. - """ +def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path, monkeypatch): + """Catalog SACC output retains the unwindowed workspace bandpower window.""" ver = cv._test_version - cv._pseudo_cls = {ver: {}} + cv.cell_method = "catalog" + cv._pseudo_cls = {ver: {"tomo_bin_all_tomo_bin_all": {}}} + saved = {} + save_pseudo_cl = cv._save_pseudo_cl + + def capture_workspace( + ver, out_path, tomo_bin_pair, ell_eff, cl_all, wsp, nside=None + ): + saved["workspace"] = wsp + saved["nside"] = nside + return save_pseudo_cl( + ver, out_path, tomo_bin_pair, ell_eff, cl_all, wsp, nside=nside + ) + + monkeypatch.setattr(cv, "_save_pseudo_cl", capture_workspace) out_path = cv._output_path(f"pseudo_cl_{ver}.sacc") - cv.calculate_pseudo_cl_catalog(ver, out_path) + cv.calculate_pseudo_cl_catalog(ver, out_path, tomo_bin_a="all", tomo_bin_b="all") assert os.path.exists(out_path) s = sacc_io.load(out_path, allow_unblinded=True) ell, ee, bb, eb, window = sacc_io.get_pseudo_cl(s, SACC_BIN) assert window is not None # the shared BandpowerWindow rides the part + readback = cv._load_pseudo_cl(out_path, ("all", "all")) + npt.assert_array_equal(readback["BE"], eb) + assert not np.shares_memory(readback["BE"], readback["EB"]) + assert saved["nside"] is None + window_ells, unwindowed_weights = bandpower_window_from_workspace( + saved["workspace"] + ) + assert len(window_ells) == window.weight.shape[0] + npt.assert_allclose(window.weight, unwindowed_weights, rtol=1e-12, atol=1e-15) npt.assert_allclose( ell, @@ -585,13 +765,160 @@ def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): atol=ATOL_CAT, ) # The end-to-end catalog EE matches the primitive get_pseudo_cls_catalog EE - # (same computation, FITS round-trip) -- consistency, not an independent pin. + # (same computation, SACC round-trip) -- consistency, not an independent pin. cat_gal = fits.getdata(cv.cc[ver]["shear"]["path"]) - params = get_params_rho_tau(cv.cc[ver], survey=ver) - _, cl_prim, _ = cv.get_pseudo_cls_catalog(catalog=cat_gal, params=params) + params = get_params_rho_tau(cv.cc[ver]) + _, cl_prim, _ = cv.get_pseudo_cls_catalog( + catalog=cat_gal, params=params, tomo_bin_a="all", tomo_bin_b="all" + ) npt.assert_allclose(ee, cl_prim[0], rtol=RTOL_CAT, atol=ATOL_CAT) +def test_calculate_pseudo_cl_map_sacc_pixel_window(cv, monkeypatch): + """Map SACC output multiplies its workspace window by pw²(ℓ).""" + ver = cv._test_version + cv.cell_method = "map" + cv.noise_bias_method = "analytic" + cv._pseudo_cls = {ver: {"tomo_bin_all_tomo_bin_all": {}}} + saved = {} + save_pseudo_cl = cv._save_pseudo_cl + + def capture_workspace( + ver, out_path, tomo_bin_pair, ell_eff, cl_all, wsp, nside=None + ): + saved["workspace"] = wsp + saved["nside"] = nside + return save_pseudo_cl( + ver, out_path, tomo_bin_pair, ell_eff, cl_all, wsp, nside=nside + ) + + monkeypatch.setattr(cv, "_save_pseudo_cl", capture_workspace) + out_path = cv._output_path(f"pseudo_cl_map_{ver}.sacc") + cv.calculate_pseudo_cl_map(ver, NSIDE, out_path, "all", "all") + + s = sacc_io.load(out_path, allow_unblinded=True) + _ell, _ee, _bb, _eb, window = sacc_io.get_pseudo_cl(s, SACC_BIN) + assert saved["nside"] == NSIDE + window_ells, unwindowed_weights = bandpower_window_from_workspace( + saved["workspace"] + ) + assert len(window_ells) == window.weight.shape[0] + pw2 = healpy.pixwin(NSIDE, lmax=len(window_ells) - 1) ** 2 + nonzero = np.abs(unwindowed_weights) > 1e-20 + npt.assert_allclose( + window.weight[nonzero] / unwindowed_weights[nonzero], + np.broadcast_to(pw2[:, None], unwindowed_weights.shape)[nonzero], + rtol=1e-12, + ) + + +def test_fiducial_pixel_window_applies_only_to_map(): + """iNKA fiducials carry pw² on maps and remain unchanged for catalogues.""" + nside = 4 + ell_grid = np.arange(12) + fiducial = { + "W1xW1": 1.0 + ell_grid.astype(float), + "W1xW2": 2.0 + 2 * ell_grid.astype(float), + } + original = {key: value.copy() for key, value in fiducial.items()} + + catalog_fiducial = _apply_pixel_window_to_fiducial_cl(fiducial, nside, "catalog") + map_fiducial = _apply_pixel_window_to_fiducial_cl(fiducial, nside, "map") + pw2 = healpy.pixwin(nside, lmax=len(ell_grid) - 1) ** 2 + + for key, cl in original.items(): + npt.assert_array_equal(catalog_fiducial[key], cl) + npt.assert_allclose(map_fiducial[key], cl * pw2, rtol=1e-14, atol=0.0) + npt.assert_array_equal(fiducial[key], cl) + + +def test_calculate_pseudo_cl_catalog_end_to_end_tomo(cv, tmp_path): + """End-to-end catalog path: FITS round-trip of ell + EE/EB/BB. + + The catalog method has no random noise debiasing, so it is reproducible to + the same ~2e-12 catalog-path float noise. save_pseudo_cl stores ELL/EE/EB/BB + (it drops the BE row); we pin the round-tripped table. + """ + ver = cv._test_version + cv._pseudo_cls = { + ver: { + "tomo_bin_1_tomo_bin_1": {}, + "tomo_bin_1_tomo_bin_2": {}, + "tomo_bin_2_tomo_bin_2": {}, + } + } + out_path = cv._output_path(f"pseudo_cl_cat_{ver}.fits") + tomo_bin_ids, tomo_bin_pairs = cv._get_tomo_bins(ver) + + result_to_compare = np.load(REFERENCE_TOMO, allow_pickle=True)["arr_0"].item() + for tomo_bin_a, tomo_bin_b in tomo_bin_pairs: + out_path = cv._output_path( + f"pseudo_cl_cat_{ver}_{tomo_bin_a}_{tomo_bin_b}.fits" + ) + cv.calculate_pseudo_cl_catalog( + ver, out_path, tomo_bin_a=tomo_bin_a, tomo_bin_b=tomo_bin_b + ) + + assert os.path.exists(out_path) + d = fits.getdata(out_path) + # FITS gives big-endian f8; normalize for value comparison. + ell = np.asarray(d["ELL"], dtype=np.float64) + ee = np.asarray(d["EE"], dtype=np.float64) + eb = np.asarray(d["EB"], dtype=np.float64) + be = np.asarray(d["BE"], dtype=np.float64) + bb = np.asarray(d["BB"], dtype=np.float64) + + npt.assert_allclose( + ell, + result_to_compare[f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}"][ + "pseudo_cl" + ]["ELL"], + rtol=RTOL_DET, + atol=ATOL_DET, + ) + npt.assert_allclose( + ee, + result_to_compare[f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}"][ + "pseudo_cl" + ]["EE"], + rtol=RTOL_CAT, + atol=ATOL_CAT, + ) + npt.assert_allclose( + eb, + result_to_compare[f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}"][ + "pseudo_cl" + ]["EB"], + rtol=RTOL_CAT, + atol=ATOL_CAT, + ) + npt.assert_allclose( + be, + result_to_compare[f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}"][ + "pseudo_cl" + ]["BE"], + rtol=RTOL_CAT, + atol=ATOL_CAT, + ) + npt.assert_allclose( + bb, + result_to_compare[f"tomo_bin_{tomo_bin_a}_tomo_bin_{tomo_bin_b}"][ + "pseudo_cl" + ]["BB"], + rtol=RTOL_CAT, + atol=ATOL_CAT, + ) + + # The end-to-end catalog EE matches the primitive get_pseudo_cls_catalog EE + # (same computation, FITS round-trip) -- consistency, not an independent pin. + cat_gal = fits.getdata(cv.cc[ver]["shear"]["path"]) + params = get_params_rho_tau(cv.cc[ver]) + _, cl_prim, _ = cv.get_pseudo_cls_catalog( + catalog=cat_gal, params=params, tomo_bin_a=tomo_bin_a, tomo_bin_b=tomo_bin_b + ) + npt.assert_allclose(ee, cl_prim[0], rtol=RTOL_CAT, atol=ATOL_CAT) + + def test_calculate_pseudo_cl_out_path_born_at_declared_name(cv): """calculate_pseudo_cl(out_path=...) writes to the given path, never the untagged native name — so the tagged and diagnostic rules stay disjoint.""" @@ -600,7 +927,7 @@ def test_calculate_pseudo_cl_out_path_born_at_declared_name(cv): tagged = cv._output_path(f"pseudo_cl_{ver}_powspace_nbins=32.sacc") native = cv._output_path(f"pseudo_cl_{ver}.sacc") - cv.calculate_pseudo_cl(out_path=tagged) + cv.calculate_pseudo_cl(compute_tomography=False, out_path=tagged) assert os.path.exists(tagged) assert not os.path.exists(native) # no undeclared native basename touched @@ -611,4 +938,15 @@ def test_calculate_pseudo_cl_out_path_rejects_multiversion(cv): rather than write every version to the same path.""" cv.versions = [cv._test_version, "SecondVersion"] with pytest.raises(ValueError, match="one part to one path"): - cv.calculate_pseudo_cl(out_path=cv._output_path("pseudo_cl_x.sacc")) + cv.calculate_pseudo_cl( + compute_tomography=False, out_path=cv._output_path("pseudo_cl_x.sacc") + ) + + +def test_calculate_pseudo_cl_out_path_rejects_tomography(cv): + """out_path names the non-tomographic SACC part; tomographic pairs keep + their own per-pair paths, so asking for both must fail loudly.""" + with pytest.raises(ValueError, match="compute_tomography=False"): + cv.calculate_pseudo_cl( + compute_tomography=True, out_path=cv._output_path("pseudo_cl_x.sacc") + ) diff --git a/src/sp_validation/tests/test_psf_leakage.py b/src/sp_validation/tests/test_psf_leakage.py new file mode 100644 index 00000000..fe91b3bd --- /dev/null +++ b/src/sp_validation/tests/test_psf_leakage.py @@ -0,0 +1,295 @@ +"""PSF-leakage helpers of the cosmology validation: alpha, xi_psf_sys, summaries. + +Synthetic inputs only. The object-wise smoke test runs the real +shear_psf_leakage regression on a toy catalogue and therefore needs the +scientific stack (the container). +""" + +from types import SimpleNamespace + +import numpy as np +import pytest +import yaml + +from sp_validation.cosmo_val.psf_systematics import PSFSystematicsMixin + +mixin = PSFSystematicsMixin() + +N_THETA = 5 +RHO = np.full(N_THETA, 2.0) +TINY = np.diag(np.full(N_THETA, 1e-16)) + + +def _xi_sys(tau_a, tau_b, sig_a, sig_b, same_bin, seed=0, n_samples=40_000): + return mixin._compute_scale_dependent_xi_psf_sys( + RHO, + tau_a, + tau_b, + TINY, + np.diag(np.full(N_THETA, sig_a**2)), + np.diag(np.full(N_THETA, sig_b**2)), + n_samples=n_samples, + same_bin=same_bin, + seed=seed, + ) + + +def test_xi_sys_central_value(): + tau_a = np.linspace(1.0, 2.0, N_THETA) + tau_b = np.linspace(0.5, 3.0, N_THETA) + xi, _ = _xi_sys(tau_a, tau_b, 0.01, 0.02, same_bin=False) + np.testing.assert_allclose(xi, tau_a * tau_b / RHO) + + +def test_xi_sys_auto_pair_reuses_tau_draw(): + mu, sig = 1.0, 0.01 + tau = np.full(N_THETA, mu) + _, err = _xi_sys(tau, tau, sig, sig, same_bin=True) + np.testing.assert_allclose(err, 2 * mu * sig / RHO, rtol=0.03) + + +def test_xi_sys_cross_pair_uses_both_variances(): + mu_a, mu_b, sig_a, sig_b = 1.0, 3.0, 0.01, 0.03 + _, err = _xi_sys( + np.full(N_THETA, mu_a), np.full(N_THETA, mu_b), sig_a, sig_b, same_bin=False + ) + expected = np.sqrt(mu_b**2 * sig_a**2 + mu_a**2 * sig_b**2) / RHO + np.testing.assert_allclose(err, expected, rtol=0.03) + + +def test_xi_sys_errors_reproducible(): + tau = np.full(N_THETA, 1.0) + _, err_1 = _xi_sys(tau, tau, 0.1, 0.1, same_bin=False, seed=7) + _, err_2 = _xi_sys(tau, tau, 0.1, 0.1, same_bin=False, seed=7) + _, err_3 = _xi_sys(tau, tau, 0.1, 0.1, same_bin=False, seed=8) + np.testing.assert_array_equal(err_1, err_2) + assert not np.array_equal(err_1, err_3) + + +def _handlers(rho, tau, var_rho, var_tau): + theta = np.geomspace(1, 100, len(rho)) + rho_h = SimpleNamespace( + rho_stats={"theta": theta, "rho_0_p": rho, "varrho_0_p": var_rho} + ) + tau_h = SimpleNamespace(tau_stats={"tau_0_p": tau, "vartau_0_p": var_tau}) + return rho_h, tau_h + + +def test_alpha_leakage_central_value_and_reproducible(): + rho = np.linspace(1.0, 2.0, N_THETA) + tau = np.linspace(0.01, 0.03, N_THETA) + rho_h, tau_h = _handlers(rho, tau, np.full(N_THETA, 1e-4), np.full(N_THETA, 1e-6)) + theta, alpha, err = mixin._get_alpha_leakage(rho_h, tau_h, seed=3) + np.testing.assert_array_equal(theta, rho_h.rho_stats["theta"]) + np.testing.assert_allclose(alpha, tau / rho) + _, _, err_again = mixin._get_alpha_leakage(rho_h, tau_h, seed=3) + np.testing.assert_array_equal(err, err_again) + assert np.all(err > 0) + + +def test_alpha_summaries_recover_affine_model(): + c, m = 0.01, 2e-4 + theta = np.geomspace(1, 100, 12) + alpha_err = np.linspace(1e-3, 3e-3, theta.size) + alpha = c + m * theta + + out = PSFSystematicsMixin._alpha_summaries(theta, alpha, alpha_err) + + assert out["alpha_0"].nominal_value == pytest.approx(c, rel=1e-8) + assert 0 < out["alpha_0"].std_dev < alpha_err.max() + assert out["alpha_1"].nominal_value == pytest.approx(alpha[0]) + assert out["alpha_1"].std_dev == pytest.approx(alpha_err[0]) + w = 1 / alpha_err**2 + mean = np.sum(w * alpha) / np.sum(w) + assert out["alpha_mean"].nominal_value == pytest.approx(mean) + std = np.sqrt(np.sum(w * (alpha - mean) ** 2) / np.sum(w)) + assert out["alpha_mean"].std_dev == pytest.approx(std) + + +def _objectwise_config(tmp_path, n_gal=3000, seed=11): + """Toy shear catalogue with PSF columns and two tomographic bins.""" + from astropy.table import Table + + rng = np.random.default_rng(seed) + cat_dir = tmp_path / "catalog" + cat_dir.mkdir() + output_dir = tmp_path / "output" + output_dir.mkdir() + + e1_psf = rng.normal(0, 0.03, n_gal) + e2_psf = rng.normal(0, 0.03, n_gal) + tomo_bin = rng.integers(1, 3, n_gal) + leak = np.where(tomo_bin == 1, 0.05, 0.15) + Table( + { + "RA": rng.uniform(10, 12, n_gal), + "Dec": rng.uniform(10, 12, n_gal), + "e1": leak * e1_psf + rng.normal(0, 0.02, n_gal), + "e2": leak * e2_psf + rng.normal(0, 0.02, n_gal), + "w": np.ones(n_gal), + "e1_PSF": e1_psf, + "e2_PSF": e2_psf, + "fwhm_PSF": rng.uniform(0.6, 0.8, n_gal), + "tomo_bin": tomo_bin, + } + ).write(cat_dir / "shear.fits") + + version = "TestCatalog" + config = { + "nz": {"subdir": str(cat_dir), "dndz": {"blind": "A", "path": "dndz"}}, + "paths": {"output": str(output_dir)}, + version: { + "subdir": str(cat_dir), + "pipeline": "SP", + "colour": "C0", + "marker": "o", + "shear": { + "path": "shear.fits", + "w_col": "w", + "e1_col": "e1", + "e2_col": "e2", + "e1_PSF_col": "e1_PSF", + "e2_PSF_col": "e2_PSF", + "tomo_bin_col": "tomo_bin", + "R": 1.0, + }, + }, + } + config_path = tmp_path / "config.yaml" + config_path.write_text(yaml.dump(config, sort_keys=False)) + return version, { + "catalog_config": str(config_path), + "output_dir": str(output_dir), + } + + +def test_objectwise_leakage_and_plot_smoke(tmp_path, monkeypatch): + """Object-wise fit per bin and the comparison plot, alpha(theta) stubbed. + + The rho/tau products are replaced by a synthetic alpha(theta), so this + covers the object-wise regression, the per-bin storage and the plot, not + the rho/tau loading. + """ + pytest.importorskip("shear_psf_leakage") + import matplotlib + + matplotlib.use("Agg") + # TeX text rendering dominates the run time of the ~20 diagnostic figures + monkeypatch.setitem(matplotlib.rcParams, "text.usetex", False) + from sp_validation.cosmo_val import CosmologyValidation + + version, params = _objectwise_config(tmp_path) + cv = CosmologyValidation(versions=[version], npatch=1, **params) + + theta = np.geomspace(1, 100, 6) + monkeypatch.setattr( + cv, + "_load_alpha_leakage", + lambda ver, tomo_bin_id, cov_type=None: ( + theta, + 0.01 + 1e-4 * theta, + np.full(theta.size, 1e-3), + ), + ) + + cv.plot_objectwise_leakage() + coeff = cv.leakage_coeff[version]["tomo_bin_all"] + assert {"a11", "a22", "aii_mean", "alpha_mean", "alpha_1", "alpha_0"} <= set(coeff) + assert coeff["alpha_0"].nominal_value == pytest.approx(0.01) + assert (tmp_path / "output" / "leakage_coefficients.png").exists() + + cv.plot_objectwise_leakage(tomography=True) + for tomo_bin_id, expected in ((1, 0.05), (2, 0.15)): + bin_coeff = cv.leakage_coeff[version][f"tomo_bin_{tomo_bin_id}"] + assert bin_coeff["aii_mean"].nominal_value == pytest.approx(expected, abs=0.03) + assert "alpha_mean" in bin_coeff + assert (tmp_path / "output" / "leakage_coefficients_tomo.png").exists() + + +def test_leakage_object_reads_the_selected_rows(tmp_path): + """The object-wise reader keeps the rows a galaxy mask selects.""" + pytest.importorskip("shear_psf_leakage") + from sp_validation.cosmo_val import CosmologyValidation + from sp_validation.cosmo_val.core import _LeakageObject + + version, params = _objectwise_config(tmp_path) + cv = CosmologyValidation(versions=[version], npatch=1, **params) + obj = object.__new__(_LeakageObject) + obj.entries = cv.cc[version] + + mask = cv._get_galaxy_mask(version, 2) + obj.read_data(selection=mask) + assert len(obj._dat) == mask.sum() + assert np.all(obj._dat["tomo_bin"] == 2) + + with pytest.raises(ValueError, match="different length"): + obj.read_data(selection=mask[1:]) + + +class _ObjectwiseControlFlow(PSFSystematicsMixin): + """Two versions, one of which lacks a catalogue column.""" + + def __init__(self): + self.versions = ["good", "bad"] + obj = SimpleNamespace( + check_params=lambda: None, + update_params=lambda: None, + prepare_output=lambda: None, + ) + self.results_objectwise = {"good": obj, "bad": obj} + + def _get_tomo_bins_for_versions(self, versions, tomography): + return {v: {"ids": [1, 2] if tomography else ["all"]} for v in versions} + + def _objectwise_leakage_bin(self, obj, ver, *args): + if ver == "bad": + raise KeyError("fwhm_PSF") + return {"aii_mean": 0.1} + + def print_start(self, *args): + pass + + def print_magenta(self, *args): + pass + + +def test_version_missing_a_column_stays_dropped(): + cv = _ObjectwiseControlFlow() + cv.calculate_objectwise_leakage() + assert set(cv.results_objectwise) == {"good"} + assert set(cv.leakage_coeff) == {"good"} + cv.calculate_objectwise_leakage(tomography=True) + assert "tomo_bin_1" in cv.leakage_coeff["good"] + + +@pytest.mark.parametrize( + "cov_estimate_method, expected", + [("sim", 500), ("jk", 150), ("th", None)], +) +def test_rho_tau_fit_debiases_estimated_covariances( + monkeypatch, cov_estimate_method, expected +): + """Simulation and jackknife covariances get Hartlap-debiased; theory does not. + + The debiasing count is the number of simulations (``n_sim_cov``) or of + jackknife patches the covariance was estimated from. + """ + from sp_validation.cosmo_val import psf_systematics + + seen = {} + + def fake_get_samples(*args, apply_debias, **kwargs): + seen["apply_debias"] = apply_debias + return None, None, None + + monkeypatch.setattr(psf_systematics, "get_samples", fake_get_samples) + cv = SimpleNamespace( + cov_estimate_method=cov_estimate_method, + n_sim_cov=500, + rho_tau_method="lsq", + psf_fitter=None, + basename=lambda version, tomo_bin_a="all": version, + ) + cv.get_samples = PSFSystematicsMixin.get_samples.__get__(cv) + cv.get_samples("v", {"patch_number": 150}, "all") + assert seen["apply_debias"] == expected diff --git a/src/sp_validation/tests/test_rho_tau_jackknife.py b/src/sp_validation/tests/test_rho_tau_jackknife.py new file mode 100644 index 00000000..bc64537a --- /dev/null +++ b/src/sp_validation/tests/test_rho_tau_jackknife.py @@ -0,0 +1,116 @@ +"""Jackknife layouts and covariance averaging on small synthetic catalogues.""" + +from pathlib import Path + +import numpy as np +from astropy.table import Table + +from sp_validation import rho_tau + + +def test_jackknife_draws_use_distinct_shared_layouts(tmp_path, monkeypatch): + rng = np.random.default_rng(37) + n = 400 + star = Table( + dict( + RA=rng.uniform(10, 14, n), + Dec=rng.uniform(10, 14, n), + p1=rng.normal(0, 0.03, n), + p2=rng.normal(0, 0.03, n), + s1=rng.normal(0, 0.04, n), + s2=rng.normal(0, 0.04, n), + tp=np.ones(n), + ts=np.full(n, 1.1), + ) + ) + gal = Table( + dict( + RA=rng.uniform(10, 14, n), + Dec=rng.uniform(10, 14, n), + e1=rng.normal(0, 0.2, n), + e2=rng.normal(0, 0.2, n), + w=rng.uniform(0.2, 1, n), + ) + ) + star.write(tmp_path / "stars.fits") + gal.write(tmp_path / "galaxies.fits") + config = { + "v": dict( + patch_number=7, + psf=dict( + path=str(tmp_path / "stars.fits"), + ra_col="RA", + dec_col="Dec", + e1_PSF_col="p1", + e2_PSF_col="p2", + e1_star_col="s1", + e2_star_col="s2", + PSF_size="tp", + star_size="ts", + ), + shear=dict( + path=str(tmp_path / "galaxies.fits"), + ra_col="RA", + dec_col="Dec", + e1_col="e1", + e2_col="e2", + w_col="w", + ), + ) + } + layouts, catalogs, chunks = [], [], {"rho": [], "tau": []} + compute = rho_tau.RhoStat.compute_rho_stats + save = np.save + + def record_compute(self, catalog_id, *args, **kwargs): + cats = self.catalogs.catalogs_dict + psf = cats[f"psf_{catalog_id}"] + layouts.append(psf.patch_centers.copy()) + catalogs.append(psf) + for prefix in ("psf_error", "psf_size_error", "gal"): + np.testing.assert_array_equal( + cats[f"{prefix}_{catalog_id}"].patch_centers, psf.patch_centers + ) + # Materialise TreeCorr's patch cache before the next draw. + assert len(psf.patches) == 7 + return compute(self, catalog_id, *args, **kwargs) + + def record_save(path, arr, *args, **kwargs): + for kind in chunks: + if Path(path).stem in {f"cov_{kind}_tau{i}" for i in range(3)}: + chunks[kind].append(np.array(arr).copy()) + return save(path, arr, *args, **kwargs) + + monkeypatch.setattr(rho_tau.RhoStat, "compute_rho_stats", record_compute) + monkeypatch.setattr(np, "save", record_save) + rho_tau.get_jackknife_cov( + config, + "v", + dict( + min_sep=1, + max_sep=150, + nbins=2, + sep_units="arcmin", + num_threads=1, + cross_patch_weight="match", + ), + str(tmp_path), + "rho", + "tau", + npatch=7, + ncov=3, + ) + assert len(layouts) == 3 + assert all( + not np.array_equal(layouts[i], layouts[j]) for i in range(3) for j in range(i) + ) + assert len({id(cat) for cat in catalogs}) == 3 + for kind in chunks: + assert len(chunks[kind]) == 3 + assert any(not np.array_equal(chunks[kind][0], cov) for cov in chunks[kind][1:]) + np.testing.assert_allclose( + np.load(tmp_path / f"cov_{kind}_{kind}_jk.npy"), + np.mean(chunks[kind], axis=0), + rtol=1e-13, + atol=0, + ) diff --git a/src/sp_validation/tests/test_sacc_io_one_covariance.py b/src/sp_validation/tests/test_sacc_io_one_covariance.py index bbe90ca3..d69fb64c 100644 --- a/src/sp_validation/tests/test_sacc_io_one_covariance.py +++ b/src/sp_validation/tests/test_sacc_io_one_covariance.py @@ -28,26 +28,41 @@ # --------------------------------------------------------------------------- # # Synthetic OneCovariance-shaped fixtures # --------------------------------------------------------------------------- # +def _ell(i): + """The multipole of ℓ-bin ``i`` in the synthetic tables.""" + return 10.0 * (i + 1) + + def _one_cov_table(cov_gauss, cov_all): """Flatten two n x n matrices into a OneCovariance ``covariance_list`` table. - Reproduces the real flat output: one row per ``(i, j)`` element pair in - row-major order ``k = i·n + j``, with the Gaussian value in column 10 and - the Gaussian+non-Gaussian value in column 9. Columns 0-8 and the index - columns are filled with self-documenting placeholder values (the reshape - only reads cols 9/10, but a realistic width proves it does not spill). + Reproduces the real flat output of a single tomographic bin, as + OneCovariance's ``__write_cov_list`` writes it (every ℓ pair, both + orders): one row per ``(i, j)`` ℓ-bin pair in row-major order + ``k = i·n + j``, ℓ_i and ℓ_j in columns 1 and 2, the bin indices of the + four fields (all ``1``) in columns 5-8, the Gaussian value in column 10 and + the total (Gaussian + non-Gaussian) value in column 9. Columns 0, 3 and 4 + hold placeholders the reshape does not read. """ n = cov_gauss.shape[0] rows = [] for i in range(n): for j in range(n): - row = np.arange(11.0) # placeholder cols 0-8 (+ overwritten 9,10) + row = np.arange(11.0) # placeholder cols 0, 3, 4 + row[1], row[2] = _ell(i), _ell(j) + row[5:9] = 1 row[9] = cov_all[i, j] row[10] = cov_gauss[i, j] rows.append(row) return np.array(rows) +def _row_of(table, i, j): + """Index of the table row holding ℓ-bin pair ``(i, j)``.""" + [k] = np.flatnonzero((table[:, 1] == _ell(i)) & (table[:, 2] == _ell(j))) + return k + + def _spd(n, seed): """Symmetric positive-definite matrix of size ``n`` (a valid covariance).""" a = np.random.default_rng(seed).normal(size=(n, n)) @@ -75,7 +90,9 @@ def test_covariance_blocks_reshapes_to_hand_built_matrix(): WHY TEETH: (a) ``gaussian=True`` vs ``False`` must return the two *different* matrices, proving the column flag is load-bearing; (b) perturbing a single entry of the flat input must change exactly that entry of the reshaped - block, proving the reshape actually reads the table (not a constant). + block, proving the reshape actually reads the table (not a constant). The + table stays symmetric, as a covariance is, so the entry is perturbed with + its mirror. """ cov_gauss = _spd(4, seed=1) cov_all = _spd(4, seed=2) @@ -93,13 +110,14 @@ def test_covariance_blocks_reshapes_to_hand_built_matrix(): # TEETH: gaussian and gauss+ng select different columns -> different blocks. assert not np.allclose(block_g, block_a) - # TEETH: a perturbation of one flat-table entry moves exactly that block - # entry (row k = i·n + j, col 10 for gaussian). + # TEETH: a perturbation of one flat-table entry (and its mirror) moves + # exactly that block entry (and its mirror), col 10 for gaussian. perturbed = table.copy() - perturbed[2 * 4 + 1, 10] += 5.0 # element (i=2, j=1) + perturbed[[_row_of(table, 1, 2), _row_of(table, 2, 1)], 10] += 5.0 [(_, block_p)] = sio.covariance_blocks(perturbed, selector, gaussian=True) - npt.assert_allclose(block_p[2, 1] - block_g[2, 1], 5.0, rtol=1e-12) - block_p[2, 1] = block_g[2, 1] + for i, j in ((1, 2), (2, 1)): + npt.assert_allclose(block_p[i, j] - block_g[i, j], 5.0, rtol=1e-12) + block_p[i, j] = block_g[i, j] npt.assert_allclose(block_p, block_g, rtol=1e-12) # nothing else moved diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index c4963b33..74181190 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -152,9 +152,11 @@ def get_bandpower_windows(self): assert np.array_equal(data["BB"], cl_all[3]) -def test_pseudo_cl_to_sacc_real_namaster(tmp_path): - """A real small-nside NaMaster workspace's window survives the writer.""" +def test_pseudo_cl_to_sacc_real_namaster_pixel_window(tmp_path): + """The SACC window gains pw²(ℓ) only when a map nside is supplied.""" pytest.importorskip("pymaster") + import healpy as hp + from sp_validation.pseudo_cl import get_pseudo_cls_map nside = 32 @@ -169,9 +171,19 @@ def test_pseudo_cl_to_sacc_real_namaster(tmp_path): ell_r, ee, bb, eb, window = sio.get_pseudo_cl(s2, (0, 0)) assert np.array_equal(ell_r, ell_eff) assert np.array_equal(ee, cl_all[0]) and np.array_equal(bb, cl_all[3]) - # window columns correspond to the bandpowers, one per ell_eff + # With nside unset, the workspace window carries no pixel window. assert window.weight.shape[1] == len(ell_eff) + mapped = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell_eff, cl_all, wsp, nside=nside) + mapped_window = mapped.get_bandpower_windows(mapped.indices(sio.CL_EE)) + pw2 = hp.pixwin(nside, lmax=window.weight.shape[0] - 1) ** 2 + nonzero = np.abs(window.weight) > 1e-20 + np.testing.assert_allclose( + mapped_window.weight[nonzero] / window.weight[nonzero], + np.broadcast_to(pw2[:, None], window.weight.shape)[nonzero], + rtol=1e-12, + ) + def test_cosebis_to_sacc(tmp_path): En, Bn = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 diff --git a/src/sp_validation/tests/test_statistics.py b/src/sp_validation/tests/test_statistics.py index 4e11a45b..d9e08277 100644 --- a/src/sp_validation/tests/test_statistics.py +++ b/src/sp_validation/tests/test_statistics.py @@ -152,48 +152,130 @@ def test_chi2_and_pte_diagonal_reduces_to_sum_of_squares(): def test_cov_from_one_covariance_selects_gaussian_column(): - """Pin the reshaped matrix and prove the gaussian column selection. - - WHAT IS PINNED: ``cov_from_one_covariance`` reads a flat OneCovariance - table with one row per ``(i, j)`` pair (row-major, ``k = i*n_bins + j``) - and lays the chosen column into a square matrix. Column 10 carries the - Gaussian-only term (``gaussian=True``); column 9 the Gaussian+non-Gaussian - term (``gaussian=False``). With a deterministic table whose entries encode - their row and column, both reshaped matrices are pinned as literals. - - WHY TEETH: the only difference between the two calls is the column index - (10 vs 9), so the two pinned matrices differ by exactly 1 in every entry, - proving the gaussian flag selects the right column. A companion check - places a unique tag ``10*i + j`` in column 10 and asserts the reshape is - row-major (``cov[i, j]`` lands at row ``i*n_bins + j``), so a transposed - or column-major refactor would change the recovered matrix. - """ - # Two-bin table -> 4 rows; entry (row k, col c) = 10*k + c, so the - # chosen-column values are distinct and self-documenting. Row k carries - # the (i, j) pair with k = i*n_bins + j, so rows 0..3 -> (0,0),(0,1), - # (1,0),(1,1). Column 10 holds values 10, 20, 30, 40; column 9 holds - # 9, 19, 29, 39. - one_cov = np.array([np.arange(11.0) + 10.0 * k for k in range(4)]) + """Pin the reshaped matrix and prove the gaussian column selection.""" + # obs, ell1, ell2, s1, s2, tomoi, tomoj, tomok, tomol, cov, covg, covng, covssc + one_cov = np.array( + [ + [0.0, 10.0, 10.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 99.0, 100.0, 0.0, 0.0], + [0.0, 10.0, 20.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 100.0, 101.0, 0.0, 0.0], + [0.0, 20.0, 20.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 110.0, 111.0, 0.0, 0.0], + ] + ) cov_gauss = cov_from_one_covariance(one_cov, gaussian=True) cov_nongauss = cov_from_one_covariance(one_cov, gaussian=False) - npt.assert_allclose(cov_gauss, [[10.0, 20.0], [30.0, 40.0]], rtol=1e-12) - npt.assert_allclose(cov_nongauss, [[9.0, 19.0], [29.0, 39.0]], rtol=1e-12) + npt.assert_allclose(cov_gauss, [[100.0, 101.0], [101.0, 111.0]], rtol=1e-12) + npt.assert_allclose(cov_nongauss, [[99.0, 100.0], [100.0, 110.0]], rtol=1e-12) - # TEETH: the gaussian flag shifts the column by one, so every entry of the - # gaussian matrix exceeds its non-gaussian counterpart by exactly 1. + # TEETH: the gaussian flag shifts the column by one, so every entry of + # the gaussian matrix exceeds its non-gaussian counterpart by exactly 1. npt.assert_allclose(cov_gauss - cov_nongauss, 1.0, rtol=1e-12) - # TEETH: the (i, j) -> row k = i*n_bins + j layout is row-major. Tag - # column 10 with 10*i + j and check it lands at cov[i, j], not cov[j, i]. - tagged = np.zeros((4, 11)) - for i in range(2): - for j in range(2): - tagged[i * 2 + j, 10] = 10.0 * i + j + +def test_cov_from_one_covariance_orders_tomo_blocks(): + """Pin the tomo-block ordering against ``combinations_with_replacement``.""" + # obs, ell1, ell2, s1, s2, tomoi, tomoj, tomok, tomol, cov, covg, covng, covssc + one_cov = np.array( + [ + [ + 0.0, + 10.0, + 10.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + 0.0, + 0.0, + 0.0, + ], # (a,b)=(0,0) + [ + 0.0, + 10.0, + 10.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 2.0, + 0.0, + 1.0, + 0.0, + 0.0, + ], # (0,1) + [ + 0.0, + 10.0, + 10.0, + 1.0, + 1.0, + 1.0, + 1.0, + 2.0, + 2.0, + 1.0, + 2.0, + 0.0, + 0.0, + ], # (0,2) + [ + 0.0, + 10.0, + 10.0, + 1.0, + 1.0, + 1.0, + 2.0, + 1.0, + 2.0, + 10.0, + 11.0, + 0.0, + 0.0, + ], # (1,1) + [ + 0.0, + 10.0, + 10.0, + 1.0, + 1.0, + 1.0, + 2.0, + 2.0, + 2.0, + 11.0, + 12.0, + 0.0, + 0.0, + ], # (1,2) + [ + 0.0, + 10.0, + 10.0, + 1.0, + 1.0, + 2.0, + 2.0, + 2.0, + 2.0, + 21.0, + 22.0, + 0.0, + 0.0, + ], # (2,2) + ] + ) + + cov_gauss = cov_from_one_covariance(one_cov, gaussian=True) + npt.assert_allclose( - cov_from_one_covariance(tagged, gaussian=True), - [[0.0, 1.0], [10.0, 11.0]], + cov_gauss, + [[0.0, 1.0, 2.0], [1.0, 11.0, 12.0], [2.0, 12.0, 22.0]], rtol=1e-12, ) diff --git a/uv.lock b/uv.lock index 96ab1885..7d11c489 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'linux'", @@ -287,6 +287,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2f/c1/cbcd754fb615c9cccb08a854d1a6f728e7a640680098dbdcf762b566544a/blosc2-4.14.1-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6a2208f001580d497a050aba1a4083d5ba54adbe1e166cee4faf4731f0e96c9a", size = 6697746, upload-time = "2026-09-30T10:49:25.052Z" }, ] +[[package]] +name = "cached-property" +version = "2.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/4b/3d870836119dbe9a5e3c9a61af8cc1a8b69d75aea564572e385882d5aefb/cached_property-2.0.1.tar.gz", hash = "sha256:484d617105e3ee0e4f1f58725e72a8ef9e93deee462222dbd51cd91230897641", size = 10574, upload-time = "2024-10-25T15:43:55.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/0e/7d8225aab3bc1a0f5811f8e1b557aa034ac04bdf641925b30d3caf586b28/cached_property-2.0.1-py3-none-any.whl", hash = "sha256:f617d70ab1100b7bcf6e42228f9ddcb78c676ffa167278d9f730d1c2fba69ccb", size = 7428, upload-time = "2024-10-25T15:43:54.711Z" }, +] + [[package]] name = "cadcutils" version = "1.6.2" @@ -600,15 +609,15 @@ dependencies = [ ] [[package]] -name = "cosmology" -version = "2022.10.9" +name = "cosmology-compat-camb" +version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c9/38/04099bb2b6a626bd69a3f117afc60f2c47c08899e0730e0e48c898bf4745/cosmology-2022.10.9.tar.gz", hash = "sha256:0c2857c9bf1fdd09f1f11ab5765df0389a4101f3a900fa11251c3f37696f02d4", size = 8488, upload-time = "2022-10-10T10:18:16.229Z" } +sdist = { url = "https://files.pythonhosted.org/packages/08/2e/a637aae1677d02337be371c9a7b7e9d5051199bb58a2386781230adcee45/cosmology_compat_camb-0.2.0.tar.gz", hash = "sha256:e36fda04a78e16fc1a5c4aa22c30ec1bf64bbe0b092aa1b894ca746ce5e5551b", size = 4562, upload-time = "2025-05-08T10:31:43.226Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/2c/c80db37593ad593e3f6c60705fbdf690b33f22c89edd5507d2731346cd87/cosmology-2022.10.9-py3-none-any.whl", hash = "sha256:3903658c2474177a1a1c75771ae14458d93200c516f8fc6c3f4d776f3287b288", size = 9341, upload-time = "2022-10-10T10:18:14.928Z" }, + { url = "https://files.pythonhosted.org/packages/2e/42/1a657f9b3540f355f34af735d8643aae1480620689a72423fe69aeeb78e5/cosmology_compat_camb-0.2.0-py3-none-any.whl", hash = "sha256:eb6e74290bb6a6a60a47d1338fc233ba1697058ed7f01a01de1180dbee1bd75d", size = 3903, upload-time = "2025-05-08T10:31:41.997Z" }, ] [[package]] @@ -871,6 +880,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/83/ae/676feae8e4644a6d7169951a97f61c56f416c73f67bf1761f2461d75cc81/deprecated-3.0.0-py3-none-any.whl", hash = "sha256:58204cf4a7f6270d547af5c278ee7a6bb56045a4b3d8441a1cd11660f41b7939", size = 21912, upload-time = "2026-09-26T13:58:09.458Z" }, ] +[[package]] +name = "deprecation" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788, upload-time = "2020-04-20T14:23:38.738Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178, upload-time = "2020-04-20T14:23:36.581Z" }, +] + [[package]] name = "dill" version = "0.4.1" @@ -1150,7 +1171,7 @@ dependencies = [ [[package]] name = "glass" -version = "2025.3" +version = "2026.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "array-api-compat" }, @@ -1159,9 +1180,9 @@ dependencies = [ { name = "healpy" }, { name = "transformcl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/61/18/6d1387d74f597aacde573ffa9e7d28faa23c74414f69f56bffe9d3ccffaf/glass-2025.3.tar.gz", hash = "sha256:1f55bd5e85dc4eb3964bc6cf9a61e8da0cba4b3ecd60531d58384a27debcda3d", size = 274314, upload-time = "2025-12-12T16:38:18.096Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c1/c3/3b7d18cca8d0fa41dbb646ce16935804b380dcf766e7385799f44829b419/glass-2026.2.tar.gz", hash = "sha256:cf5ef6cb76b4738f8dc05ddfc18c359c558bb36e9cd09ca0115fa3ca3deb491f", size = 66236, upload-time = "2026-06-04T15:18:39.155Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/99/25a79eb65032d54868b4fc85fd105741858f493c391e9663954c15398e80/glass-2025.3-py3-none-any.whl", hash = "sha256:1cd0cd21aa25f76917235812164a9aa666032222ffb7bacfae766cd12f62b26b", size = 58522, upload-time = "2025-12-12T16:38:16.557Z" }, + { url = "https://files.pythonhosted.org/packages/7c/d1/55ea5e63e81db82806e92ba9f9c2b7277c0c2afe57bf8c1d3763e18005b2/glass-2026.2-py3-none-any.whl", hash = "sha256:0af0b5cd8708040925699c307076fb1c79bbb7e6cba4522764bcc21dd3bdadb8", size = 63746, upload-time = "2026-06-04T15:18:37.895Z" }, ] [[package]] @@ -1194,6 +1215,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/ff/556b2423f449a7515af6b8222a4d7833cbe09ff3e8d2f0b80471f5f6d02e/google_crc32c-1.9.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0abe7e202c25909869c35672ab0f2fe748a7acf276eb78577332a7c38999740f", size = 36858, upload-time = "2026-09-24T21:38:15.799Z" }, ] +[[package]] +name = "gsl" +version = "0.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/36/a8a99ab67918beb6ee8722fb6c27512796a5ff8ed4c10036f787eef51569/gsl-0.0.3.tar.gz", hash = "sha256:85a6071e00a284510c963b97ab74ef8a3e7d1786c5f52dc2ff444cbb7d94ee3c", size = 44404, upload-time = "2018-05-30T09:02:51.203Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/1e/14d4c4657bc7130b900c65033861ee0db508d2d9206f4223f5da7c855acc/gsl-0.0.3-py3-none-any.whl", hash = "sha256:081c77375bf65cb759a3e5129275b8806be18648452f50f7c420f40dac60529a", size = 39432, upload-time = "2018-05-30T09:02:49.93Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1295,6 +1325,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b1/c0/13b3c0482824fa5ff1a4710e505658848d68ea955eba4dd37c6cd29862cc/healsparse-1.15.0-py3-none-any.whl", hash = "sha256:71e75706bb5b8fe31344990e461f58d7ea9dd72aa236f0bdd270f63033bf8a1a", size = 69817, upload-time = "2026-09-11T15:07:30.778Z" }, ] +[[package]] +name = "hmf" +version = "3.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astropy" }, + { name = "cached-property" }, + { name = "camb" }, + { name = "click" }, + { name = "deprecation" }, + { name = "numpy" }, + { name = "rich" }, + { name = "scipy" }, + { name = "tomli-w" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/af/4cfddf1687501059bf933aeac3e0650b480e514debb9434e2c3d5c948354/hmf-3.7.2.tar.gz", hash = "sha256:3dd18fc49a8335af2cc015864f4b7f683cd05f9a598b2e70742924b46790df74", size = 2522789, upload-time = "2026-10-04T21:55:43.106Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/e5/ca23219486c8d21fffebcb1c85e4402a9354f5fcb9973e6d7c4e4dd76ff3/hmf-3.7.2-py3-none-any.whl", hash = "sha256:ac159e65ba22e91b83778ab57aa64478ae34017e0a6a71fcb143f3562b7e26c4", size = 116689, upload-time = "2026-10-04T21:55:41.607Z" }, +] + [[package]] name = "hpack" version = "4.2.0" @@ -2021,6 +2071,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1b/3d/a93a1332b1412301ddb454f206507b5e0b3679e012471fbbb075ca3b0845/lenspack-1.0.0-py3-none-any.whl", hash = "sha256:f9c632697516bb681e16b3eff66a7210d6355926179b95aa7060c8936c2259cc", size = 37746, upload-time = "2020-09-04T08:33:50.114Z" }, ] +[[package]] +name = "levin" +version = "0.0.1" +source = { git = "https://github.com/sachaguer/OneCovariance.git?rev=main#70265be3a4bffcfded7b9251a3c4cbe8159d793f" } +dependencies = [ + { name = "astropy" }, + { name = "future" }, + { name = "gsl" }, + { name = "healpy" }, + { name = "hmf" }, + { name = "matplotlib" }, + { name = "numpy" }, + { name = "pip" }, + { name = "pyyaml" }, + { name = "scipy" }, + { name = "wget" }, +] + [[package]] name = "llvmlite" version = "0.50.0" @@ -2908,6 +2976,15 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/b8/02/5bf3639f5b77e9b183011c08541c5039ba3d04f5316c70312b48a8e003a9/pims-0.7.tar.gz", hash = "sha256:55907a4c301256086d2aa4e34a5361b9109f24e375c2071e1117b9491e82946b", size = 87779, upload-time = "2024-06-10T19:20:42.842Z" } +[[package]] +name = "pip" +version = "26.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/15/4500e320e6b101ec3b719ae85b697d9940b6cda672bc555bd6016fc60c6f/pip-26.2.1.tar.gz", hash = "sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f", size = 1848877, upload-time = "2026-08-04T22:51:14.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl", hash = "sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e", size = 1816632, upload-time = "2026-08-04T22:51:12.472Z" }, +] + [[package]] name = "platformdirs" version = "4.12.3" @@ -3821,6 +3898,7 @@ dependencies = [ { name = "jupyterlab" }, { name = "jupytext" }, { name = "lenspack" }, + { name = "levin" }, { name = "lmfit" }, { name = "matplotlib" }, { name = "numba" }, @@ -3864,7 +3942,7 @@ docs = [ { name = "sphinxcontrib-bibtex" }, ] glass = [ - { name = "cosmology" }, + { name = "cosmology-compat-camb" }, { name = "fitsio" }, { name = "glass" }, { name = "glass-ext-camb" }, @@ -3888,7 +3966,7 @@ requires-dist = [ { name = "clmm" }, { name = "colorama" }, { name = "cosmo-numba", git = "https://github.com/cailmdaley/cosmo-numba.git?rev=a64cb2ed13e595a16ad056a128ef24594b811504" }, - { name = "cosmology", marker = "extra == 'glass'", specifier = "==2022.10.9" }, + { name = "cosmology-compat-camb", marker = "extra == 'glass'", specifier = "==0.2.0" }, { name = "cosmosis", marker = "extra == 'workflow'", specifier = ">=3.25" }, { name = "cryptography" }, { name = "cs-util", git = "https://github.com/CosmoStat/cs_util.git?rev=develop" }, @@ -3896,7 +3974,7 @@ requires-dist = [ { name = "fast-pt", marker = "extra == 'workflow'", specifier = ">=3.2,<4" }, { name = "fitsio", marker = "extra == 'glass'" }, { name = "getdist", git = "https://github.com/benabed/getdist.git?rev=113cd22a9a0d013b6f72fe734be81f260f3d3be5" }, - { name = "glass", marker = "extra == 'glass'", specifier = "==2025.3" }, + { name = "glass", marker = "extra == 'glass'", specifier = "==2026.2" }, { name = "glass-ext-camb", marker = "extra == 'glass'", specifier = "==2023.6" }, { name = "h5py" }, { name = "healpy" }, @@ -3907,6 +3985,7 @@ requires-dist = [ { name = "jupyterlab" }, { name = "jupytext", specifier = ">=1.15" }, { name = "lenspack" }, + { name = "levin", git = "https://github.com/sachaguer/OneCovariance.git?rev=main" }, { name = "lmfit" }, { name = "matplotlib" }, { name = "mpi4py", marker = "extra == 'workflow'" }, @@ -4216,6 +4295,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/45/c7b5c3168458db837e8ceab06dc77824e18202679d0463f0e8f002143a97/tinycss2-1.5.1-py3-none-any.whl", hash = "sha256:3415ba0f5839c062696996998176c4a3751d18b7edaaeeb658c9ce21ec150661", size = 28404, upload-time = "2025-11-23T10:29:08.676Z" }, ] +[[package]] +name = "tomli-w" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/75/241269d1da26b624c0d5e110e8149093c759b7a286138f4efd61a60e75fe/tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021", size = 7184, upload-time = "2025-01-15T12:07:24.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, +] + [[package]] name = "toolz" version = "1.1.0" @@ -4422,6 +4510,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/d2/cc4dc1271e464942db7ee278baae2daa99ee77cb2af744025c04da585a3e/websocket_client-1.9.2-py3-none-any.whl", hash = "sha256:e1a673830a9c7bfa47b1cd3d5e4178f4c9651d80a4eab02c9c23a1c3ec6250ce", size = 95786, upload-time = "2026-08-31T14:08:39.899Z" }, ] +[[package]] +name = "wget" +version = "3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/6a/62e288da7bcda82b935ff0c6cfe542970f04e29c756b0e147251b2fb251f/wget-3.2.zip", hash = "sha256:35e630eca2aa50ce998b9b1a127bb26b30dfee573702782aa982f875e3f16061", size = 10857, upload-time = "2015-10-22T15:26:37.51Z" } + [[package]] name = "widgetsnbextension" version = "4.0.16" diff --git a/workflow/common.py b/workflow/common.py index de0f71b2..111da5f5 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -375,20 +375,39 @@ def get_shear_catalog(wildcards): def cv_basename(version, fiducial=None): - """Reproduce CosmologyValidation.basename() for a version. + """Reproduce CosmologyValidation.basename() for a version's ("all", "all") pair. - Mirrors the f-string in cosmo_val.py so rule outputs match exactly what the - method writes. Uses fiducial binning (min_sep/max_sep/nbins/npatch). + Mirrors the f-string of ``CosmologyValidation.basename(version)`` so rule + outputs match exactly what the method writes. Uses fiducial binning + (min_sep/max_sep/nbins/npatch). """ fiducial = fiducial or FIDUCIAL return ( - f"{version}_minsep={fiducial['min_sep']}" + f"{version}_tomo_bin_all_minsep={fiducial['min_sep']}" f"_maxsep={fiducial['max_sep']}" f"_nbins={fiducial['nbins']}" f"_npatch={fiducial['npatch']}" ) +def cv_rho_stats(version, fiducial=None): + """ρ-statistics FITS that CosmologyValidation writes for a version.""" + base = cv_basename(version, fiducial) + return str(COSMO_VAL / "rho_tau_stats" / f"rho_stats_{base}.fits") + + +def cv_tau_stats(version, fiducial=None): + """τ-statistics FITS that CosmologyValidation writes for a version.""" + base = cv_basename(version, fiducial) + return str(COSMO_VAL / "rho_tau_stats" / f"tau_stats_{base}.fits") + + +def cv_cov_tau(version, fiducial=None): + """Theoretical τ covariance that CosmologyValidation writes for a version.""" + base = cv_basename(version, fiducial) + return str(COSMO_VAL / "rho_tau_stats" / f"cov_tau_{base}_th.npy") + + # CosmologyValidation constructor kwargs read from config["cosmo_val"]. Every # keyword with a default is either here or explicitly exempted in # src/sp_validation/tests/test_cv_init_params.py, so no default applies silently. @@ -397,6 +416,7 @@ def cv_basename(version, fiducial=None): "cov_estimate_method", "compute_cov_rho", "n_cov", + "n_sim_cov", "theta_min", "theta_max", "nbins", diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 8d9839c6..f4eb0e2a 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -50,24 +50,12 @@ CV_BINNING = ( def cv_xi_txt(version): - """Path to the 2pcf data vector calculate_2pcf writes for a version. + """Path to the ξ± TreeCorr dump calculate_2pcf_version writes for a version. - Mirrors the out_fname f-string in cosmo_val.calculate_2pcf: - {ver}_xi_minsep=..._maxsep=..._nbins=..._npatch=...txt + Mirrors RealSpaceMixin._xi_txt_path for the ("all", "all") pair: + xi_{ver}_tomo_bin_all_minsep=..._maxsep=..._nbins=..._npatch=...txt """ - return str(COSMO_VAL / f"{version}_xi_{xi_binning('reporting')}.txt") - - -def cv_rho_stats(version): - return str( - COSMO_VAL / "rho_tau_stats" / f"rho_stats_{cv_basename(version, CV_FIDUCIAL)}.fits" - ) - - -def cv_tau_stats(version): - return str( - COSMO_VAL / "rho_tau_stats" / f"tau_stats_{cv_basename(version, CV_FIDUCIAL)}.fits" - ) + return str(COSMO_VAL / f"xi_{version}_tomo_bin_all_{xi_binning('reporting')}.txt") def _pure_eb_stub(version): @@ -220,7 +208,7 @@ CV_INIT = cv_init_params(config) rule cv_plot_rho_stats: """Overlay rho statistics across all versions.""" input: - rho=[cv_rho_stats(v) for v in CV_VERSIONS], + rho=[cv_rho_stats(v, CV_FIDUCIAL) for v in CV_VERSIONS], output: sentinel=str(CV_SENTINELS / "plot_rho_stats.done"), params: @@ -234,7 +222,7 @@ rule cv_plot_rho_stats: rule cv_plot_tau_stats: """Overlay tau statistics across all versions.""" input: - tau=[cv_tau_stats(v) for v in CV_VERSIONS], + tau=[cv_tau_stats(v, CV_FIDUCIAL) for v in CV_VERSIONS], output: sentinel=str(CV_SENTINELS / "plot_tau_stats.done"), params: @@ -248,8 +236,8 @@ rule cv_plot_tau_stats: rule cv_rho_tau_fits: """Fit the PSF-error model (alpha/beta/eta) and propagate to xi_psf_sys.""" input: - rho=[cv_rho_stats(v) for v in CV_VERSIONS], - tau=[cv_tau_stats(v) for v in CV_VERSIONS], + rho=[cv_rho_stats(v, CV_FIDUCIAL) for v in CV_VERSIONS], + tau=[cv_tau_stats(v, CV_FIDUCIAL) for v in CV_VERSIONS], output: sentinel=str(CV_SENTINELS / "rho_tau_fits.done"), params: @@ -280,6 +268,9 @@ rule cv_footprints: rule cv_objectwise_leakage: """Object-wise PSF-leakage regression vs scale-dependent alpha (all versions).""" + input: + rho=[cv_rho_stats(v, CV_FIDUCIAL) for v in CV_VERSIONS], + tau=[cv_tau_stats(v, CV_FIDUCIAL) for v in CV_VERSIONS], output: sentinel=str(CV_SENTINELS / "objectwise_leakage.done"), params: @@ -345,8 +336,8 @@ rule cv_ratio_xi_sys_xi: """Ratio of PSF systematics (xi_psf_sys) to the cosmic-shear signal (xi+).""" input: xi=[cv_xi_txt(v) for v in CV_VERSIONS], - rho=[cv_rho_stats(v) for v in CV_VERSIONS], - tau=[cv_tau_stats(v) for v in CV_VERSIONS], + rho=[cv_rho_stats(v, CV_FIDUCIAL) for v in CV_VERSIONS], + tau=[cv_tau_stats(v, CV_FIDUCIAL) for v in CV_VERSIONS], output: ratio=str(COSMO_VAL / "ratio_xi_sys_xi.png"), params: diff --git a/workflow/rules/covariance.smk b/workflow/rules/covariance.smk index 1bc0efd0..c028ec99 100644 --- a/workflow/rules/covariance.smk +++ b/workflow/rules/covariance.smk @@ -212,17 +212,14 @@ rule covariance_glass_mock: "../scripts/compute_glass_mock_covariance.py" -# fiducial_binning_suffix() defined in Snakefile - - rule generate_glass_mock_rhotau_samples: """Generate sampled tau statistics for glass mocks. Only tau is sampled; inference_prep_glass_mock uses real rho data. """ input: - cov_tau=str(COSMO_VAL / f"rho_tau_stats/cov_tau_{FIDUCIAL['mock_version']}{fiducial_binning_suffix()}_th.npy"), - ref_tau=str(COSMO_VAL / f"rho_tau_stats/tau_stats_{FIDUCIAL['mock_version']}{fiducial_binning_suffix()}.fits"), + cov_tau=cv_cov_tau(FIDUCIAL["mock_version"]), + ref_tau=cv_tau_stats(FIDUCIAL["mock_version"]), output: tau="results/glass_mock_rhotau_samples/{mock_id}/tau_stats_sampled.fits", params: diff --git a/workflow/rules/inference.smk b/workflow/rules/inference.smk index 735de462..154fa9ac 100644 --- a/workflow/rules/inference.smk +++ b/workflow/rules/inference.smk @@ -58,10 +58,10 @@ rule inference_prep: # n(z) file: the catalogue entry's nz_file=lambda w: redshift_path(w.version), # rho/tau stats - rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + rho_stats=lambda w: cv_rho_stats(w.version, w), + tau_stats=lambda w: cv_tau_stats(w.version, w), # tau covariance (tracked as dependency) - tau_cov=str(COSMO_VAL / "rho_tau_stats/cov_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}_th.npy"), + tau_cov=lambda w: cv_cov_tau(w.version, w), pseudo_cl=lambda w: pseudo_cl_assets(w.version)[0], pseudo_cl_cov=lambda w: pseudo_cl_assets(w.version)[1], output: @@ -130,10 +130,10 @@ rule inference_prep_glass_mock: # n(z) file nz_file=redshift_path(FIDUCIAL["mock_version"]), # Rho/tau stats: rho from real data, tau sampled - rho_stats=str(COSMO_VAL / f"rho_tau_stats/rho_stats_{FIDUCIAL['mock_version']}{fiducial_binning_suffix()}.fits"), + rho_stats=cv_rho_stats(FIDUCIAL["mock_version"]), tau_stats="results/glass_mock_rhotau_samples/{mock_id}/tau_stats_sampled.fits", # Tau covariance (real data) - tau_cov=str(COSMO_VAL / f"rho_tau_stats/cov_tau_{FIDUCIAL['mock_version']}{fiducial_binning_suffix()}_th.npy"), + tau_cov=cv_cov_tau(FIDUCIAL["mock_version"]), # C_ell data for dual config generation cl_file=f"{GLASS_MOCK_DIR}/cl_glass_mock_{{mock_id}}_4096.npy", cl_cov=pseudo_cl_assets(FIDUCIAL["mock_version"])[1], diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 15174fa2..93abebe4 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -22,7 +22,7 @@ rule xi: input: catalog=get_shear_catalog, output: - txt=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), + txt=str(COSMO_VAL / "xi_{version}_tomo_bin_all_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), sacc=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 24 params: @@ -46,10 +46,10 @@ rule xi: rule rho_tau_stats: output: - rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_tomo_bin_all_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_tomo_bin_all_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), # Born-as-SACC ρ/τ part, written alongside the FITS. - rho_tau=str(COSMO_VAL / "rho_tau_stats/rho_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), + rho_tau=str(COSMO_VAL / "rho_tau_stats/rho_tau_{version}_tomo_bin_all_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 48 params: ver="{version}", diff --git a/workflow/scripts/cv_plot_2pcf.py b/workflow/scripts/cv_plot_2pcf.py index 5251e7a5..c589edb3 100644 --- a/workflow/scripts/cv_plot_2pcf.py +++ b/workflow/scripts/cv_plot_2pcf.py @@ -1,14 +1,14 @@ -"""Rule cv_plot_2pcf: n_pairs / xi± overlay across versions. +"""Rule cv_plot_2pcf: xi± overlay across versions. -Reads each version's xi txt (declared inputs, produced by cv_2pcf); calls -plot_2pcf, which re-reads the existing txt files rather than recomputing. -Writes figures under the output dir. Sentinel-tracked: plot_2pcf emits several -figures whose names are internal. +Reads each version's xi txt (declared inputs, produced by the xi rule); calls +plot_2pcf, which reads the existing txt files back rather than recomputing, for +the non-tomographic ("all", "all") pair. Writes figures under the output dir. +Sentinel-tracked: plot_2pcf emits several figures whose names are internal. """ from cv_runner import _unbuffer_streams, make_cv, touch_sentinels _unbuffer_streams() cv = make_cv(snakemake) -cv.plot_2pcf() +cv.plot_2pcf(tomography=False, show=False) touch_sentinels(snakemake) diff --git a/workflow/scripts/cv_plot_rho_stats.py b/workflow/scripts/cv_plot_rho_stats.py index c99bd646..a55a1377 100644 --- a/workflow/scripts/cv_plot_rho_stats.py +++ b/workflow/scripts/cv_plot_rho_stats.py @@ -9,5 +9,5 @@ _unbuffer_streams() cv = make_cv(snakemake) -cv.plot_rho_stats() +cv.plot_rho_stats(savefig="rho_stats.png", show=False) touch_sentinels(snakemake) diff --git a/workflow/scripts/cv_plot_tau_stats.py b/workflow/scripts/cv_plot_tau_stats.py index 2a00b3a5..a2c6f365 100644 --- a/workflow/scripts/cv_plot_tau_stats.py +++ b/workflow/scripts/cv_plot_tau_stats.py @@ -1,6 +1,7 @@ """Rule cv_plot_tau_stats: tau-statistics overlay across versions. -Reads tau_stats_{base}.fits for every version (declared inputs); writes +Reads tau_stats_{base}.fits for every version (declared inputs), with error +bars from the version's tau covariance of the configured type; writes tau_stats.png into the leakage output dir. Sentinel-tracked (see rho_stats). """ @@ -8,5 +9,10 @@ _unbuffer_streams() cv = make_cv(snakemake) -cv.plot_tau_stats() +cv.plot_tau_stats( + tomography=False, + cov_type=cv.cov_estimate_method, + savefig="tau_stats.png", + show=False, +) touch_sentinels(snakemake) diff --git a/workflow/scripts/cv_ratio_xi_sys_xi.py b/workflow/scripts/cv_ratio_xi_sys_xi.py index 79afd89a..3aebd8b8 100644 --- a/workflow/scripts/cv_ratio_xi_sys_xi.py +++ b/workflow/scripts/cv_ratio_xi_sys_xi.py @@ -1,15 +1,18 @@ """Rule cv_ratio_xi_sys_xi: ratio of PSF systematics to cosmic-shear signal. -Joins two upstream chains: the 2pcf data vector (xi txt, reloaded via -calculate_2pcf) and xi_psf_sys (recomputed in memory from the rho/tau FITS via -the lazy cv.xi_psf_sys property — the PSF-error fit is not persisted by -cosmo_val.py). Both are declared as inputs so the DAG shows the join. Writes -ratio_xi_sys_xi.png at a fixed path (declared output). +Joins two upstream chains for the non-tomographic ("all", "all") pair: the +2pcf data vector (xi txt, read back by calculate_2pcf) and xi_psf_sys +(recomputed in memory from the rho/tau FITS via the lazy cv.xi_psf_sys +property — the PSF-error fit is not persisted). Both are declared as inputs so +the DAG shows the join. Writes ratio_xi_sys_xi.png at a fixed path (declared +output). """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs _unbuffer_streams() cv = make_cv(snakemake) -cv.plot_ratio_xi_sys_xi(offset=snakemake.params.get("offset", 0.1)) +cv.plot_ratio_xi_sys_xi( + tomography=False, offset=snakemake.params.get("offset", 0.1), show=False +) verify_outputs(snakemake) diff --git a/workflow/scripts/cv_rho_tau_fits.py b/workflow/scripts/cv_rho_tau_fits.py index 64b24b67..6e02ba10 100644 --- a/workflow/scripts/cv_rho_tau_fits.py +++ b/workflow/scripts/cv_rho_tau_fits.py @@ -13,5 +13,10 @@ _unbuffer_streams() cv = make_cv(snakemake) if cv.rho_tau_method != "none": - cv.plot_rho_tau_fits() + cv.plot_rho_tau_fits( + tomography=False, + savefig_contours="contours_tau_stat.png", + savefig_xi_psf_sys="xi_psf_sys", + show=False, + ) touch_sentinels(snakemake) diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index e1b96da6..53679944 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -126,7 +126,7 @@ def generate_pseudo_cl( cv = CosmologyValidation(**cv_kwargs) # Pseudo-Cls only (no covariance), born directly at the final out_path. - cv.calculate_pseudo_cl(out_path=out_path) + cv.calculate_pseudo_cl(compute_tomography=False, out_path=out_path) if os.path.exists(out_path): # Readback of the part just written — a legitimate pre-blind consumer. diff --git a/workflow/scripts/generate_pseudo_cl_cov.py b/workflow/scripts/generate_pseudo_cl_cov.py index 247c5f33..0364f1bf 100644 --- a/workflow/scripts/generate_pseudo_cl_cov.py +++ b/workflow/scripts/generate_pseudo_cl_cov.py @@ -1,13 +1,13 @@ """Generate pseudo-Cl covariances (no data vector). Dual-mode. Under Snakemake (``script:`` directive) the injected ``snakemake`` -object supplies the parameters and the native product is renamed to the tagged -output filename the rule declares; as a standalone CLI (argparse) the same -compute runs from explicit flags and the primitive's native -``pseudo_cl_cov_{ver}.fits`` is left in place under ``--out`` (no rename — each -lc/ASTRA recipe gets its own output directory, so the untagged native name is -unambiguous and the primitives' skip-if-exists never collides across nbins -runs). The CLI form is what the lightcone/ASTRA recipe calls, so the +object supplies the parameters, the covariance is computed in a private +working directory and moved to the tagged output filename the rule declares; +as a standalone CLI (argparse) the same compute runs from explicit flags and +the native ``pseudo_cl/pseudo_cl_cov_non_tomo_{ver}_from_iNKA_…fits`` (with its +``iNKA_block_{ver}/`` block) is left in place under ``--out`` (each lc/ASTRA +recipe gets its own output directory, so the methods' skip-if-exists never +reuses another configuration's covariance). The CLI form is what the lightcone/ASTRA recipe calls, so the measurement is driven directly (no nested Snakemake) with lc handling orchestration: @@ -25,8 +25,11 @@ """ import argparse +import gc import json import os +import shutil +import tempfile from astropy.io import fits @@ -51,9 +54,10 @@ def generate_pseudo_cl_cov( version : str Catalog version (e.g., "SP_v1.4.6_leak_corr") output_dir : str - Directory the covariance FITS file is written into. The primitive writes - its native ``pseudo_cl_cov_{version}.fits`` here; callers that need a - tagged filename rename it themselves (see ``_from_snakemake``). + Output directory of the ``CosmologyValidation`` run. The covariance is + written to its native path under ``output_dir/pseudo_cl/``; callers + that need a tagged filename move it themselves (see + ``_from_snakemake``). cat_config : str Path to catalog configuration YAML nside : int @@ -73,7 +77,7 @@ def generate_pseudo_cl_cov( Returns ------- str - Path to the primitive's native ``pseudo_cl_cov_{version}.fits`` product. + Path to the native non-tomographic iNKA covariance FITS. """ os.makedirs(output_dir, exist_ok=True) @@ -129,39 +133,47 @@ def generate_pseudo_cl_cov( cv = CosmologyValidation(**cv_kwargs) - # Calculate covariance only + # Covariance of the ("all", "all") pair only print("Calculating covariance...") - cv.calculate_pseudo_cl_eb_cov() + cv.calculate_pseudo_cl_inka_cov(compute_tomography=False) - # Report on the native product (renamed by the Snakemake caller, if any) - src_cov = os.path.join(output_dir, f"pseudo_cl_cov_{version}.fits") + # Report on the native product (moved by the Snakemake caller, if any) + src_cov = cv._output_path_pseudo_cl_cov(version, "iNKA", tomography=False) if os.path.exists(src_cov): with fits.open(src_cov) as hdul: # CV outputs covariance blocks as COVAR_XX_YY extensions - cov = hdul["COVAR_BB_BB"].data - n_ell = int(len(cov) ** 0.5) - print(f"Generated covariance matrix: {n_ell}x{n_ell}") + n_row, n_col = hdul["COVAR_BB_BB"].data.shape + print(f"Generated BB covariance block: {n_row}x{n_col}") return src_cov def _from_snakemake(smk): p = smk.params output_cov = smk.output.pseudo_cl_cov - src_cov = generate_pseudo_cl_cov( - version=p["version"], - output_dir=os.path.dirname(output_cov), - cat_config=p["cat_config"], - nside=int(p["nside"]), - npatch=int(p["npatch"]), - cosmo_params=p.get("cosmo_params", None), - binning=p["binning"], - nbins=int(p["nbins"]), - power=float(p.get("power", 0.5)), - ) - # Snakemake declares a tagged output filename; rename the native product to it. - if os.path.exists(src_cov) and src_cov != output_cov: - os.rename(src_cov, output_cov) - print(f"Saved to: {output_cov}") + out_dir = os.path.dirname(output_cov) + os.makedirs(out_dir, exist_ok=True) + # The iNKA blocks and the merged covariance are cached under names that + # carry the binning only, so the job computes in a directory of its own and + # never picks up a block another configuration left behind. On NFS, files + # still held open at exit leave .nfs placeholders that block the removal, + # so a leftover work directory is tolerated rather than failing the job. + with tempfile.TemporaryDirectory( + dir=out_dir, prefix=".pseudo_cl_cov_", ignore_cleanup_errors=True + ) as work: + src_cov = generate_pseudo_cl_cov( + version=p["version"], + output_dir=work, + cat_config=p["cat_config"], + nside=int(p["nside"]), + npatch=int(p["npatch"]), + cosmo_params=p.get("cosmo_params", None), + binning=p["binning"], + nbins=int(p["nbins"]), + power=float(p.get("power", 0.5)), + ) + shutil.move(src_cov, output_cov) + gc.collect() + print(f"Saved to: {output_cov}") def _from_cli(argv=None): diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 28c8c521..1fcd9a0c 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -14,7 +14,8 @@ The measurement is binning-agnostic: the reporting and the fine integration grids are the same compute with different ``--min-sep/--max-sep/--nbins``. -``CosmologyValidation.calculate_2pcf`` writes the ``.txt`` dump (a raw +``CosmologyValidation.calculate_2pcf_version`` measures the non-tomographic +``("all", "all")`` pair and writes its ``xi_{basename}.txt`` dump (a raw byproduct); the ξ± data product is born as SACC here, a *part* named by its binning and tagged with its ``--grid``. The part carries the covariance the measurement estimated: the dense jackknife covariance when it had patches, the @@ -68,13 +69,14 @@ def run_2pcf( # so the SACC provenance metadata stamps the npatch actually measured npatch=npatch, ) - gg = cv.calculate_2pcf( - ver=ver, + gg = cv.calculate_2pcf_version( + ver, npatch=npatch, + compute_tomography=False, min_sep=min_sep, max_sep=max_sep, nbins=nbins, - ) + )["tomo_bin_all_tomo_bin_all"] # Born-as-SACC ξ± part. theta = meanr; theta_nom = rnom. jackknife = gg.var_method == "jackknife" diff --git a/workflow/scripts/run_glass_mock_pseudo_cl.py b/workflow/scripts/run_glass_mock_pseudo_cl.py index 9d90c080..9f9e9371 100644 --- a/workflow/scripts/run_glass_mock_pseudo_cl.py +++ b/workflow/scripts/run_glass_mock_pseudo_cl.py @@ -4,7 +4,7 @@ NaMaster estimator. Matches the real data pipeline binning exactly. Mock catalogs have columns: RA, Dec, e1, e2, w. -Polarization convention: pol_factor=True → field=[e1, -e2] (matches pipeline). +Polarization convention: pol_factor=-1 → field=[e1, -e2] (matches pipeline). """ import numpy as np @@ -49,7 +49,7 @@ print(f"Binning: {nbins} powspace bins, ell=[{ell_eff[0]:.1f}, {ell_eff[-1]:.1f}]") # Create NaMaster spin-2 field from catalog -# pol_factor=True: e2 sign flip to match IAU polarization convention +# pol_factor=-1: e2 sign flip to match IAU polarization convention print("Creating NaMaster field...") f_all = nmt.NmtFieldCatalog( positions=[ra, dec], diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index a798a1b6..d04da3f0 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -19,5 +19,5 @@ catalog_config=params["cat_config"], output_dir=params["output_dir"], ) -cv.calculate_rho_tau_stats() +cv.calculate_rho_tau_stats(tomography=False) verify_outputs(snakemake) diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py index b11acf3b..438f91a1 100644 --- a/workflow/tests/test_dag.py +++ b/workflow/tests/test_dag.py @@ -33,7 +33,7 @@ def test_assemble_resolves(toy): f"pseudo_cl_cov_{version}_{harmonic}.fits", f"{version}_cosebis.sacc", f"{version}_pure_eb.sacc", - f"rho_tau_{version}_{reporting}.sacc", + f"rho_tau_{version}_tomo_bin_all_{reporting}.sacc", }, job.input diff --git a/workflow/tests/test_glass_suite.py b/workflow/tests/test_glass_suite.py index 187bde09..50e4e84b 100644 --- a/workflow/tests/test_glass_suite.py +++ b/workflow/tests/test_glass_suite.py @@ -39,17 +39,17 @@ def test_glass_rules_share_suite(toy, tmp_path, suite): (data_dir / f"cl_glass_mock_{seed:05d}_4096.npy").touch() version = config["fiducial"]["mock_version"] - suffix = toy.common.fiducial_binning_suffix(config["fiducial"]) tag = toy.common.pseudo_cl_tag(config) + base = toy.common.cv_basename(version, config["fiducial"]) dependencies = [ toy.root / "data" / "nz_SP_v0.1_A.txt", toy.cosmo_val / f"pseudo_cl_cov_{version}_{tag}.fits", *[ toy.cosmo_val / "rho_tau_stats" / name for name in ( - f"rho_stats_{version}{suffix}.fits", - f"tau_stats_{version}{suffix}.fits", - f"cov_tau_{version}{suffix}_th.npy", + f"rho_stats_{base}.fits", + f"tau_stats_{base}.fits", + f"cov_tau_{base}_th.npy", ) ], ]