From fa9a8807369bb1d55b0e048f1ebc1f91da80b45d Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Thu, 27 Mar 2025 17:33:06 -0400 Subject: [PATCH 01/56] Planck coadded beams? --- sofind/products/beams/beams_planck_npipe.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sofind/products/beams/beams_planck_npipe.yaml b/sofind/products/beams/beams_planck_npipe.yaml index 237cb52..96c11a7 100644 --- a/sofind/products/beams/beams_planck_npipe.yaml +++ b/sofind/products/beams/beams_planck_npipe.yaml @@ -1,7 +1,8 @@ system_paths: della: perlmutter: - niagara: /gpfs/fs0/project/r/rbond/jiaqu/actpol/ +# niagara: /gpfs/fs0/project/r/rbond/jiaqu/actpol/ + niagara: /gpfs/fs1/scratch/r/rbond/jaejoonk/planck_beams_from_nersc/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/beams/planck_beams/ allowed_qids_configs: @@ -12,6 +13,6 @@ allowed_qids: 'all' allowed_qids_extra_kwargs: split_beam_file_template: 'npipe_DR6_{split_num}x{split_num}/bl_T_npipe_DR6_{split_num}x{split_num}_{freq}{split_num}x{freq}{split_num}.dat' -# coadd_beam_file_template: 'coadd_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' +coadd_beam_file_template: 'npipe_DR6_coadd/bl_T_npipe_DR6_coadd_{freq}GHzx{freq}GHz.dat' -norm: False \ No newline at end of file +norm: False From b5b02476b1e0a268db4c215a422f9bce63ee4713 Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Thu, 27 Mar 2025 17:49:57 -0400 Subject: [PATCH 02/56] planck beams for coadd + gpc --- sofind/datamodels/act_dr6v4.yaml | 3 ++- sofind/products/beams/beams_planck_npipe.yaml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index e2d8d37..706972d 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -11,6 +11,7 @@ masks: mnms_masks_config: mnms_masks.yaml lensing_masks: lensing_masks.yaml lensing_masks_20240919b: lensing_masks_20240919b.yaml + lensing_masks_20250318: lensing_masks_20250318.yaml catalogs: inpaint_catalogs_config: inpaint_catalogs.yaml @@ -43,4 +44,4 @@ beams: transfer_func: tf_dr6v4_230523: tf_dr6v4_230523.yaml tf_dr6v4_240416: tf_dr6v4_240416.yaml - tf_dummy: tf_dummy.yaml \ No newline at end of file + tf_dummy: tf_dummy.yaml diff --git a/sofind/products/beams/beams_planck_npipe.yaml b/sofind/products/beams/beams_planck_npipe.yaml index 96c11a7..22be5b1 100644 --- a/sofind/products/beams/beams_planck_npipe.yaml +++ b/sofind/products/beams/beams_planck_npipe.yaml @@ -4,6 +4,7 @@ system_paths: # niagara: /gpfs/fs0/project/r/rbond/jiaqu/actpol/ niagara: /gpfs/fs1/scratch/r/rbond/jaejoonk/planck_beams_from_nersc/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/beams/planck_beams/ + penn-gpc: /data6/DR6_plus/planck_beams/ allowed_qids_configs: - planck_qids.yaml From ba7629cd8a40076a8ca49fb2714539d320cbe9dc Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Tue, 17 Jun 2025 15:32:53 -0400 Subject: [PATCH 03/56] rebasing planck_beams to main --- sofind/products/masks/lensing_masks_20250318.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 sofind/products/masks/lensing_masks_20250318.yaml diff --git a/sofind/products/masks/lensing_masks_20250318.yaml b/sofind/products/masks/lensing_masks_20250318.yaml new file mode 100644 index 0000000..edfc7c1 --- /dev/null +++ b/sofind/products/masks/lensing_masks_20250318.yaml @@ -0,0 +1,14 @@ +system_paths: + perlmutter: /global/cfs/projectdirs/act/data/masks/dr6v4_lensing_20250318_masks/baseline/ + niagara: /gpfs/fs0/project/r/rbond/msyriac/masks/dr6v4_lensing_20250318_masks/baseline/ + dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/masks/ + penn-gpc: /data5/act/masks/dr6v4_lensing_20250318_masks/baseline/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +dr6v4_20250318: + mask_fn: dr6v4_lensing_20251803_{daynight}_enhanced_mask_{skyfrac}.fits \ No newline at end of file From 8e7fdcdf2e17dc4216950ec8f905ed6124f954b3 Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Tue, 17 Jun 2025 16:25:44 -0400 Subject: [PATCH 04/56] added gpc path for planck maps --- sofind/products/maps/planck_npipe.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sofind/products/maps/planck_npipe.yaml b/sofind/products/maps/planck_npipe.yaml index ceeb492..ba80c97 100644 --- a/sofind/products/maps/planck_npipe.yaml +++ b/sofind/products/maps/planck_npipe.yaml @@ -1,7 +1,8 @@ system_paths: perlmutter: /global/cfs/projectdirs/act/data/planck/npipe/fejer1_equ/ niagara: /gpfs/fs0/project/r/rbond/sigurdkn/actpol/planck/npipe/fejer1_equ/ - + penn-gpc: /data6/DR6_plus/npipe_fejer1_equ/ + allowed_qids_configs: - planck_qids.yaml From 2556faef10b4f009a312b87bed2d3979f1660a68 Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Tue, 17 Jun 2025 16:27:53 -0400 Subject: [PATCH 05/56] dummy poleff/calibration for Planck + gpc path --- sofind/products/calibrations/dummy.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sofind/products/calibrations/dummy.yaml b/sofind/products/calibrations/dummy.yaml index 1d32dfd..f26ddff 100644 --- a/sofind/products/calibrations/dummy.yaml +++ b/sofind/products/calibrations/dummy.yaml @@ -3,6 +3,7 @@ system_paths: della: perlmutter: niagara: /scratch/r/rbond/iabril/ACT_DR6_products/calibration/dummy/ + penn-gpc: /data6/DR6_plus/dummy_products/calibration/dummy/ allowed_qids_configs: 'all' @@ -13,4 +14,4 @@ allowed_qids_extra_kwargs: subproduct_kwargs_orders: cals_file_template: dummy_cals.pkl poleffs_file_template: dummy_poleffs.pkl -key_template: all \ No newline at end of file +key_template: all From e3665de976910ef5e9779af613eafcce62b50d8f Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Mon, 30 Jun 2025 14:35:19 -0400 Subject: [PATCH 06/56] night+day+Planck data compatibility with gpc --- sofind/datamodels/act_dr6v4.yaml | 1 + sofind/datamodels/planck_npipe.yaml | 6 +++++- sofind/products/calibrations/dr6v4_calday.yaml | 3 ++- sofind/products/masks/lensing_masks_20250513.yaml | 14 ++++++++++++++ sofind/products/noise_models/act_dr6v4.yaml | 3 ++- sofind/products/transfer_func/tf_dummy.yaml | 2 +- 6 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 sofind/products/masks/lensing_masks_20250513.yaml diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index 706972d..cea440c 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -12,6 +12,7 @@ masks: lensing_masks: lensing_masks.yaml lensing_masks_20240919b: lensing_masks_20240919b.yaml lensing_masks_20250318: lensing_masks_20250318.yaml + lensing_masks_20250513: lensing_masks_20250513.yaml catalogs: inpaint_catalogs_config: inpaint_catalogs.yaml diff --git a/sofind/datamodels/planck_npipe.yaml b/sofind/datamodels/planck_npipe.yaml index cdb996e..2efdde6 100644 --- a/sofind/datamodels/planck_npipe.yaml +++ b/sofind/datamodels/planck_npipe.yaml @@ -13,4 +13,8 @@ calibrations: dummy: dummy.yaml transfer_func: - tf_dummy: tf_dummy.yaml \ No newline at end of file + tf_dummy: tf_dummy.yaml + +masks: + lensing_masks_20250513: lensing_masks_20250513.yaml + lensing_masks_20240919b: lensing_masks_20240919b.yaml diff --git a/sofind/products/calibrations/dr6v4_calday.yaml b/sofind/products/calibrations/dr6v4_calday.yaml index ed7b333..7d471a1 100644 --- a/sofind/products/calibrations/dr6v4_calday.yaml +++ b/sofind/products/calibrations/dr6v4_calday.yaml @@ -3,6 +3,7 @@ system_paths: della: perlmutter: niagara: /scratch/r/rbond/iabril/ACT_DR6_products/calibration/dr6v4_calday/ + penn-gpc: /data6/DR6_plus/dr6v4_calday/ allowed_qids_configs: - act_dr6vX_qids.yaml @@ -20,4 +21,4 @@ allowed_qids_extra_kwargs: subproduct_kwargs_orders: cals_file_template: gains.pkl poleffs_file_template: poleffs.pkl -key_template: dr6_{array}_{freq} \ No newline at end of file +key_template: dr6_{array}_{freq} diff --git a/sofind/products/masks/lensing_masks_20250513.yaml b/sofind/products/masks/lensing_masks_20250513.yaml new file mode 100644 index 0000000..9bfc8f7 --- /dev/null +++ b/sofind/products/masks/lensing_masks_20250513.yaml @@ -0,0 +1,14 @@ +system_paths: + perlmutter: + niagara: /gpfs/fs0/project/r/rbond/msyriac/masks/dr6v4_lensing_250513_masks/ + dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/masks/ + penn-gpc: /data6/DR6_plus/masks/dr6v4_lensing_250513_masks/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +dr6v4_20250513: + mask_fn: dr6v4_lensing_250513_{daynight}_enhanced_mask_{skyfrac}.fits diff --git a/sofind/products/noise_models/act_dr6v4.yaml b/sofind/products/noise_models/act_dr6v4.yaml index 171fe90..20b78b6 100644 --- a/sofind/products/noise_models/act_dr6v4.yaml +++ b/sofind/products/noise_models/act_dr6v4.yaml @@ -3,6 +3,7 @@ system_paths: perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/simulations/mnms/ niagara: /scratch/r/rbond/jaejoonk/dr6plus_noise_sims/ + penn-gpc: /data6/DR6_plus/noise_sims/ allowed_qids_configs: - act_dr6vX_qids.yaml @@ -276,4 +277,4 @@ fdw_cmbmask_ivfwhm2: w_lmax_j: 5300 w_lmin: 10 model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' - sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' \ No newline at end of file + sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' diff --git a/sofind/products/transfer_func/tf_dummy.yaml b/sofind/products/transfer_func/tf_dummy.yaml index 67cea01..3cbdd69 100644 --- a/sofind/products/transfer_func/tf_dummy.yaml +++ b/sofind/products/transfer_func/tf_dummy.yaml @@ -2,7 +2,7 @@ system_paths: perlmutter: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/transfer_functions/dummy_tf/ niagara: /scratch/r/rbond/jaejoonk/lensing_pipeline_data/transfer_functions/dummy_tf/ - penn-gpc: + penn-gpc: /data6/DR6_plus/dummy_products/dummy_tf/ allowed_qids_configs: 'all' From 32f716a6ac3ab0c008b0d906ccf59c4b914e14ad Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Tue, 1 Jul 2025 11:36:35 -0400 Subject: [PATCH 07/56] fixed planck beams path --- sofind/products/beams/beams_planck_npipe.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sofind/products/beams/beams_planck_npipe.yaml b/sofind/products/beams/beams_planck_npipe.yaml index 22be5b1..2ef37e1 100644 --- a/sofind/products/beams/beams_planck_npipe.yaml +++ b/sofind/products/beams/beams_planck_npipe.yaml @@ -2,7 +2,7 @@ system_paths: della: perlmutter: # niagara: /gpfs/fs0/project/r/rbond/jiaqu/actpol/ - niagara: /gpfs/fs1/scratch/r/rbond/jaejoonk/planck_beams_from_nersc/ + niagara: /gpfs/fs0/scratch/r/rbond/jaejoonk/planck_beams_from_nersc/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/beams/planck_beams/ penn-gpc: /data6/DR6_plus/planck_beams/ From f1fb72b1678ab7b09ce41c50a21a5327f4e523b5 Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Fri, 4 Jul 2025 02:08:49 -0400 Subject: [PATCH 08/56] added more gpc paths --- sofind/products/beams/beams_dummy.yaml | 4 ++-- .../products/calibrations/dr6v4_calday_tonight_daydeep.yaml | 3 ++- .../products/calibrations/dr6v4_calday_tonight_daywide.yaml | 3 ++- sofind/products/noise_models/act_dr6v4_day.yaml | 1 + 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sofind/products/beams/beams_dummy.yaml b/sofind/products/beams/beams_dummy.yaml index 201e4cf..ae3dd6e 100644 --- a/sofind/products/beams/beams_dummy.yaml +++ b/sofind/products/beams/beams_dummy.yaml @@ -2,7 +2,7 @@ system_paths: perlmutter: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/v4_maps/dummy_beams/ niagara: /scratch/r/rbond/iabril/ACT_DR6_products/beams/dummy_beams/ - penn-gpc: + penn-gpc: /data6/DR6_plus/dummy_products/beams/dummy_beams/ allowed_qids_configs: 'all' @@ -13,4 +13,4 @@ allowed_qids_extra_kwargs: split_beam_file_template: 'dummy_beam.txt' coadd_beam_file_template: 'dummy_beam.txt' -norm: False \ No newline at end of file +norm: False diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daydeep.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daydeep.yaml index 0323a0f..c7bc6eb 100644 --- a/sofind/products/calibrations/dr6v4_calday_tonight_daydeep.yaml +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daydeep.yaml @@ -3,6 +3,7 @@ system_paths: della: perlmutter: niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/cal_day_to_nightdefault/ + penn-gpc: /data6/DR6_plus/calibrations/cal_day_to_nightdefault/ allowed_qids_configs: - act_dr6vX_qids.yaml @@ -18,4 +19,4 @@ allowed_qids_extra_kwargs: subproduct_kwargs_orders: cals_file_template: cal_day_to_nightdefault_daydeep.pkl poleffs_file_template: -key_template: dr6_{array}_{freq} \ No newline at end of file +key_template: dr6_{array}_{freq} diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daywide.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daywide.yaml index 31708c8..c53578c 100644 --- a/sofind/products/calibrations/dr6v4_calday_tonight_daywide.yaml +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daywide.yaml @@ -3,6 +3,7 @@ system_paths: della: perlmutter: niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/cal_day_to_nightdefault/ + penn-gpc: /data6/DR6_plus/calibrations/cal_day_to_nightdefault/ allowed_qids_configs: - act_dr6vX_qids.yaml @@ -16,4 +17,4 @@ allowed_qids_extra_kwargs: subproduct_kwargs_orders: cals_file_template: cal_day_to_nightdefault_daywide.pkl poleffs_file_template: -key_template: dr6_{array}_{freq} \ No newline at end of file +key_template: dr6_{array}_{freq} diff --git a/sofind/products/noise_models/act_dr6v4_day.yaml b/sofind/products/noise_models/act_dr6v4_day.yaml index c8992db..68e5adf 100644 --- a/sofind/products/noise_models/act_dr6v4_day.yaml +++ b/sofind/products/noise_models/act_dr6v4_day.yaml @@ -3,6 +3,7 @@ system_paths: perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/simulations/mnms/ niagara: /scratch/r/rbond/jaejoonk/dr6plus_noise_sims/ + penn-gpc: /data6/DR6_plus/noise_sims/ allowed_qids_configs: - act_dr6vX_qids.yaml From 7692341fc4b8370c0661ba5d31cfe7f13e80f2cc Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Fri, 4 Jul 2025 23:02:27 -0400 Subject: [PATCH 09/56] planck noise sim path for GPC --- sofind/products/maps/planck_npipe_noise_sims.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sofind/products/maps/planck_npipe_noise_sims.yaml b/sofind/products/maps/planck_npipe_noise_sims.yaml index 786a886..0a54300 100644 --- a/sofind/products/maps/planck_npipe_noise_sims.yaml +++ b/sofind/products/maps/planck_npipe_noise_sims.yaml @@ -1,6 +1,7 @@ system_paths: niagara: /project/r/rbond/jiaqu/ - + penn-gpc: /data6/DR6_plus/ + allowed_qids_configs: - planck_qids.yaml From 16d85f41670d5a41c9399063ea37af9df149fd19 Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Thu, 10 Jul 2025 15:12:59 -0400 Subject: [PATCH 10/56] corrected noise_sims path, sorry --- sofind/products/maps/planck_npipe_noise_sims.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sofind/products/maps/planck_npipe_noise_sims.yaml b/sofind/products/maps/planck_npipe_noise_sims.yaml index 0a54300..f8b57ec 100644 --- a/sofind/products/maps/planck_npipe_noise_sims.yaml +++ b/sofind/products/maps/planck_npipe_noise_sims.yaml @@ -39,6 +39,6 @@ allowed_qids_extra_kwargs: # using maptag for sim index (e.g. "0200") and split_num between "A" and "B" # interfaced by solenspipe.preprocessing so shouldn't have to be manually accessed # THESE ARE HEALPIX MAPS -split_map_file_template: 'npipe6v20{split_num}_sim/{maptag}/npipe6v20{split_num}_{freq}_map.fits' +split_map_file_template: 'npipe6v20{split_num}_sim/{maptag}/residual/residual_npipe6v20{split_num}_{freq}_{maptag}.fits' # should really never be using coadd=True # coadd_map_file_template: 'npipe6v20A_sim/{maptag}/npipe6v20A_{freq}_map.fits' From 94ec9f990af2b1eccb66bdce6ac272299f5e0e48 Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Thu, 17 Jul 2025 11:50:47 -0400 Subject: [PATCH 11/56] updated perlmutter paths for dr6+ --- sofind/products/calibrations/dr6v4_cal_240410.yaml | 2 +- sofind/products/calibrations/dr6v4_poleff_231113.yaml | 2 +- sofind/products/masks/lensing_masks_20250513.yaml | 2 +- sofind/products/noise_models/act_dr6v4.yaml | 5 ++++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sofind/products/calibrations/dr6v4_cal_240410.yaml b/sofind/products/calibrations/dr6v4_cal_240410.yaml index 5767c64..1a54ade 100644 --- a/sofind/products/calibrations/dr6v4_cal_240410.yaml +++ b/sofind/products/calibrations/dr6v4_cal_240410.yaml @@ -1,5 +1,5 @@ system_paths: - perlmutter: + perlmutter: /pscratch/sd/j/jaejoonk/nilc_products/calibration/dr6_v4_240410/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/dr6_v4_240410/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_240410/ penn-gpc: /data5/act/DR6_products/calibration/dr6_v4_240410/ diff --git a/sofind/products/calibrations/dr6v4_poleff_231113.yaml b/sofind/products/calibrations/dr6v4_poleff_231113.yaml index f041d65..8d82b7d 100644 --- a/sofind/products/calibrations/dr6v4_poleff_231113.yaml +++ b/sofind/products/calibrations/dr6v4_poleff_231113.yaml @@ -1,5 +1,5 @@ system_paths: - perlmutter: + perlmutter: /pscratch/sd/j/jaejoonk/nilc_products/calibration/dr6_v4_231113/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/dr6_v4_231113/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_231113/ penn-gpc: /data5/act/DR6_products/calibration/dr6_v4_231113/ diff --git a/sofind/products/masks/lensing_masks_20250513.yaml b/sofind/products/masks/lensing_masks_20250513.yaml index 9bfc8f7..6c11028 100644 --- a/sofind/products/masks/lensing_masks_20250513.yaml +++ b/sofind/products/masks/lensing_masks_20250513.yaml @@ -1,5 +1,5 @@ system_paths: - perlmutter: + perlmutter: /global/cfs/cdirs/act/data/iabril/masks/ niagara: /gpfs/fs0/project/r/rbond/msyriac/masks/dr6v4_lensing_250513_masks/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/masks/ penn-gpc: /data6/DR6_plus/masks/dr6v4_lensing_250513_masks/ diff --git a/sofind/products/noise_models/act_dr6v4.yaml b/sofind/products/noise_models/act_dr6v4.yaml index 20b78b6..ba98277 100644 --- a/sofind/products/noise_models/act_dr6v4.yaml +++ b/sofind/products/noise_models/act_dr6v4.yaml @@ -1,6 +1,9 @@ system_paths: della: /scratch/gpfs/ACT/dr6v4/simulations/noise - perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise + # these are alms + perlmutter: /pscratch/sd/j/jaejoonk/nilc_products/noise_sims/ + # these are enmap versions + #perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/simulations/mnms/ niagara: /scratch/r/rbond/jaejoonk/dr6plus_noise_sims/ penn-gpc: /data6/DR6_plus/noise_sims/ From 2dad19a4991e01e259d8e655f3a1b4d01bc8b4ac Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Fri, 1 Aug 2025 20:02:24 -0400 Subject: [PATCH 12/56] default calibration path for gpc --- sofind/products/calibrations/act_dr6.02.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sofind/products/calibrations/act_dr6.02.yaml b/sofind/products/calibrations/act_dr6.02.yaml index a983447..f8754e6 100644 --- a/sofind/products/calibrations/act_dr6.02.yaml +++ b/sofind/products/calibrations/act_dr6.02.yaml @@ -3,6 +3,7 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/calibrations perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/simulations/noise/calibrations niagara: /scratch/r/rbond/iabril/ACT_DR6_products/calibration/dr6_v4_published/ + penn-gpc: /data5/act/DR6_products/calibration/dr6_v4_published/ allowed_qids_configs: - act_dr6vX_qids.yaml @@ -20,4 +21,4 @@ allowed_qids_extra_kwargs: subproduct_kwargs_orders: cals_file_template: act_dr6.02_cals.pkl poleffs_file_template: act_dr6.02_poleffs.pkl -key_template: dr6_{array}_{freq} \ No newline at end of file +key_template: dr6_{array}_{freq} From e862bd2fc3a97162a825fe4843788fa3da2510cb Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Wed, 6 Aug 2025 20:29:09 -0400 Subject: [PATCH 13/56] dr6.02 beams and calibrations for lensing null test splits --- sofind/datamodels/act_dr6v4.yaml | 8 ++++++++ sofind/products/beams/README.md | 11 ++++++++++- sofind/products/beams/beams_dr6.02_el_split.yaml | 16 ++++++++++++++++ .../products/beams/beams_dr6.02_inout_split.yaml | 16 ++++++++++++++++ .../products/beams/beams_dr6.02_pwv_split.yaml | 16 ++++++++++++++++ sofind/products/beams/beams_dr6.02_t_split.yaml | 16 ++++++++++++++++ .../calibrations/act_dr6.02_el_split.yaml | 1 + .../calibrations/act_dr6.02_inout_split.yaml | 1 + .../calibrations/act_dr6.02_pwv_split.yaml | 1 + .../calibrations/act_dr6.02_t_split.yaml | 1 + 10 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 sofind/products/beams/beams_dr6.02_el_split.yaml create mode 100644 sofind/products/beams/beams_dr6.02_inout_split.yaml create mode 100644 sofind/products/beams/beams_dr6.02_pwv_split.yaml create mode 100644 sofind/products/beams/beams_dr6.02_t_split.yaml diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index cea440c..94d80c9 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -34,6 +34,10 @@ calibrations: dr6v4_calday: dr6v4_calday.yaml dr6v4_calday_tonight_daydeep: dr6v4_calday_tonight_daydeep.yaml dr6v4_calday_tonight_daywide: dr6v4_calday_tonight_daywide.yaml + act_dr6.02_el_split: act_dr6.02_el_split.yaml + act_dr6.02_inout_split: act_dr6.02_el_split.yaml + act_dr6.02_pwv_split: act_dr6.02_el_split.yaml + act_dr6.02_t_split: act_dr6.02_el_split.yaml beams: beams_v4_20230130 : beams_v4_20230130.yaml @@ -41,6 +45,10 @@ beams: beams_v4_day_20240115: beams_v4_day_20240115.yaml beams_nemo: beams_nemo.yaml beams_dummy: beams_dummy.yaml + beams_dr6.02_el_split: beams_dr6.02_el_split.yaml + beams_dr6.02_pwv_split: beams_dr6.02_pwv_split.yaml + beams_dr6.02_inout_split: beams_dr6.02_inout_split.yaml + beams_dr6.02_t_split: beams_dr6.02_t_split.yaml transfer_func: tf_dr6v4_230523: tf_dr6v4_230523.yaml diff --git a/sofind/products/beams/README.md b/sofind/products/beams/README.md index 8dcfc09..0e74f44 100644 --- a/sofind/products/beams/README.md +++ b/sofind/products/beams/README.md @@ -10,6 +10,7 @@ | beams_v4_day_20240115.yaml| __coadd__: `pa4a_dw`, `pa4a_dd`, `pa4b_dw`, `pa4b_dd`, `pa5a_dw`, `pa5a_dd`, `pa5b_dw`, `pa5b_dd`, `pa6a_dw`, `pa6a_dd`, `pa6b_dw`, `pa6b_dd`| | beams_planck_npipe.yaml| __splits__: `p01`, `p02`, `p03`, `p04` `p05`, `p06`, `p07`, `p08`, `p09` | beams_dummy.yaml | __coadd__: all, __splits__: all | +| beams_dr6.02_el_split.yaml | __coadd__: `pa4a`, `pa4b`, `pa5a`, `pa5b`, `pa6a`, `pa6b`
__splits__: `pa4a`, `pa4b`, `pa5a`, `pa5b`, `pa6a`, `pa6b`
| ## Code snippets @@ -27,4 +28,12 @@ read_beam(subproduct, qid, coadd=True) Check if beam needs to be normalised, if the information is available: ``` get_if_norm_beam(subproduct) -``` \ No newline at end of file +``` + +## Required keyword arguments + +| Config File (`subproduct`) | `keyword`(s): `value`(s) | +| beams_dr6.02_el_split.yaml | `el_split`: `el1`, `el2`, `el3` | +| beams_dr6.02_pwv_split.yaml | `pwv_split`: `pwv1`, `pwv2` | +| beams_dr6.02_inout_split.yaml | `inout_split`: `inout1`, `inout2` | +| beams_dr6.02_t_split.yaml | `time_split`: `t1`, `t2` | \ No newline at end of file diff --git a/sofind/products/beams/beams_dr6.02_el_split.yaml b/sofind/products/beams/beams_dr6.02_el_split.yaml new file mode 100644 index 0000000..e293b51 --- /dev/null +++ b/sofind/products/beams/beams_dr6.02_el_split.yaml @@ -0,0 +1,16 @@ +system_paths: + perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/beams/main_beams/elevation_split/ + penn-gpc: /data6/DR6_plus/main_beams/elevation_split/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +# requires el_split parameter + +split_beam_file_template: '{el_split}/set{split_num}_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' +coadd_beam_file_template: '{el_split}/coadd_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' + +norm: False diff --git a/sofind/products/beams/beams_dr6.02_inout_split.yaml b/sofind/products/beams/beams_dr6.02_inout_split.yaml new file mode 100644 index 0000000..e765601 --- /dev/null +++ b/sofind/products/beams/beams_dr6.02_inout_split.yaml @@ -0,0 +1,16 @@ +system_paths: + perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/beams/main_beams/inout_split/ + penn-gpc: /data6/DR6_plus/main_beams/inout_split/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +# requires el_split parameter + +split_beam_file_template: '{inout_split}/set{split_num}_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' +coadd_beam_file_template: '{inout_split}/coadd_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' + +norm: False diff --git a/sofind/products/beams/beams_dr6.02_pwv_split.yaml b/sofind/products/beams/beams_dr6.02_pwv_split.yaml new file mode 100644 index 0000000..5b61eb3 --- /dev/null +++ b/sofind/products/beams/beams_dr6.02_pwv_split.yaml @@ -0,0 +1,16 @@ +system_paths: + perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/beams/main_beams/pwv_split/ + penn-gpc: /data6/DR6_plus/main_beams/pwv_split/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +# requires el_split parameter + +split_beam_file_template: '{pwv_split}/set{split_num}_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' +coadd_beam_file_template: '{pwv_split}/coadd_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' + +norm: False diff --git a/sofind/products/beams/beams_dr6.02_t_split.yaml b/sofind/products/beams/beams_dr6.02_t_split.yaml new file mode 100644 index 0000000..083f742 --- /dev/null +++ b/sofind/products/beams/beams_dr6.02_t_split.yaml @@ -0,0 +1,16 @@ +system_paths: + perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/beams/main_beams/time_split/ + penn-gpc: /data6/DR6_plus/main_beams/time_split/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +# requires el_split parameter + +split_beam_file_template: '{t_split}/set{split_num}_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' +coadd_beam_file_template: '{t_split}/coadd_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' + +norm: False diff --git a/sofind/products/calibrations/act_dr6.02_el_split.yaml b/sofind/products/calibrations/act_dr6.02_el_split.yaml index da5f4ef..2da276c 100644 --- a/sofind/products/calibrations/act_dr6.02_el_split.yaml +++ b/sofind/products/calibrations/act_dr6.02_el_split.yaml @@ -1,6 +1,7 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/calibrations perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/simulations/noise/calibrations + penn-gpc: /data6/DR6_plus/calibrations/dr6.02_calibrations allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/act_dr6.02_inout_split.yaml b/sofind/products/calibrations/act_dr6.02_inout_split.yaml index 6cc0548..e32c39e 100644 --- a/sofind/products/calibrations/act_dr6.02_inout_split.yaml +++ b/sofind/products/calibrations/act_dr6.02_inout_split.yaml @@ -1,6 +1,7 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/calibrations perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/simulations/noise/calibrations + penn-gpc: /data6/DR6_plus/calibrations/dr6.02_calibrations allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/act_dr6.02_pwv_split.yaml b/sofind/products/calibrations/act_dr6.02_pwv_split.yaml index e825ee7..d0c7555 100644 --- a/sofind/products/calibrations/act_dr6.02_pwv_split.yaml +++ b/sofind/products/calibrations/act_dr6.02_pwv_split.yaml @@ -1,6 +1,7 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/calibrations perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/simulations/noise/calibrations + penn-gpc: /data6/DR6_plus/calibrations/dr6.02_calibrations allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/act_dr6.02_t_split.yaml b/sofind/products/calibrations/act_dr6.02_t_split.yaml index 8936b1c..bffca2d 100644 --- a/sofind/products/calibrations/act_dr6.02_t_split.yaml +++ b/sofind/products/calibrations/act_dr6.02_t_split.yaml @@ -1,6 +1,7 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/calibrations perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/simulations/noise/calibrations + penn-gpc: /data6/DR6_plus/calibrations/dr6.02_calibrations allowed_qids_configs: - act_dr6vX_qids.yaml From 6791132c9657fa0553929142645c8548594f08d6 Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Sat, 9 Aug 2025 11:11:29 +0100 Subject: [PATCH 14/56] new system paths --- sofind/products/beams/beams_v4_20230902.yaml | 1 + sofind/products/beams/beams_v4_day_20240115.yaml | 2 +- sofind/products/calibrations/act_dr6.02.yaml | 1 + sofind/products/calibrations/dr6v4_calday_tonight_daydeep.yaml | 3 ++- sofind/products/calibrations/dr6v4_calday_tonight_daywide.yaml | 1 + sofind/products/catalogs/inpaint_catalogs.yaml | 1 + sofind/products/maps/act_dr6v4.yaml | 1 + sofind/products/masks/lensing_masks_20250513.yaml | 1 + sofind/products/noise_models/act_dr6v4.yaml | 3 ++- sofind/products/noise_models/act_dr6v4_day.yaml | 3 ++- sofind/products/transfer_func/tf_dr6v4_240416.yaml | 2 +- 11 files changed, 14 insertions(+), 5 deletions(-) diff --git a/sofind/products/beams/beams_v4_20230902.yaml b/sofind/products/beams/beams_v4_20230902.yaml index 5a2cb0b..863db3c 100644 --- a/sofind/products/beams/beams_v4_20230902.yaml +++ b/sofind/products/beams/beams_v4_20230902.yaml @@ -3,6 +3,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/v4_maps/20230902_beams/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/beams_path/20230902/ penn-gpc: /data5/act/beams/20230902_beams/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/20230902_beams/ allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/beams/beams_v4_day_20240115.yaml b/sofind/products/beams/beams_v4_day_20240115.yaml index 6c08b10..4b58e9a 100644 --- a/sofind/products/beams/beams_v4_day_20240115.yaml +++ b/sofind/products/beams/beams_v4_day_20240115.yaml @@ -2,7 +2,7 @@ system_paths: niagara: /gpfs/fs0/project/r/rbond/sigurdkn/actpol/daybeam_srcs/20240115/release/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/v4_maps/20240115_day_beams/ penn-gpc: /data5/act/DR6_products/v4_maps/daytime/20240115_day_beams/ - + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/20240115_day_beams/ allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/act_dr6.02.yaml b/sofind/products/calibrations/act_dr6.02.yaml index a983447..bfb5a99 100644 --- a/sofind/products/calibrations/act_dr6.02.yaml +++ b/sofind/products/calibrations/act_dr6.02.yaml @@ -3,6 +3,7 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/calibrations perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/simulations/noise/calibrations niagara: /scratch/r/rbond/iabril/ACT_DR6_products/calibration/dr6_v4_published/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/act_dr6.02/ allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daydeep.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daydeep.yaml index 0323a0f..492bd14 100644 --- a/sofind/products/calibrations/dr6v4_calday_tonight_daydeep.yaml +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daydeep.yaml @@ -3,7 +3,8 @@ system_paths: della: perlmutter: niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/cal_day_to_nightdefault/ - + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/cal_day_to_nightdefault/ + allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daywide.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daywide.yaml index 31708c8..b7627b0 100644 --- a/sofind/products/calibrations/dr6v4_calday_tonight_daywide.yaml +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daywide.yaml @@ -3,6 +3,7 @@ system_paths: della: perlmutter: niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/cal_day_to_nightdefault/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/cal_day_to_nightdefault/ allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/catalogs/inpaint_catalogs.yaml b/sofind/products/catalogs/inpaint_catalogs.yaml index e7e40c8..40b0e7d 100644 --- a/sofind/products/catalogs/inpaint_catalogs.yaml +++ b/sofind/products/catalogs/inpaint_catalogs.yaml @@ -4,6 +4,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/catalogues/inpaint_catalogs niagara: /home/r/rbond/jiaqu/projects/catalogs/inpaint_catalogs_20241002 penn-gpc: /data5/act/catalogs/inpaint_catalogs_20241002 + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/inpaint_catalogs_20241002/ allowed_qids_configs: 'all' diff --git a/sofind/products/maps/act_dr6v4.yaml b/sofind/products/maps/act_dr6v4.yaml index 26ee581..6d54cda 100644 --- a/sofind/products/maps/act_dr6v4.yaml +++ b/sofind/products/maps/act_dr6v4.yaml @@ -4,6 +4,7 @@ system_paths: niagara: /gpfs/fs0/project/r/rbond/sigurdkn/actpol/maps/dr6v4_20230316/release dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/v4_maps/release_20230316/ penn-gpc: /data5/act/maps/dr6v4_20230316 + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/maps/ allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/masks/lensing_masks_20250513.yaml b/sofind/products/masks/lensing_masks_20250513.yaml index 508ab36..465b379 100644 --- a/sofind/products/masks/lensing_masks_20250513.yaml +++ b/sofind/products/masks/lensing_masks_20250513.yaml @@ -3,6 +3,7 @@ system_paths: niagara: /gpfs/fs0/project/r/rbond/msyriac/masks/dr6v4_lensing_250513_masks/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/masks/ penn-gpc: /data5/act/DR6_products/masks/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/masks/dr6v4_lensing_250513_masks/ allowed_qids_configs: 'all' diff --git a/sofind/products/noise_models/act_dr6v4.yaml b/sofind/products/noise_models/act_dr6v4.yaml index 171fe90..677adcd 100644 --- a/sofind/products/noise_models/act_dr6v4.yaml +++ b/sofind/products/noise_models/act_dr6v4.yaml @@ -3,7 +3,8 @@ system_paths: perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/simulations/mnms/ niagara: /scratch/r/rbond/jaejoonk/dr6plus_noise_sims/ - + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/simulations/mnms/ + allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/noise_models/act_dr6v4_day.yaml b/sofind/products/noise_models/act_dr6v4_day.yaml index c8992db..f66b0c1 100644 --- a/sofind/products/noise_models/act_dr6v4_day.yaml +++ b/sofind/products/noise_models/act_dr6v4_day.yaml @@ -3,7 +3,8 @@ system_paths: perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/simulations/mnms/ niagara: /scratch/r/rbond/jaejoonk/dr6plus_noise_sims/ - + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/simulations/mnms/ + allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/transfer_func/tf_dr6v4_240416.yaml b/sofind/products/transfer_func/tf_dr6v4_240416.yaml index 8579a76..af011d9 100644 --- a/sofind/products/transfer_func/tf_dr6v4_240416.yaml +++ b/sofind/products/transfer_func/tf_dr6v4_240416.yaml @@ -3,7 +3,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/transfer_functions/dr6_v4_legacy_240416/ niagara: /scratch/r/rbond/jaejoonk/lensing_pipeline_data/transfer_functions/dr6_v4_legacy_240416/ penn-gpc: /data5/act/DR6_products/transfer_functions/dr6_v4_legacy_240416/ - + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/transfer_func/dr6_v4_legacy_240416/ allowed_qids_configs: - act_dr6vX_qids.yaml From de85ea58a284967b3e99dfdecc545a647f4489c8 Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Sat, 9 Aug 2025 11:12:17 +0100 Subject: [PATCH 15/56] nemo v3 and v4 templates --- sofind/datamodels/act_nemo.yaml | 3 +- sofind/products/beams/beams_nemo.yaml | 1 + sofind/products/beams/beams_nemo_v4.yaml | 71 ++++++++++++++++++++++++ sofind/products/maps/act_nemov3.yaml | 1 + sofind/products/maps/act_nemov4.yaml | 14 +++++ 5 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 sofind/products/beams/beams_nemo_v4.yaml create mode 100644 sofind/products/maps/act_nemov4.yaml diff --git a/sofind/datamodels/act_nemo.yaml b/sofind/datamodels/act_nemo.yaml index c7d532d..de5e0e2 100644 --- a/sofind/datamodels/act_nemo.yaml +++ b/sofind/datamodels/act_nemo.yaml @@ -1,4 +1,5 @@ qids_config: act_nemo.yaml maps: - nemo_v3: act_nemov3.yaml \ No newline at end of file + nemo_v3: act_nemov3.yaml + nemo_v4: act_nemov4.yaml \ No newline at end of file diff --git a/sofind/products/beams/beams_nemo.yaml b/sofind/products/beams/beams_nemo.yaml index dded487..f3e05ca 100644 --- a/sofind/products/beams/beams_nemo.yaml +++ b/sofind/products/beams/beams_nemo.yaml @@ -3,6 +3,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/beams/planck_beams/ niagara: /scratch/r/rbond/jaejoonk/lensing_pipeline_data/planck_beams/ penn-gpc: /data5/act/beams/planck_beams/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/planck_beams/ allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/beams/beams_nemo_v4.yaml b/sofind/products/beams/beams_nemo_v4.yaml new file mode 100644 index 0000000..9d3e01b --- /dev/null +++ b/sofind/products/beams/beams_nemo_v4.yaml @@ -0,0 +1,71 @@ +system_paths: + della: + dirac: + niagara: + penn-gpc: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/beams_nemo_v4/ + +allowed_qids_configs: + - act_dr6vX_qids.yaml + - planck_qids.yaml + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + p01: + nemo_freq: 'f090' + p02: + nemo_freq: 'f090' + p03: + nemo_freq: 'f090' + p04: + nemo_freq: 'f090' + p05: + nemo_freq: 'f150' + p06: + nemo_freq: 'f150' + p07: + nemo_freq: 'f150' + p08: + nemo_freq: 'f150' + p09: + nemo_freq: 'f150' + pa4a: + nemo_freq: 'f150' + pa4a_dd: + nemo_freq: 'f150' + pa4a_dw: + nemo_freq: 'f150' + pa4b: + nemo_freq: 'f220' + pa4b_dd: + nemo_freq: 'f220' + pa4b_dw: + nemo_freq: 'f220' + pa5a: + nemo_freq: 'f090' + pa5a_dd: + nemo_freq: 'f090' + pa5a_dw: + nemo_freq: 'f090' + pa5b: + nemo_freq: 'f150' + pa5b_dd: + nemo_freq: 'f150' + pa5b_dw: + nemo_freq: 'f150' + pa6a: + nemo_freq: 'f090' + pa6a_dd: + nemo_freq: 'f090' + pa6a_dw: + nemo_freq: 'f090' + pa6b: + nemo_freq: 'f150' + pa6b_dd: + nemo_freq: 'f150' + pa6b_dw: + nemo_freq: 'f150' + +coadd_beam_file_template: 'beam_{nemo_freq}_tform.txt' +norm: True diff --git a/sofind/products/maps/act_nemov3.yaml b/sofind/products/maps/act_nemov3.yaml index 8050f5e..148bd40 100644 --- a/sofind/products/maps/act_nemov3.yaml +++ b/sofind/products/maps/act_nemov3.yaml @@ -2,6 +2,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/models/ niagara: /scratch/r/rbond/jaejoonk/lensing_pipeline_data/models/ penn-gpc: /data5/act/DR6_products/models/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/models/ allowed_qids_configs: - act_nemo.yaml diff --git a/sofind/products/maps/act_nemov4.yaml b/sofind/products/maps/act_nemov4.yaml new file mode 100644 index 0000000..148bd40 --- /dev/null +++ b/sofind/products/maps/act_nemov4.yaml @@ -0,0 +1,14 @@ +system_paths: + dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/models/ + niagara: /scratch/r/rbond/jaejoonk/lensing_pipeline_data/models/ + penn-gpc: /data5/act/DR6_products/models/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/models/ + +allowed_qids_configs: + - act_nemo.yaml + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +coadd_map_file_template: 'nemomodel_dr6_all_clustersSNR5{freq}down2_fejer1.fits' \ No newline at end of file From e349af91a040c1589a45266abd4b853a0066d67b Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Sat, 9 Aug 2025 11:12:28 +0100 Subject: [PATCH 16/56] new mask --- sofind/products/masks/lensing_masks_20250804.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 sofind/products/masks/lensing_masks_20250804.yaml diff --git a/sofind/products/masks/lensing_masks_20250804.yaml b/sofind/products/masks/lensing_masks_20250804.yaml new file mode 100644 index 0000000..208e847 --- /dev/null +++ b/sofind/products/masks/lensing_masks_20250804.yaml @@ -0,0 +1,12 @@ +system_paths: + perlmutter: + niagara: + dirac: + penn-gpc: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/masks/dr6v4_lensing_250804_masks/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: \ No newline at end of file From ee509350a57d30de84de57ce13132f7a216d35b4 Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Sat, 9 Aug 2025 11:12:50 +0100 Subject: [PATCH 17/56] leakage, nominal --- sofind/products/beams/beams_v4_leakage.yaml | 24 +++++++++++++++++++++ sofind/products/beams/beams_v4_nominal.yaml | 24 +++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 sofind/products/beams/beams_v4_leakage.yaml create mode 100644 sofind/products/beams/beams_v4_nominal.yaml diff --git a/sofind/products/beams/beams_v4_leakage.yaml b/sofind/products/beams/beams_v4_leakage.yaml new file mode 100644 index 0000000..73a0767 --- /dev/null +++ b/sofind/products/beams/beams_v4_leakage.yaml @@ -0,0 +1,24 @@ +system_paths: + perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/beams/leakage_beams/nominal/ + dirac: + niagara: + penn-gpc: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/leakage_beams/nominal/ + +allowed_qids_configs: +- act_dr6vX_qids.yaml + +allowed_qids: +- pa4a +- pa4b +- pa5a +- pa5b +- pa6a +- pa6b + +allowed_qids_extra_kwargs: + +split_beam_file_template: '' +coadd_beam_file_template: '{array}_{freq}_gamma_t2{leakage_comp}.txt' + +norm: False \ No newline at end of file diff --git a/sofind/products/beams/beams_v4_nominal.yaml b/sofind/products/beams/beams_v4_nominal.yaml new file mode 100644 index 0000000..254b8d4 --- /dev/null +++ b/sofind/products/beams/beams_v4_nominal.yaml @@ -0,0 +1,24 @@ +system_paths: + perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/beams/main_beams/nominal/ + dirac: + niagara: + penn-gpc: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/nominal/ + +allowed_qids_configs: +- act_dr6vX_qids.yaml + +allowed_qids: +- pa4a +- pa4b +- pa5a +- pa5b +- pa6a +- pa6b + +allowed_qids_extra_kwargs: + +split_beam_file_template: 'set{split_num}_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' +coadd_beam_file_template: 'coadd_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' + +norm: True \ No newline at end of file From e579008b983331110a486b2cf943bcefe04b6c34 Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Sat, 9 Aug 2025 11:13:07 +0100 Subject: [PATCH 18/56] chhanges reflected in datamodel --- sofind/datamodels/act_dr6v4.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index cd77d4e..29ac7b5 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -10,6 +10,7 @@ maps: masks: mnms_masks_config: mnms_masks.yaml lensing_masks_20250513: lensing_masks_20250513.yaml + lensing_masks_20250804: lensing_masks_20250804.yaml lensing_masks_20240919b: lensing_masks_20240919b.yaml catalogs: @@ -36,6 +37,9 @@ beams: beams_v4_20230902 : beams_v4_20230902.yaml beams_v4_day_20240115: beams_v4_day_20240115.yaml beams_nemo: beams_nemo.yaml + beams_nemo_v4: beams_nemo_v4.yaml + beams_nominal: beams_v4_nominal.yaml + beams_leakage: beams_v4_leakage.yaml transfer_func: tf_dr6v4_230523: tf_dr6v4_230523.yaml From cdb4b44dd1e67574455587a1c0e23266519ba90e Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Mon, 25 Aug 2025 17:18:44 -0400 Subject: [PATCH 19/56] updated inpaint catalogs of dr6v4 noise models to 20241002 --- sofind/products/noise_models/act_dr6v4.yaml | 8 ++++---- sofind/products/noise_models/act_dr6v4_day.yaml | 2 +- sofind/products/noise_models/act_dr6v4_el_split.yaml | 8 ++++---- sofind/products/noise_models/act_dr6v4_inout_split.yaml | 8 ++++---- sofind/products/noise_models/act_dr6v4_pwv_split.yaml | 8 ++++---- sofind/products/noise_models/act_dr6v4_t_split.yaml | 8 ++++---- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/sofind/products/noise_models/act_dr6v4.yaml b/sofind/products/noise_models/act_dr6v4.yaml index 9b417e5..7a1e7a2 100644 --- a/sofind/products/noise_models/act_dr6v4.yaml +++ b/sofind/products/noise_models/act_dr6v4.yaml @@ -48,7 +48,7 @@ tile_cmbmask: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -90,7 +90,7 @@ tile_cmbmask_ivfwhm2: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -132,7 +132,7 @@ fdw_cmbmask: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -212,7 +212,7 @@ fdw_cmbmask_ivfwhm2: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: diff --git a/sofind/products/noise_models/act_dr6v4_day.yaml b/sofind/products/noise_models/act_dr6v4_day.yaml index 962b81c..832260f 100644 --- a/sofind/products/noise_models/act_dr6v4_day.yaml +++ b/sofind/products/noise_models/act_dr6v4_day.yaml @@ -47,7 +47,7 @@ tile_cmbmask_daydeep: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: diff --git a/sofind/products/noise_models/act_dr6v4_el_split.yaml b/sofind/products/noise_models/act_dr6v4_el_split.yaml index 97f1775..e1cbe5e 100644 --- a/sofind/products/noise_models/act_dr6v4_el_split.yaml +++ b/sofind/products/noise_models/act_dr6v4_el_split.yaml @@ -41,7 +41,7 @@ tile_cmbmask: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -83,7 +83,7 @@ tile_cmbmask_ivfwhm2: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -125,7 +125,7 @@ fdw_cmbmask: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -205,7 +205,7 @@ fdw_cmbmask_ivfwhm2: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: diff --git a/sofind/products/noise_models/act_dr6v4_inout_split.yaml b/sofind/products/noise_models/act_dr6v4_inout_split.yaml index cc66968..e377f9e 100644 --- a/sofind/products/noise_models/act_dr6v4_inout_split.yaml +++ b/sofind/products/noise_models/act_dr6v4_inout_split.yaml @@ -41,7 +41,7 @@ tile_cmbmask: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -83,7 +83,7 @@ tile_cmbmask_ivfwhm2: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -125,7 +125,7 @@ fdw_cmbmask: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -205,7 +205,7 @@ fdw_cmbmask_ivfwhm2: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: diff --git a/sofind/products/noise_models/act_dr6v4_pwv_split.yaml b/sofind/products/noise_models/act_dr6v4_pwv_split.yaml index a17533a..b24ffce 100644 --- a/sofind/products/noise_models/act_dr6v4_pwv_split.yaml +++ b/sofind/products/noise_models/act_dr6v4_pwv_split.yaml @@ -41,7 +41,7 @@ tile_cmbmask: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -83,7 +83,7 @@ tile_cmbmask_ivfwhm2: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -125,7 +125,7 @@ fdw_cmbmask: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -205,7 +205,7 @@ fdw_cmbmask_ivfwhm2: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: diff --git a/sofind/products/noise_models/act_dr6v4_t_split.yaml b/sofind/products/noise_models/act_dr6v4_t_split.yaml index 2db1e3e..a02b7d2 100644 --- a/sofind/products/noise_models/act_dr6v4_t_split.yaml +++ b/sofind/products/noise_models/act_dr6v4_t_split.yaml @@ -37,7 +37,7 @@ tile_cmbmask: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -79,7 +79,7 @@ tile_cmbmask_ivfwhm2: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -121,7 +121,7 @@ fdw_cmbmask: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -201,7 +201,7 @@ fdw_cmbmask_ivfwhm2: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: From be084dc9ca0f7401392f3d80b8e2e4a3539af190 Mon Sep 17 00:00:00 2001 From: Mathew Madhavacheril Date: Wed, 27 Aug 2025 10:24:43 -0400 Subject: [PATCH 20/56] remove beam dummy (beams should never be non-unity) --- sofind/datamodels/planck_npipe.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/sofind/datamodels/planck_npipe.yaml b/sofind/datamodels/planck_npipe.yaml index 2efdde6..8eb6c7e 100644 --- a/sofind/datamodels/planck_npipe.yaml +++ b/sofind/datamodels/planck_npipe.yaml @@ -7,7 +7,6 @@ maps: beams: beams_nemo: beams_nemo.yaml beams_npipe: beams_planck_npipe.yaml - beams_dummy: beams_dummy.yaml calibrations: dummy: dummy.yaml From d7ff0b2498ed1d62647ef0b557e8c1bcd12c2618 Mon Sep 17 00:00:00 2001 From: qujia7 Date: Tue, 2 Sep 2025 00:04:56 -0400 Subject: [PATCH 21/56] include trillium --- sofind/products/beams/beams_nemo.yaml | 1 + sofind/products/beams/beams_v4_20230902.yaml | 1 + sofind/products/calibrations/act_dr6.02.yaml | 2 +- sofind/products/calibrations/dr6v4_cal_230523.yaml | 2 +- sofind/products/calibrations/dr6v4_cal_240410.yaml | 2 +- .../products/calibrations/dr6v4_poleff_231113.yaml | 2 +- sofind/products/catalogs/inpaint_catalogs.yaml | 2 +- .../cluster_model/dr6_nemo_cluster_model_snr5.yaml | 12 ++++++++++++ sofind/products/maps/act_dr6v4.yaml | 2 +- sofind/products/maps/act_nemov3.yaml | 1 + sofind/products/maps/planck_npipe.yaml | 3 ++- sofind/products/masks/lensing_masks_20250513.yaml | 2 +- sofind/products/noise_models/act_dr6v4.yaml | 1 + sofind/products/transfer_func/tf_dr6v4_240416.yaml | 1 + sofind/systems.py | 3 ++- 15 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 sofind/products/cluster_model/dr6_nemo_cluster_model_snr5.yaml diff --git a/sofind/products/beams/beams_nemo.yaml b/sofind/products/beams/beams_nemo.yaml index dded487..f7e7c72 100644 --- a/sofind/products/beams/beams_nemo.yaml +++ b/sofind/products/beams/beams_nemo.yaml @@ -3,6 +3,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/beams/planck_beams/ niagara: /scratch/r/rbond/jaejoonk/lensing_pipeline_data/planck_beams/ penn-gpc: /data5/act/beams/planck_beams/ + trillium: /project/rrg-rbond-ac/jaejoonk/lensing_pipeline_data/planck_beams/ allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/beams/beams_v4_20230902.yaml b/sofind/products/beams/beams_v4_20230902.yaml index 5a2cb0b..9f2a163 100644 --- a/sofind/products/beams/beams_v4_20230902.yaml +++ b/sofind/products/beams/beams_v4_20230902.yaml @@ -3,6 +3,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/v4_maps/20230902_beams/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/beams_path/20230902/ penn-gpc: /data5/act/beams/20230902_beams/ + trillium: /project/rrg-rbond-ac/jaejoonk/lensing_pipeline_data/beams_path/20230902/ allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/act_dr6.02.yaml b/sofind/products/calibrations/act_dr6.02.yaml index a983447..9c85e2f 100644 --- a/sofind/products/calibrations/act_dr6.02.yaml +++ b/sofind/products/calibrations/act_dr6.02.yaml @@ -3,7 +3,7 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/calibrations perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/simulations/noise/calibrations niagara: /scratch/r/rbond/iabril/ACT_DR6_products/calibration/dr6_v4_published/ - + trillium: /scratch//iabril/ACT_DR6_products/calibration/dr6_v4_published/ allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/dr6v4_cal_230523.yaml b/sofind/products/calibrations/dr6v4_cal_230523.yaml index f388f7f..86addb7 100644 --- a/sofind/products/calibrations/dr6v4_cal_230523.yaml +++ b/sofind/products/calibrations/dr6v4_cal_230523.yaml @@ -3,7 +3,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/dr6_v4_230523/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_230523/ penn-gpc: /data5/act/DR6_products/calibration/dr6_v4_230523/ - + trillium: /project/rrg-rbond-ac/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_230523/ allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/dr6v4_cal_240410.yaml b/sofind/products/calibrations/dr6v4_cal_240410.yaml index 5767c64..768ed87 100644 --- a/sofind/products/calibrations/dr6v4_cal_240410.yaml +++ b/sofind/products/calibrations/dr6v4_cal_240410.yaml @@ -3,7 +3,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/dr6_v4_240410/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_240410/ penn-gpc: /data5/act/DR6_products/calibration/dr6_v4_240410/ - + trillium: /project/rrg-rbond-ac/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_240410/ allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/dr6v4_poleff_231113.yaml b/sofind/products/calibrations/dr6v4_poleff_231113.yaml index f041d65..39b7293 100644 --- a/sofind/products/calibrations/dr6v4_poleff_231113.yaml +++ b/sofind/products/calibrations/dr6v4_poleff_231113.yaml @@ -3,7 +3,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/dr6_v4_231113/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_231113/ penn-gpc: /data5/act/DR6_products/calibration/dr6_v4_231113/ - + trillium: /project/rrg-rbond-ac/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_231113/ allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/catalogs/inpaint_catalogs.yaml b/sofind/products/catalogs/inpaint_catalogs.yaml index 40b0e7d..4bf8c2f 100644 --- a/sofind/products/catalogs/inpaint_catalogs.yaml +++ b/sofind/products/catalogs/inpaint_catalogs.yaml @@ -5,7 +5,7 @@ system_paths: niagara: /home/r/rbond/jiaqu/projects/catalogs/inpaint_catalogs_20241002 penn-gpc: /data5/act/catalogs/inpaint_catalogs_20241002 souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/inpaint_catalogs_20241002/ - + trillium: /project/rrg-rbond-ac/jiaqu/catalogs/inpaint_catalogs_20241002/ allowed_qids_configs: 'all' allowed_qids: 'all' diff --git a/sofind/products/cluster_model/dr6_nemo_cluster_model_snr5.yaml b/sofind/products/cluster_model/dr6_nemo_cluster_model_snr5.yaml new file mode 100644 index 0000000..e7845bd --- /dev/null +++ b/sofind/products/cluster_model/dr6_nemo_cluster_model_snr5.yaml @@ -0,0 +1,12 @@ +system_paths: + perlmutter: + niagara: /home/r/rbond/jiaqu/sims/models/ + dirac: + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +cluster_fn_template: "nemomodel_dr6_all_clustersSNR{SNR}{frequency}{downgrade}.fits" diff --git a/sofind/products/maps/act_dr6v4.yaml b/sofind/products/maps/act_dr6v4.yaml index 26ee581..c361002 100644 --- a/sofind/products/maps/act_dr6v4.yaml +++ b/sofind/products/maps/act_dr6v4.yaml @@ -4,7 +4,7 @@ system_paths: niagara: /gpfs/fs0/project/r/rbond/sigurdkn/actpol/maps/dr6v4_20230316/release dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/v4_maps/release_20230316/ penn-gpc: /data5/act/maps/dr6v4_20230316 - + trillium: /project/rrg-rbond-ac/sigurdkn/actpol/maps/dr6v4_20230316/release allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/maps/act_nemov3.yaml b/sofind/products/maps/act_nemov3.yaml index 8050f5e..77aaa40 100644 --- a/sofind/products/maps/act_nemov3.yaml +++ b/sofind/products/maps/act_nemov3.yaml @@ -2,6 +2,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/models/ niagara: /scratch/r/rbond/jaejoonk/lensing_pipeline_data/models/ penn-gpc: /data5/act/DR6_products/models/ + trillium: /scratch/jaejoonk/lensing_pipeline_data/models/ allowed_qids_configs: - act_nemo.yaml diff --git a/sofind/products/maps/planck_npipe.yaml b/sofind/products/maps/planck_npipe.yaml index ceeb492..563ae8a 100644 --- a/sofind/products/maps/planck_npipe.yaml +++ b/sofind/products/maps/planck_npipe.yaml @@ -1,7 +1,8 @@ system_paths: perlmutter: /global/cfs/projectdirs/act/data/planck/npipe/fejer1_equ/ niagara: /gpfs/fs0/project/r/rbond/sigurdkn/actpol/planck/npipe/fejer1_equ/ - + trillium: /project/rrg-rbond-ac/sigurdkn/actpol/planck/npipe/fejer1_equ/ + allowed_qids_configs: - planck_qids.yaml diff --git a/sofind/products/masks/lensing_masks_20250513.yaml b/sofind/products/masks/lensing_masks_20250513.yaml index 465b379..2f9ed9f 100644 --- a/sofind/products/masks/lensing_masks_20250513.yaml +++ b/sofind/products/masks/lensing_masks_20250513.yaml @@ -4,7 +4,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/masks/ penn-gpc: /data5/act/DR6_products/masks/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/masks/dr6v4_lensing_250513_masks/ - + trillium: /project/rrg-rbond-ac/msyriac/masks/dr6v4_lensing_250513_masks/ allowed_qids_configs: 'all' allowed_qids: 'all' diff --git a/sofind/products/noise_models/act_dr6v4.yaml b/sofind/products/noise_models/act_dr6v4.yaml index 171fe90..557f239 100644 --- a/sofind/products/noise_models/act_dr6v4.yaml +++ b/sofind/products/noise_models/act_dr6v4.yaml @@ -3,6 +3,7 @@ system_paths: perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/simulations/mnms/ niagara: /scratch/r/rbond/jaejoonk/dr6plus_noise_sims/ + trillium: /scratch/jaejoonk/dr6plus_noise_sims/ allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/transfer_func/tf_dr6v4_240416.yaml b/sofind/products/transfer_func/tf_dr6v4_240416.yaml index 8579a76..c26f70f 100644 --- a/sofind/products/transfer_func/tf_dr6v4_240416.yaml +++ b/sofind/products/transfer_func/tf_dr6v4_240416.yaml @@ -3,6 +3,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/transfer_functions/dr6_v4_legacy_240416/ niagara: /scratch/r/rbond/jaejoonk/lensing_pipeline_data/transfer_functions/dr6_v4_legacy_240416/ penn-gpc: /data5/act/DR6_products/transfer_functions/dr6_v4_legacy_240416/ + trillium: /scratch/jaejoonk/lensing_pipeline_data/transfer_functions/dr6_v4_legacy_240416/ allowed_qids_configs: diff --git a/sofind/systems.py b/sofind/systems.py index 436a59c..af1da5f 100644 --- a/sofind/systems.py +++ b/sofind/systems.py @@ -6,5 +6,6 @@ 'dirac', 'niagara', 'penn-gpc', - 'souk' + 'souk', + 'trillium' ] From dca4e51754cee7af1f3a523f17f7905f083042c3 Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Tue, 2 Sep 2025 14:32:27 -0400 Subject: [PATCH 22/56] Trillium paths / entries for products on Niagara (temporarily on Joshua's scratch) --- sofind/products/beams/beams_nemo.yaml | 1 + sofind/products/beams/beams_planck_npipe.yaml | 2 +- sofind/products/beams/beams_v4_20230902.yaml | 3 ++- sofind/products/beams/beams_v4_day_20240115.yaml | 3 ++- sofind/products/calibrations/dr6v4_cal_230523.yaml | 3 ++- sofind/products/calibrations/dr6v4_cal_240410.yaml | 3 ++- sofind/products/calibrations/dr6v4_calday.yaml | 1 + sofind/products/calibrations/dr6v4_calday_tonight_daydeep.yaml | 1 + sofind/products/calibrations/dr6v4_calday_tonight_daywide.yaml | 1 + sofind/products/calibrations/dr6v4_poleff_231113.yaml | 3 ++- sofind/products/catalogs/inpaint_catalogs.yaml | 1 + sofind/products/maps/act_dr6v4.yaml | 1 + sofind/products/maps/act_dr6v4_el_split.yaml | 1 + sofind/products/maps/act_dr6v4_inout_split.yaml | 1 + sofind/products/maps/act_dr6v4_pwv_split.yaml | 1 + sofind/products/maps/act_dr6v4_t_split.yaml | 1 + sofind/products/maps/act_nemov3.yaml | 3 ++- sofind/products/maps/act_nemov4.yaml | 3 ++- sofind/products/maps/planck_npipe.yaml | 1 + sofind/products/maps/planck_npipe_noise_sims.yaml | 1 + sofind/products/masks/lensing_masks_20240919b.yaml | 1 + sofind/products/masks/lensing_masks_20250318.yaml | 3 ++- sofind/products/masks/lensing_masks_20250513.yaml | 1 + sofind/products/masks/lensing_masks_20250804.yaml | 3 ++- sofind/products/masks/mnms_masks.yaml | 3 ++- sofind/products/noise_models/act_dr6v4.yaml | 2 ++ sofind/products/noise_models/act_dr6v4_day.yaml | 2 ++ sofind/products/noise_models/act_dr6v4_el_split.yaml | 2 +- sofind/products/transfer_func/tf_dr6v4_240416.yaml | 1 + sofind/systems.py | 3 ++- 30 files changed, 43 insertions(+), 13 deletions(-) diff --git a/sofind/products/beams/beams_nemo.yaml b/sofind/products/beams/beams_nemo.yaml index f3e05ca..5843edd 100644 --- a/sofind/products/beams/beams_nemo.yaml +++ b/sofind/products/beams/beams_nemo.yaml @@ -2,6 +2,7 @@ system_paths: della: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/beams/planck_beams/ niagara: /scratch/r/rbond/jaejoonk/lensing_pipeline_data/planck_beams/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/planck_beams/ penn-gpc: /data5/act/beams/planck_beams/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/planck_beams/ diff --git a/sofind/products/beams/beams_planck_npipe.yaml b/sofind/products/beams/beams_planck_npipe.yaml index 2ef37e1..9fe21db 100644 --- a/sofind/products/beams/beams_planck_npipe.yaml +++ b/sofind/products/beams/beams_planck_npipe.yaml @@ -1,8 +1,8 @@ system_paths: della: perlmutter: -# niagara: /gpfs/fs0/project/r/rbond/jiaqu/actpol/ niagara: /gpfs/fs0/scratch/r/rbond/jaejoonk/planck_beams_from_nersc/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/planck_beams/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/beams/planck_beams/ penn-gpc: /data6/DR6_plus/planck_beams/ diff --git a/sofind/products/beams/beams_v4_20230902.yaml b/sofind/products/beams/beams_v4_20230902.yaml index 863db3c..262fec4 100644 --- a/sofind/products/beams/beams_v4_20230902.yaml +++ b/sofind/products/beams/beams_v4_20230902.yaml @@ -2,6 +2,7 @@ system_paths: perlmutter: /global/cfs/cdirs/act/data/adriaand/beams/20230902_beams/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/v4_maps/20230902_beams/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/beams_path/20230902/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/beams_path/20230902/ penn-gpc: /data5/act/beams/20230902_beams/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/20230902_beams/ @@ -21,4 +22,4 @@ allowed_qids_extra_kwargs: split_beam_file_template: 'set{split_num}_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' coadd_beam_file_template: 'coadd_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' -norm: True \ No newline at end of file +norm: True diff --git a/sofind/products/beams/beams_v4_day_20240115.yaml b/sofind/products/beams/beams_v4_day_20240115.yaml index 4b58e9a..cdd09eb 100644 --- a/sofind/products/beams/beams_v4_day_20240115.yaml +++ b/sofind/products/beams/beams_v4_day_20240115.yaml @@ -1,5 +1,6 @@ system_paths: niagara: /gpfs/fs0/project/r/rbond/sigurdkn/actpol/daybeam_srcs/20240115/release/ + trillium: /project/rrg-rbond-ac/sigurdkn/actpol/daybeam_srcs/20240115/release/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/v4_maps/20240115_day_beams/ penn-gpc: /data5/act/DR6_products/v4_maps/daytime/20240115_day_beams/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/20240115_day_beams/ @@ -25,4 +26,4 @@ allowed_qids_extra_kwargs: coadd_beam_file_template: 'beam_full_{array}_{freq}_{daynight}_beam.txt' split_beam_file_template: 'beam_full_{array}_{freq}_{daynight}_beam.txt' # no splits exists for day beams -norm: False \ No newline at end of file +norm: False diff --git a/sofind/products/calibrations/dr6v4_cal_230523.yaml b/sofind/products/calibrations/dr6v4_cal_230523.yaml index f388f7f..1d1a72e 100644 --- a/sofind/products/calibrations/dr6v4_cal_230523.yaml +++ b/sofind/products/calibrations/dr6v4_cal_230523.yaml @@ -2,6 +2,7 @@ system_paths: perlmutter: /global/cfs/cdirs/act/data/alaposta/calibrations/dr6_v4_230523/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/dr6_v4_230523/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_230523/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/dr6_v4_230523/ penn-gpc: /data5/act/DR6_products/calibration/dr6_v4_230523/ allowed_qids_configs: @@ -20,4 +21,4 @@ allowed_qids_extra_kwargs: subproduct_kwargs_orders: cals_file_template: calibs_dict_ACTxACT-NPIPExNPIPE.pkl poleffs_file_template: -key_template: dr6_{array}_{freq} \ No newline at end of file +key_template: dr6_{array}_{freq} diff --git a/sofind/products/calibrations/dr6v4_cal_240410.yaml b/sofind/products/calibrations/dr6v4_cal_240410.yaml index 1a54ade..42b2d7d 100644 --- a/sofind/products/calibrations/dr6v4_cal_240410.yaml +++ b/sofind/products/calibrations/dr6v4_cal_240410.yaml @@ -1,6 +1,7 @@ system_paths: perlmutter: /pscratch/sd/j/jaejoonk/nilc_products/calibration/dr6_v4_240410/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/dr6_v4_240410/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/dr6_v4_240410/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_240410/ penn-gpc: /data5/act/DR6_products/calibration/dr6_v4_240410/ @@ -20,4 +21,4 @@ allowed_qids_extra_kwargs: subproduct_kwargs_orders: cals_file_template: calibs_dict_ACTxACT-PR3xPR3.pkl poleffs_file_template: -key_template: dr6_{array}_{freq} \ No newline at end of file +key_template: dr6_{array}_{freq} diff --git a/sofind/products/calibrations/dr6v4_calday.yaml b/sofind/products/calibrations/dr6v4_calday.yaml index 7d471a1..b180adf 100644 --- a/sofind/products/calibrations/dr6v4_calday.yaml +++ b/sofind/products/calibrations/dr6v4_calday.yaml @@ -2,6 +2,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/dr6v4_calday/ della: perlmutter: + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/dr6v4_calday/ niagara: /scratch/r/rbond/iabril/ACT_DR6_products/calibration/dr6v4_calday/ penn-gpc: /data6/DR6_plus/dr6v4_calday/ diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daydeep.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daydeep.yaml index b03f07b..3844f4d 100644 --- a/sofind/products/calibrations/dr6v4_calday_tonight_daydeep.yaml +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daydeep.yaml @@ -2,6 +2,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/cal_day_to_nightdefault/ della: perlmutter: + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/cal_day_to_nightdefault/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/cal_day_to_nightdefault/ penn-gpc: /data6/DR6_plus/calibrations/cal_day_to_nightdefault/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/cal_day_to_nightdefault/ diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daywide.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daywide.yaml index 9fdd5b4..3da6b76 100644 --- a/sofind/products/calibrations/dr6v4_calday_tonight_daywide.yaml +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daywide.yaml @@ -2,6 +2,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/cal_day_to_nightdefault/ della: perlmutter: + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/cal_day_to_nightdefault/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/cal_day_to_nightdefault/ penn-gpc: /data6/DR6_plus/calibrations/cal_day_to_nightdefault/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/cal_day_to_nightdefault/ diff --git a/sofind/products/calibrations/dr6v4_poleff_231113.yaml b/sofind/products/calibrations/dr6v4_poleff_231113.yaml index 8d82b7d..f802e0f 100644 --- a/sofind/products/calibrations/dr6v4_poleff_231113.yaml +++ b/sofind/products/calibrations/dr6v4_poleff_231113.yaml @@ -1,6 +1,7 @@ system_paths: perlmutter: /pscratch/sd/j/jaejoonk/nilc_products/calibration/dr6_v4_231113/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/dr6_v4_231113/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/dr6_v4_231113/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_231113/ penn-gpc: /data5/act/DR6_products/calibration/dr6_v4_231113/ @@ -20,4 +21,4 @@ allowed_qids_extra_kwargs: subproduct_kwargs_orders: cals_file_template: poleffs_file_template: poleff_EE.pkl -key_template: dr6_{array}_{freq} \ No newline at end of file +key_template: dr6_{array}_{freq} diff --git a/sofind/products/catalogs/inpaint_catalogs.yaml b/sofind/products/catalogs/inpaint_catalogs.yaml index 40b0e7d..6a7adeb 100644 --- a/sofind/products/catalogs/inpaint_catalogs.yaml +++ b/sofind/products/catalogs/inpaint_catalogs.yaml @@ -2,6 +2,7 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/catalogs perlmutter: /global/cfs/cdirs/act/data/catalogs/inpaint_catalogs_20220316 dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/catalogues/inpaint_catalogs + trillium: /scratch/jaejoonk/DR6plus_lensing_data/catalogs/inpaint_catalogs_20241002 niagara: /home/r/rbond/jiaqu/projects/catalogs/inpaint_catalogs_20241002 penn-gpc: /data5/act/catalogs/inpaint_catalogs_20241002 souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/inpaint_catalogs_20241002/ diff --git a/sofind/products/maps/act_dr6v4.yaml b/sofind/products/maps/act_dr6v4.yaml index 6d54cda..c4170b1 100644 --- a/sofind/products/maps/act_dr6v4.yaml +++ b/sofind/products/maps/act_dr6v4.yaml @@ -1,6 +1,7 @@ system_paths: della: /scratch/gpfs/ACT/dr6v4/maps/dr6v4_20230316/release perlmutter: /global/cfs/cdirs/act/data/sigurdkn/actpol/maps/dr6v4_20230316/release + trillium: /project/rrg-rbond-ac/sigurdkn/actpol/maps/dr6v4_20230316/release niagara: /gpfs/fs0/project/r/rbond/sigurdkn/actpol/maps/dr6v4_20230316/release dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/v4_maps/release_20230316/ penn-gpc: /data5/act/maps/dr6v4_20230316 diff --git a/sofind/products/maps/act_dr6v4_el_split.yaml b/sofind/products/maps/act_dr6v4_el_split.yaml index 359d1b0..991a3b5 100644 --- a/sofind/products/maps/act_dr6v4_el_split.yaml +++ b/sofind/products/maps/act_dr6v4_el_split.yaml @@ -1,6 +1,7 @@ system_paths: della: /scratch/gpfs/ACT/dr6v4/maps/dr6v4_20230316/release perlmutter: /global/cfs/cdirs/act/data/sigurdkn/actpol/maps/dr6v4_20230316/release + trillium: /project/rrg-rbond-ac/sigurdkn/actpol/maps/dr6v4_20230316/release niagara: /gpfs/fs0/project/r/rbond/sigurdkn/actpol/maps/dr6v4_20230316/release penn-gpc: /data5/act/maps/dr6v4_20230316 diff --git a/sofind/products/maps/act_dr6v4_inout_split.yaml b/sofind/products/maps/act_dr6v4_inout_split.yaml index 1659363..46b6903 100644 --- a/sofind/products/maps/act_dr6v4_inout_split.yaml +++ b/sofind/products/maps/act_dr6v4_inout_split.yaml @@ -1,6 +1,7 @@ system_paths: della: /scratch/gpfs/ACT/dr6v4/maps/dr6v4_20230316/release perlmutter: /global/cfs/cdirs/act/data/sigurdkn/actpol/maps/dr6v4_20230316/release + trillium: /project/rrg-rbond-ac/sigurdkn/actpol/maps/dr6v4_20230316/release niagara: /gpfs/fs0/project/r/rbond/sigurdkn/actpol/maps/dr6v4_20230316/release penn-gpc: /data5/act/maps/dr6v4_20230316 diff --git a/sofind/products/maps/act_dr6v4_pwv_split.yaml b/sofind/products/maps/act_dr6v4_pwv_split.yaml index d373bc9..40fbf7a 100644 --- a/sofind/products/maps/act_dr6v4_pwv_split.yaml +++ b/sofind/products/maps/act_dr6v4_pwv_split.yaml @@ -1,6 +1,7 @@ system_paths: della: /scratch/gpfs/ACT/dr6v4/maps/dr6v4_20230316/release perlmutter: /global/cfs/cdirs/act/data/sigurdkn/actpol/maps/dr6v4_20230316/release + trillium: /project/rrg-rbond-ac/sigurdkn/actpol/maps/dr6v4_20230316/release niagara: /gpfs/fs0/project/r/rbond/sigurdkn/actpol/maps/dr6v4_20230316/release penn-gpc: /data5/act/maps/dr6v4_20230316 diff --git a/sofind/products/maps/act_dr6v4_t_split.yaml b/sofind/products/maps/act_dr6v4_t_split.yaml index 992ca8c..718b226 100644 --- a/sofind/products/maps/act_dr6v4_t_split.yaml +++ b/sofind/products/maps/act_dr6v4_t_split.yaml @@ -1,6 +1,7 @@ system_paths: della: /scratch/gpfs/ACT/dr6v4/maps/dr6v4_20230316/release perlmutter: /global/cfs/cdirs/act/data/sigurdkn/actpol/maps/dr6v4_20230316/release + trillium: /project/rrg-rbond-ac/sigurdkn/actpol/maps/dr6v4_20230316/release niagara: /gpfs/fs0/project/r/rbond/sigurdkn/actpol/maps/dr6v4_20230316/release penn-gpc: /data5/act/maps/dr6v4_20230316 diff --git a/sofind/products/maps/act_nemov3.yaml b/sofind/products/maps/act_nemov3.yaml index 148bd40..75d4207 100644 --- a/sofind/products/maps/act_nemov3.yaml +++ b/sofind/products/maps/act_nemov3.yaml @@ -1,5 +1,6 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/models/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/models/ niagara: /scratch/r/rbond/jaejoonk/lensing_pipeline_data/models/ penn-gpc: /data5/act/DR6_products/models/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/models/ @@ -11,4 +12,4 @@ allowed_qids: 'all' allowed_qids_extra_kwargs: -coadd_map_file_template: 'nemomodel_dr6_all_clustersSNR5{freq}down2_fejer1.fits' \ No newline at end of file +coadd_map_file_template: 'nemomodel_dr6_all_clustersSNR5{freq}down2_fejer1.fits' diff --git a/sofind/products/maps/act_nemov4.yaml b/sofind/products/maps/act_nemov4.yaml index 148bd40..75d4207 100644 --- a/sofind/products/maps/act_nemov4.yaml +++ b/sofind/products/maps/act_nemov4.yaml @@ -1,5 +1,6 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/models/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/models/ niagara: /scratch/r/rbond/jaejoonk/lensing_pipeline_data/models/ penn-gpc: /data5/act/DR6_products/models/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/models/ @@ -11,4 +12,4 @@ allowed_qids: 'all' allowed_qids_extra_kwargs: -coadd_map_file_template: 'nemomodel_dr6_all_clustersSNR5{freq}down2_fejer1.fits' \ No newline at end of file +coadd_map_file_template: 'nemomodel_dr6_all_clustersSNR5{freq}down2_fejer1.fits' diff --git a/sofind/products/maps/planck_npipe.yaml b/sofind/products/maps/planck_npipe.yaml index ba80c97..20cc9dd 100644 --- a/sofind/products/maps/planck_npipe.yaml +++ b/sofind/products/maps/planck_npipe.yaml @@ -1,5 +1,6 @@ system_paths: perlmutter: /global/cfs/projectdirs/act/data/planck/npipe/fejer1_equ/ + trillium: /project/rrg-rbond-ac/sigurdkn/actpol/planck/npipe/fejer1_equ/ niagara: /gpfs/fs0/project/r/rbond/sigurdkn/actpol/planck/npipe/fejer1_equ/ penn-gpc: /data6/DR6_plus/npipe_fejer1_equ/ diff --git a/sofind/products/maps/planck_npipe_noise_sims.yaml b/sofind/products/maps/planck_npipe_noise_sims.yaml index f8b57ec..b5efc96 100644 --- a/sofind/products/maps/planck_npipe_noise_sims.yaml +++ b/sofind/products/maps/planck_npipe_noise_sims.yaml @@ -1,4 +1,5 @@ system_paths: + trillium: /project/rrg-rbond-ac/jiaqu/ niagara: /project/r/rbond/jiaqu/ penn-gpc: /data6/DR6_plus/ diff --git a/sofind/products/masks/lensing_masks_20240919b.yaml b/sofind/products/masks/lensing_masks_20240919b.yaml index 52dd36d..3ace054 100644 --- a/sofind/products/masks/lensing_masks_20240919b.yaml +++ b/sofind/products/masks/lensing_masks_20240919b.yaml @@ -1,5 +1,6 @@ system_paths: perlmutter: + trillium: /project/rrg-rbond-ac/msyriac/masks/dr6v4_lensing_20240919b_masks/baseline/ niagara: /gpfs/fs0/project/r/rbond/msyriac/masks/dr6v4_lensing_20240919b_masks/baseline/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/masks/ penn-gpc: /data5/act/masks/dr6v4_lensing_20240919b_masks/baseline/ diff --git a/sofind/products/masks/lensing_masks_20250318.yaml b/sofind/products/masks/lensing_masks_20250318.yaml index edfc7c1..abd7493 100644 --- a/sofind/products/masks/lensing_masks_20250318.yaml +++ b/sofind/products/masks/lensing_masks_20250318.yaml @@ -1,5 +1,6 @@ system_paths: perlmutter: /global/cfs/projectdirs/act/data/masks/dr6v4_lensing_20250318_masks/baseline/ + trillium: /project/rrg-rbond-ac/msyriac/masks/dr6v4_lensing_20250318_masks/baseline/ niagara: /gpfs/fs0/project/r/rbond/msyriac/masks/dr6v4_lensing_20250318_masks/baseline/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/masks/ penn-gpc: /data5/act/masks/dr6v4_lensing_20250318_masks/baseline/ @@ -11,4 +12,4 @@ allowed_qids: 'all' allowed_qids_extra_kwargs: dr6v4_20250318: - mask_fn: dr6v4_lensing_20251803_{daynight}_enhanced_mask_{skyfrac}.fits \ No newline at end of file + mask_fn: dr6v4_lensing_20251803_{daynight}_enhanced_mask_{skyfrac}.fits diff --git a/sofind/products/masks/lensing_masks_20250513.yaml b/sofind/products/masks/lensing_masks_20250513.yaml index 4929fdc..0477996 100644 --- a/sofind/products/masks/lensing_masks_20250513.yaml +++ b/sofind/products/masks/lensing_masks_20250513.yaml @@ -1,5 +1,6 @@ system_paths: perlmutter: /global/cfs/cdirs/act/data/iabril/masks/ + trillium: /project/rrg-rbond-ac/msyriac/masks/dr6v4_lensing_250513_masks/ niagara: /gpfs/fs0/project/r/rbond/msyriac/masks/dr6v4_lensing_250513_masks/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/masks/ penn-gpc: /data6/DR6_plus/masks/dr6v4_lensing_250513_masks/ diff --git a/sofind/products/masks/lensing_masks_20250804.yaml b/sofind/products/masks/lensing_masks_20250804.yaml index 208e847..8fe4f01 100644 --- a/sofind/products/masks/lensing_masks_20250804.yaml +++ b/sofind/products/masks/lensing_masks_20250804.yaml @@ -1,5 +1,6 @@ system_paths: perlmutter: + trillium: niagara: dirac: penn-gpc: @@ -9,4 +10,4 @@ allowed_qids_configs: 'all' allowed_qids: 'all' -allowed_qids_extra_kwargs: \ No newline at end of file +allowed_qids_extra_kwargs: diff --git a/sofind/products/masks/mnms_masks.yaml b/sofind/products/masks/mnms_masks.yaml index 7a1f04b..67e8675 100644 --- a/sofind/products/masks/mnms_masks.yaml +++ b/sofind/products/masks/mnms_masks.yaml @@ -2,6 +2,7 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/masks perlmutter: /global/cfs/cdirs/act/data/zatkins/data/simonsobs/mnms/masks dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/simulations/mnms/masks/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/mnms/masks/ niagara: /scratch/r/rbond/jaejoonk/mnms/masks/ penn-gpc: /data5/act/mnms/masks/ @@ -9,4 +10,4 @@ allowed_qids_configs: 'all' allowed_qids: 'all' -allowed_qids_extra_kwargs: \ No newline at end of file +allowed_qids_extra_kwargs: diff --git a/sofind/products/noise_models/act_dr6v4.yaml b/sofind/products/noise_models/act_dr6v4.yaml index 7a1e7a2..16ed510 100644 --- a/sofind/products/noise_models/act_dr6v4.yaml +++ b/sofind/products/noise_models/act_dr6v4.yaml @@ -5,6 +5,8 @@ system_paths: # these are enmap versions #perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/simulations/mnms/ + # not saving noise sims to disk anymore on trillium + trillium: niagara: /scratch/r/rbond/jaejoonk/dr6plus_noise_sims/ penn-gpc: /data6/DR6_plus/noise_sims/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/simulations/mnms/ diff --git a/sofind/products/noise_models/act_dr6v4_day.yaml b/sofind/products/noise_models/act_dr6v4_day.yaml index 832260f..e3f1921 100644 --- a/sofind/products/noise_models/act_dr6v4_day.yaml +++ b/sofind/products/noise_models/act_dr6v4_day.yaml @@ -2,6 +2,8 @@ system_paths: della: /scratch/gpfs/ACT/dr6v4/simulations/noise perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/simulations/mnms/ + # not saving noise sims to disk anymore on trillium + trillium: niagara: /scratch/r/rbond/jaejoonk/dr6plus_noise_sims/ penn-gpc: /data6/DR6_plus/noise_sims/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/simulations/mnms/ diff --git a/sofind/products/noise_models/act_dr6v4_el_split.yaml b/sofind/products/noise_models/act_dr6v4_el_split.yaml index e1cbe5e..1b97417 100644 --- a/sofind/products/noise_models/act_dr6v4_el_split.yaml +++ b/sofind/products/noise_models/act_dr6v4_el_split.yaml @@ -274,4 +274,4 @@ fdw_cmbmask_ivfwhm2: w_lmax_j: 5300 w_lmin: 10 model_file_template: '{config_name}_{noise_model_name}_null_{el_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' - sim_file_template: '{config_name}_{noise_model_name}_null_{el_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' \ No newline at end of file + sim_file_template: '{config_name}_{noise_model_name}_null_{el_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' diff --git a/sofind/products/transfer_func/tf_dr6v4_240416.yaml b/sofind/products/transfer_func/tf_dr6v4_240416.yaml index af011d9..784fbf0 100644 --- a/sofind/products/transfer_func/tf_dr6v4_240416.yaml +++ b/sofind/products/transfer_func/tf_dr6v4_240416.yaml @@ -1,6 +1,7 @@ system_paths: perlmutter: /global/cfs/cdirs/act/data/alaposta/transfer_functions/dr6_v4_legacy_240416/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/transfer_functions/dr6_v4_legacy_240416/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/transfer_functions/dr6_v4_legacy_240416/ niagara: /scratch/r/rbond/jaejoonk/lensing_pipeline_data/transfer_functions/dr6_v4_legacy_240416/ penn-gpc: /data5/act/DR6_products/transfer_functions/dr6_v4_legacy_240416/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/transfer_func/dr6_v4_legacy_240416/ diff --git a/sofind/systems.py b/sofind/systems.py index 436a59c..af1da5f 100644 --- a/sofind/systems.py +++ b/sofind/systems.py @@ -6,5 +6,6 @@ 'dirac', 'niagara', 'penn-gpc', - 'souk' + 'souk', + 'trillium' ] From 4953b60fa2490a6acfd2fc80bc1f5c568399bd44 Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Wed, 3 Sep 2025 15:26:59 -0400 Subject: [PATCH 23/56] updated overlapping trillium paths --- sofind/products/calibrations/dr6v4_cal_230523.yaml | 4 ++-- sofind/products/calibrations/dr6v4_cal_240410.yaml | 2 +- sofind/products/calibrations/dr6v4_poleff_231113.yaml | 2 +- sofind/products/catalogs/inpaint_catalogs.yaml | 4 ++-- .../products/cluster_model/dr6_nemo_cluster_model_snr5.yaml | 1 + sofind/products/masks/lensing_masks_20250513.yaml | 2 +- 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sofind/products/calibrations/dr6v4_cal_230523.yaml b/sofind/products/calibrations/dr6v4_cal_230523.yaml index 7b0e266..8142ff7 100644 --- a/sofind/products/calibrations/dr6v4_cal_230523.yaml +++ b/sofind/products/calibrations/dr6v4_cal_230523.yaml @@ -1,10 +1,10 @@ system_paths: perlmutter: /global/cfs/cdirs/act/data/alaposta/calibrations/dr6_v4_230523/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/dr6_v4_230523/ - niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_230523/ trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/dr6_v4_230523/ + niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_230523/ penn-gpc: /data5/act/DR6_products/calibration/dr6_v4_230523/ - trillium: /project/rrg-rbond-ac/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_230523/ + allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/dr6v4_cal_240410.yaml b/sofind/products/calibrations/dr6v4_cal_240410.yaml index 89d3a5b..5e0879d 100644 --- a/sofind/products/calibrations/dr6v4_cal_240410.yaml +++ b/sofind/products/calibrations/dr6v4_cal_240410.yaml @@ -4,7 +4,7 @@ system_paths: trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/dr6_v4_240410/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_240410/ penn-gpc: /data5/act/DR6_products/calibration/dr6_v4_240410/ - trillium: /project/rrg-rbond-ac/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_240410/ + allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/dr6v4_poleff_231113.yaml b/sofind/products/calibrations/dr6v4_poleff_231113.yaml index 616c8c3..eaf680a 100644 --- a/sofind/products/calibrations/dr6v4_poleff_231113.yaml +++ b/sofind/products/calibrations/dr6v4_poleff_231113.yaml @@ -4,7 +4,7 @@ system_paths: trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/dr6_v4_231113/ niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_231113/ penn-gpc: /data5/act/DR6_products/calibration/dr6_v4_231113/ - trillium: /project/rrg-rbond-ac/jaejoonk/lensing_pipeline_data/calibration/dr6_v4_231113/ + allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/catalogs/inpaint_catalogs.yaml b/sofind/products/catalogs/inpaint_catalogs.yaml index 3e5f12c..336b9a4 100644 --- a/sofind/products/catalogs/inpaint_catalogs.yaml +++ b/sofind/products/catalogs/inpaint_catalogs.yaml @@ -2,11 +2,11 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/catalogs perlmutter: /global/cfs/cdirs/act/data/catalogs/inpaint_catalogs_20220316 dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/catalogues/inpaint_catalogs - trillium: /scratch/jaejoonk/DR6plus_lensing_data/catalogs/inpaint_catalogs_20241002 + trillium: /scratch/jaejoonk/DR6plus_lensing_data/catalogs/inpaint_catalogs_20241002/ niagara: /home/r/rbond/jiaqu/projects/catalogs/inpaint_catalogs_20241002 penn-gpc: /data5/act/catalogs/inpaint_catalogs_20241002 souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/inpaint_catalogs_20241002/ - trillium: /project/rrg-rbond-ac/jiaqu/catalogs/inpaint_catalogs_20241002/ + allowed_qids_configs: 'all' allowed_qids: 'all' diff --git a/sofind/products/cluster_model/dr6_nemo_cluster_model_snr5.yaml b/sofind/products/cluster_model/dr6_nemo_cluster_model_snr5.yaml index e7845bd..eb76ab8 100644 --- a/sofind/products/cluster_model/dr6_nemo_cluster_model_snr5.yaml +++ b/sofind/products/cluster_model/dr6_nemo_cluster_model_snr5.yaml @@ -1,6 +1,7 @@ system_paths: perlmutter: niagara: /home/r/rbond/jiaqu/sims/models/ + trillium: dirac: allowed_qids_configs: 'all' diff --git a/sofind/products/masks/lensing_masks_20250513.yaml b/sofind/products/masks/lensing_masks_20250513.yaml index 6b02fa0..48059ff 100644 --- a/sofind/products/masks/lensing_masks_20250513.yaml +++ b/sofind/products/masks/lensing_masks_20250513.yaml @@ -5,7 +5,7 @@ system_paths: dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/masks/ penn-gpc: /data6/DR6_plus/masks/dr6v4_lensing_250513_masks/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/masks/dr6v4_lensing_250513_masks/ - trillium: /project/rrg-rbond-ac/msyriac/masks/dr6v4_lensing_250513_masks/ + allowed_qids_configs: 'all' allowed_qids: 'all' From d808b7265c063dd6603e685b979f2236fc589ad8 Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Thu, 4 Sep 2025 11:38:13 +0100 Subject: [PATCH 24/56] ive tried with so many masks --- sofind/datamodels/act_dr6v4.yaml | 6 ++ .../masks/lensing_masks_20240919b.yaml | 1 + .../masks/lensing_masks_20240919b_obs.yaml | 12 +++ .../masks/lensing_masks_20250513_obs.yaml | 12 +++ .../masks/lensing_masks_20250818.yaml | 12 +++ .../masks/lensing_masks_20250820.yaml | 12 +++ .../masks/lensing_masks_20250820b.yaml | 12 +++ .../masks/lensing_masks_20250826.yaml | 12 +++ .../products/noise_models/act_dr6v4_day.yaml | 88 ++++++++++++++++++- 9 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 sofind/products/masks/lensing_masks_20240919b_obs.yaml create mode 100644 sofind/products/masks/lensing_masks_20250513_obs.yaml create mode 100644 sofind/products/masks/lensing_masks_20250818.yaml create mode 100644 sofind/products/masks/lensing_masks_20250820.yaml create mode 100644 sofind/products/masks/lensing_masks_20250820b.yaml create mode 100644 sofind/products/masks/lensing_masks_20250826.yaml diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index 29ac7b5..bab9102 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -10,8 +10,14 @@ maps: masks: mnms_masks_config: mnms_masks.yaml lensing_masks_20250513: lensing_masks_20250513.yaml + lensing_masks_20250513_obs: lensing_masks_20250513_obs.yaml lensing_masks_20250804: lensing_masks_20250804.yaml + lensing_masks_20250818: lensing_masks_20250818.yaml + lensing_masks_20250820: lensing_masks_20250820.yaml + lensing_masks_20250820b: lensing_masks_20250820b.yaml + lensing_masks_20250826: lensing_masks_20250826.yaml lensing_masks_20240919b: lensing_masks_20240919b.yaml + lensing_masks_20240919b_obs: lensing_masks_20240919b_obs.yaml catalogs: inpaint_catalogs_config: inpaint_catalogs.yaml diff --git a/sofind/products/masks/lensing_masks_20240919b.yaml b/sofind/products/masks/lensing_masks_20240919b.yaml index 52dd36d..b448283 100644 --- a/sofind/products/masks/lensing_masks_20240919b.yaml +++ b/sofind/products/masks/lensing_masks_20240919b.yaml @@ -3,6 +3,7 @@ system_paths: niagara: /gpfs/fs0/project/r/rbond/msyriac/masks/dr6v4_lensing_20240919b_masks/baseline/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/masks/ penn-gpc: /data5/act/masks/dr6v4_lensing_20240919b_masks/baseline/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/masks/dr6v4_lensing_20240919b_masks/ allowed_qids_configs: 'all' diff --git a/sofind/products/masks/lensing_masks_20240919b_obs.yaml b/sofind/products/masks/lensing_masks_20240919b_obs.yaml new file mode 100644 index 0000000..876f1b0 --- /dev/null +++ b/sofind/products/masks/lensing_masks_20240919b_obs.yaml @@ -0,0 +1,12 @@ +system_paths: + perlmutter: + niagara: + dirac: + penn-gpc: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/masks/dr6v4_lensing_20240919b_masks_obs/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: diff --git a/sofind/products/masks/lensing_masks_20250513_obs.yaml b/sofind/products/masks/lensing_masks_20250513_obs.yaml new file mode 100644 index 0000000..fd61611 --- /dev/null +++ b/sofind/products/masks/lensing_masks_20250513_obs.yaml @@ -0,0 +1,12 @@ +system_paths: + perlmutter: + niagara: + dirac: + penn-gpc: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/masks/dr6v4_lensing_250513_masks_obs/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: \ No newline at end of file diff --git a/sofind/products/masks/lensing_masks_20250818.yaml b/sofind/products/masks/lensing_masks_20250818.yaml new file mode 100644 index 0000000..db3d76f --- /dev/null +++ b/sofind/products/masks/lensing_masks_20250818.yaml @@ -0,0 +1,12 @@ +system_paths: + perlmutter: + niagara: + dirac: + penn-gpc: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/masks/dr6v4_lensing_250818_masks/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: \ No newline at end of file diff --git a/sofind/products/masks/lensing_masks_20250820.yaml b/sofind/products/masks/lensing_masks_20250820.yaml new file mode 100644 index 0000000..dab7d42 --- /dev/null +++ b/sofind/products/masks/lensing_masks_20250820.yaml @@ -0,0 +1,12 @@ +system_paths: + perlmutter: + niagara: + dirac: + penn-gpc: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/masks/dr6v4_lensing_250820_masks/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: \ No newline at end of file diff --git a/sofind/products/masks/lensing_masks_20250820b.yaml b/sofind/products/masks/lensing_masks_20250820b.yaml new file mode 100644 index 0000000..898bb75 --- /dev/null +++ b/sofind/products/masks/lensing_masks_20250820b.yaml @@ -0,0 +1,12 @@ +system_paths: + perlmutter: + niagara: + dirac: + penn-gpc: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/masks/dr6v4_lensing_250820b_masks/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: \ No newline at end of file diff --git a/sofind/products/masks/lensing_masks_20250826.yaml b/sofind/products/masks/lensing_masks_20250826.yaml new file mode 100644 index 0000000..fbe3f95 --- /dev/null +++ b/sofind/products/masks/lensing_masks_20250826.yaml @@ -0,0 +1,12 @@ +system_paths: + perlmutter: + niagara: + dirac: + penn-gpc: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/masks/dr6v4_lensing_250826_masks/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: \ No newline at end of file diff --git a/sofind/products/noise_models/act_dr6v4_day.yaml b/sofind/products/noise_models/act_dr6v4_day.yaml index f66b0c1..e793e4e 100644 --- a/sofind/products/noise_models/act_dr6v4_day.yaml +++ b/sofind/products/noise_models/act_dr6v4_day.yaml @@ -63,7 +63,7 @@ tile_cmbmask_daydeep: ivar_fwhms: null ivar_lmaxs: null kfilt_lbounds: null - masks_subproduct: lensing_masks + masks_subproduct: lensing_masks_20240919b mask_est_name: dr6v4_lensing_20240919_daydeep_enhanced_mask_60.fits mask_est_edgecut: 60 mask_est_apodization: 180 @@ -105,7 +105,7 @@ tile_cmbmask_daywide: ivar_fwhms: null ivar_lmaxs: null kfilt_lbounds: null - masks_subproduct: lensing_masks + masks_subproduct: lensing_masks_20240919b mask_est_name: dr6v4_lensing_20240919_daywide_enhanced_mask_60.fits mask_est_edgecut: 60 mask_est_apodization: 180 @@ -120,3 +120,87 @@ tile_cmbmask_daywide: width_deg: 4.0 model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' + +tile_cmbmask_daydeep_ivfwhm2: + noise_model_class: Tiled + data_model_name: act_dr6v4 + subproduct: day + maps_product: maps + maps_subproduct: default + calibrated: false + calibrations_subproduct: null + catalogs_subproduct: inpaint_catalogs + catalog_name: union_catalog_regular_20241002.csv + differenced: true + dtype: f4 + enforce_equal_qid_kwargs: + - patch + - array + - daynight + - num_splits + filter_kwargs: + lim: 0.000001 + lim0: null + post_filt_rel_downgrade: 2 + iso_filt_method: harmonic + ivar_filt_method: basic + ivar_fwhms: 2 + ivar_lmaxs: null + kfilt_lbounds: null + masks_subproduct: lensing_masks_20250513 + mask_est_name: dr6v4_lensing_250513_daydeep_enhanced_mask_70.fits + mask_est_edgecut: 60 + mask_est_apodization: 180 + mask_obs_name: dr6v4_lensing_250513_daydeep_enhanced_mask_70.fits + mask_obs_edgecut: 10 + model_lim: 0.000001 + model_lim0: null + qid_names_template: '{array}_{freq}' + srcfree: true + delta_ell_smooth: 400 + height_deg: 4.0 + width_deg: 4.0 + model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' + sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' + +tile_cmbmask_daywide_ivfwhm2: + noise_model_class: Tiled + data_model_name: act_dr6v4 + subproduct: day + maps_product: maps + maps_subproduct: default + calibrated: false + calibrations_subproduct: null + catalogs_subproduct: inpaint_catalogs + catalog_name: union_catalog_regular_20241002.csv + differenced: true + dtype: f4 + enforce_equal_qid_kwargs: + - patch + - array + - daynight + - num_splits + filter_kwargs: + lim: 0.000001 + lim0: null + post_filt_rel_downgrade: 2 + iso_filt_method: harmonic + ivar_filt_method: basic + ivar_fwhms: 2 + ivar_lmaxs: null + kfilt_lbounds: null + masks_subproduct: lensing_masks_20250513 + mask_est_name: dr6v4_lensing_250513_daywide_enhanced_mask_70.fits + mask_est_edgecut: 60 + mask_est_apodization: 180 + mask_obs_name: dr6v4_lensing_250513_daywide_enhanced_mask_70.fits + mask_obs_edgecut: 10 + model_lim: 0.000001 + model_lim0: null + qid_names_template: '{array}_{freq}' + srcfree: true + delta_ell_smooth: 400 + height_deg: 4.0 + width_deg: 4.0 + model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' + sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' \ No newline at end of file From 8bbafd905a82a8f6509779fe71960791f5f6e701 Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Mon, 8 Sep 2025 21:01:59 -0400 Subject: [PATCH 25/56] bugfix in act_dr6v4 + null test split beam/cals on Trillium --- sofind/datamodels/act_dr6v4.yaml | 6 +++--- sofind/products/beams/beams_dr6.02_el_split.yaml | 3 ++- sofind/products/beams/beams_dr6.02_inout_split.yaml | 3 ++- sofind/products/beams/beams_dr6.02_pwv_split.yaml | 1 + sofind/products/beams/beams_dr6.02_t_split.yaml | 3 ++- sofind/products/calibrations/act_dr6.02_el_split.yaml | 1 + sofind/products/calibrations/act_dr6.02_inout_split.yaml | 1 + sofind/products/calibrations/act_dr6.02_pwv_split.yaml | 1 + sofind/products/calibrations/act_dr6.02_t_split.yaml | 1 + 9 files changed, 14 insertions(+), 6 deletions(-) diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index b50ad57..19432bf 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -34,9 +34,9 @@ calibrations: dr6v4_calday_tonight_daydeep: dr6v4_calday_tonight_daydeep.yaml dr6v4_calday_tonight_daywide: dr6v4_calday_tonight_daywide.yaml act_dr6.02_el_split: act_dr6.02_el_split.yaml - act_dr6.02_inout_split: act_dr6.02_el_split.yaml - act_dr6.02_pwv_split: act_dr6.02_el_split.yaml - act_dr6.02_t_split: act_dr6.02_el_split.yaml + act_dr6.02_inout_split: act_dr6.02_inout_split.yaml + act_dr6.02_pwv_split: act_dr6.02_pwv_split.yaml + act_dr6.02_t_split: act_dr6.02_t_split.yaml beams: beams_v4_20230130 : beams_v4_20230130.yaml diff --git a/sofind/products/beams/beams_dr6.02_el_split.yaml b/sofind/products/beams/beams_dr6.02_el_split.yaml index e293b51..5992799 100644 --- a/sofind/products/beams/beams_dr6.02_el_split.yaml +++ b/sofind/products/beams/beams_dr6.02_el_split.yaml @@ -1,6 +1,7 @@ system_paths: perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/beams/main_beams/elevation_split/ - penn-gpc: /data6/DR6_plus/main_beams/elevation_split/ + penn-gpc: /data6/DR6_plus/main_beams/elevation_split/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/beams_path/main_beams/elevation_split/ allowed_qids_configs: 'all' diff --git a/sofind/products/beams/beams_dr6.02_inout_split.yaml b/sofind/products/beams/beams_dr6.02_inout_split.yaml index e765601..dc8543e 100644 --- a/sofind/products/beams/beams_dr6.02_inout_split.yaml +++ b/sofind/products/beams/beams_dr6.02_inout_split.yaml @@ -1,6 +1,7 @@ system_paths: perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/beams/main_beams/inout_split/ - penn-gpc: /data6/DR6_plus/main_beams/inout_split/ + penn-gpc: /data6/DR6_plus/main_beams/inout_split/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/beams_path/main_beams/inout_split/ allowed_qids_configs: 'all' diff --git a/sofind/products/beams/beams_dr6.02_pwv_split.yaml b/sofind/products/beams/beams_dr6.02_pwv_split.yaml index 5b61eb3..d01693e 100644 --- a/sofind/products/beams/beams_dr6.02_pwv_split.yaml +++ b/sofind/products/beams/beams_dr6.02_pwv_split.yaml @@ -1,6 +1,7 @@ system_paths: perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/beams/main_beams/pwv_split/ penn-gpc: /data6/DR6_plus/main_beams/pwv_split/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/beams_path/main_beams/pwv_split/ allowed_qids_configs: 'all' diff --git a/sofind/products/beams/beams_dr6.02_t_split.yaml b/sofind/products/beams/beams_dr6.02_t_split.yaml index 083f742..3e073e9 100644 --- a/sofind/products/beams/beams_dr6.02_t_split.yaml +++ b/sofind/products/beams/beams_dr6.02_t_split.yaml @@ -1,6 +1,7 @@ system_paths: perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/beams/main_beams/time_split/ - penn-gpc: /data6/DR6_plus/main_beams/time_split/ + penn-gpc: /data6/DR6_plus/main_beams/time_split/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/beams_path/main_beams/time_split/ allowed_qids_configs: 'all' diff --git a/sofind/products/calibrations/act_dr6.02_el_split.yaml b/sofind/products/calibrations/act_dr6.02_el_split.yaml index 2da276c..ab283e1 100644 --- a/sofind/products/calibrations/act_dr6.02_el_split.yaml +++ b/sofind/products/calibrations/act_dr6.02_el_split.yaml @@ -2,6 +2,7 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/calibrations perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/simulations/noise/calibrations penn-gpc: /data6/DR6_plus/calibrations/dr6.02_calibrations + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/dr6.02_calibrations allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/act_dr6.02_inout_split.yaml b/sofind/products/calibrations/act_dr6.02_inout_split.yaml index e32c39e..5eedafa 100644 --- a/sofind/products/calibrations/act_dr6.02_inout_split.yaml +++ b/sofind/products/calibrations/act_dr6.02_inout_split.yaml @@ -2,6 +2,7 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/calibrations perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/simulations/noise/calibrations penn-gpc: /data6/DR6_plus/calibrations/dr6.02_calibrations + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/dr6.02_calibrations allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/act_dr6.02_pwv_split.yaml b/sofind/products/calibrations/act_dr6.02_pwv_split.yaml index d0c7555..55a120c 100644 --- a/sofind/products/calibrations/act_dr6.02_pwv_split.yaml +++ b/sofind/products/calibrations/act_dr6.02_pwv_split.yaml @@ -2,6 +2,7 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/calibrations perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/simulations/noise/calibrations penn-gpc: /data6/DR6_plus/calibrations/dr6.02_calibrations + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/dr6.02_calibrations allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/calibrations/act_dr6.02_t_split.yaml b/sofind/products/calibrations/act_dr6.02_t_split.yaml index bffca2d..01c7281 100644 --- a/sofind/products/calibrations/act_dr6.02_t_split.yaml +++ b/sofind/products/calibrations/act_dr6.02_t_split.yaml @@ -2,6 +2,7 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/calibrations perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/simulations/noise/calibrations penn-gpc: /data6/DR6_plus/calibrations/dr6.02_calibrations + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/dr6.02_calibrations allowed_qids_configs: - act_dr6vX_qids.yaml From e3bb643beadbbfa69a57ed9714efb9f33297239a Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Thu, 25 Sep 2025 14:51:09 -0400 Subject: [PATCH 26/56] normalized beams + updated lmax=10000 tfs --- sofind/datamodels/act_dr6v4.yaml | 1 + .../products/beams/beams_dr6.02_el_split.yaml | 2 +- .../beams/beams_dr6.02_inout_split.yaml | 2 +- .../beams/beams_dr6.02_pwv_split.yaml | 2 +- .../products/beams/beams_dr6.02_t_split.yaml | 2 +- sofind/products/beams/beams_planck_npipe.yaml | 2 +- sofind/products/calibrations/act_dr6.02.yaml | 2 +- .../masks/lensing_masks_20250915.yaml | 12 ++++++++++++ .../tf_dr6v4_240416_lmax10000.yaml | 19 +++++++++++++++++++ 9 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 sofind/products/masks/lensing_masks_20250915.yaml create mode 100644 sofind/products/transfer_func/tf_dr6v4_240416_lmax10000.yaml diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index 19432bf..8f38513 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -14,6 +14,7 @@ masks: lensing_masks_20240919b: lensing_masks_20240919b.yaml lensing_masks_20250318: lensing_masks_20250318.yaml lensing_masks_20250513: lensing_masks_20250513.yaml + lensing_masks_20250915: lensing_masks_20250915.yaml catalogs: inpaint_catalogs_config: inpaint_catalogs.yaml diff --git a/sofind/products/beams/beams_dr6.02_el_split.yaml b/sofind/products/beams/beams_dr6.02_el_split.yaml index 5992799..7a09845 100644 --- a/sofind/products/beams/beams_dr6.02_el_split.yaml +++ b/sofind/products/beams/beams_dr6.02_el_split.yaml @@ -14,4 +14,4 @@ allowed_qids_extra_kwargs: split_beam_file_template: '{el_split}/set{split_num}_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' coadd_beam_file_template: '{el_split}/coadd_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' -norm: False +norm: True diff --git a/sofind/products/beams/beams_dr6.02_inout_split.yaml b/sofind/products/beams/beams_dr6.02_inout_split.yaml index dc8543e..c13ad73 100644 --- a/sofind/products/beams/beams_dr6.02_inout_split.yaml +++ b/sofind/products/beams/beams_dr6.02_inout_split.yaml @@ -14,4 +14,4 @@ allowed_qids_extra_kwargs: split_beam_file_template: '{inout_split}/set{split_num}_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' coadd_beam_file_template: '{inout_split}/coadd_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' -norm: False +norm: True diff --git a/sofind/products/beams/beams_dr6.02_pwv_split.yaml b/sofind/products/beams/beams_dr6.02_pwv_split.yaml index d01693e..a3d7b0b 100644 --- a/sofind/products/beams/beams_dr6.02_pwv_split.yaml +++ b/sofind/products/beams/beams_dr6.02_pwv_split.yaml @@ -14,4 +14,4 @@ allowed_qids_extra_kwargs: split_beam_file_template: '{pwv_split}/set{split_num}_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' coadd_beam_file_template: '{pwv_split}/coadd_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' -norm: False +norm: True diff --git a/sofind/products/beams/beams_dr6.02_t_split.yaml b/sofind/products/beams/beams_dr6.02_t_split.yaml index 3e073e9..efa53d9 100644 --- a/sofind/products/beams/beams_dr6.02_t_split.yaml +++ b/sofind/products/beams/beams_dr6.02_t_split.yaml @@ -14,4 +14,4 @@ allowed_qids_extra_kwargs: split_beam_file_template: '{t_split}/set{split_num}_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' coadd_beam_file_template: '{t_split}/coadd_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' -norm: False +norm: True diff --git a/sofind/products/beams/beams_planck_npipe.yaml b/sofind/products/beams/beams_planck_npipe.yaml index 9fe21db..e3de51a 100644 --- a/sofind/products/beams/beams_planck_npipe.yaml +++ b/sofind/products/beams/beams_planck_npipe.yaml @@ -2,7 +2,7 @@ system_paths: della: perlmutter: niagara: /gpfs/fs0/scratch/r/rbond/jaejoonk/planck_beams_from_nersc/ - trillium: /scratch/jaejoonk/DR6plus_lensing_data/planck_beams/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/planck_beams/planck_beams_from_nersc/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/beams/planck_beams/ penn-gpc: /data6/DR6_plus/planck_beams/ diff --git a/sofind/products/calibrations/act_dr6.02.yaml b/sofind/products/calibrations/act_dr6.02.yaml index 06f4960..e206395 100644 --- a/sofind/products/calibrations/act_dr6.02.yaml +++ b/sofind/products/calibrations/act_dr6.02.yaml @@ -3,7 +3,7 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/calibrations perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/simulations/noise/calibrations niagara: /scratch/r/rbond/iabril/ACT_DR6_products/calibration/dr6_v4_published/ - trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/dr6_v4_published/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/dr6.02_calibrations/ penn-gpc: /data5/act/DR6_products/calibration/dr6_v4_published/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/act_dr6.02/ diff --git a/sofind/products/masks/lensing_masks_20250915.yaml b/sofind/products/masks/lensing_masks_20250915.yaml new file mode 100644 index 0000000..105d1d7 --- /dev/null +++ b/sofind/products/masks/lensing_masks_20250915.yaml @@ -0,0 +1,12 @@ +system_paths: + perlmutter: + trillium: /scratch/jaejoonk/DR6plus_lensing_data/masks/ + niagara: + dirac: + penn-gpc: + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: diff --git a/sofind/products/transfer_func/tf_dr6v4_240416_lmax10000.yaml b/sofind/products/transfer_func/tf_dr6v4_240416_lmax10000.yaml new file mode 100644 index 0000000..defb944 --- /dev/null +++ b/sofind/products/transfer_func/tf_dr6v4_240416_lmax10000.yaml @@ -0,0 +1,19 @@ +system_paths: + perlmutter: /global/cfs/cdirs/act/data/alaposta/transfer_functions/dr6_v4_legacy_240416/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/transfer_functions/dr6_v4_legacy_240416/ + penn-gpc: /data5/act/DR6_products/transfer_functions/dr6_v4_legacy_240416/ + +allowed_qids_configs: +- act_dr6vX_qids.yaml + +allowed_qids: +- pa4a +- pa4b +- pa5a +- pa5b +- pa6a +- pa6b + +allowed_qids_extra_kwargs: + +tf_template: 'tf_fit_dr6_{array}_{freq}_logistic_lmax10000.dat' From b0d3866648ed2165b8e26a3be1b4936d0578f45a Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Thu, 23 Oct 2025 22:18:52 -0400 Subject: [PATCH 27/56] nemo v5 beams + some daytime mask changes --- sofind/datamodels/act_dr6v4.yaml | 5 +++++ sofind/datamodels/act_nemo.yaml | 3 ++- sofind/datamodels/planck_npipe.yaml | 9 +++++---- sofind/products/beams/beams_v4_leakage.yaml | 1 + .../products/noise_models/act_dr6v4_day.yaml | 20 +++++++++++++------ 5 files changed, 27 insertions(+), 11 deletions(-) diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index 8f38513..7accbf0 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -14,6 +14,7 @@ masks: lensing_masks_20240919b: lensing_masks_20240919b.yaml lensing_masks_20250318: lensing_masks_20250318.yaml lensing_masks_20250513: lensing_masks_20250513.yaml + lensing_masks_20250826: lensing_masks_20250826.yaml lensing_masks_20250915: lensing_masks_20250915.yaml catalogs: @@ -33,7 +34,9 @@ calibrations: dr6v4_cal_240410: dr6v4_cal_240410.yaml dr6v4_poleff_231113: dr6v4_poleff_231113.yaml dr6v4_calday_tonight_daydeep: dr6v4_calday_tonight_daydeep.yaml + dr6v4_calday_tonight_daydeep_250826: dr6v4_calday_tonight_daydeep_250826.yaml dr6v4_calday_tonight_daywide: dr6v4_calday_tonight_daywide.yaml + dr6v4_calday_tonight_daywide_250826: dr6v4_calday_tonight_daywide_250826.yaml act_dr6.02_el_split: act_dr6.02_el_split.yaml act_dr6.02_inout_split: act_dr6.02_inout_split.yaml act_dr6.02_pwv_split: act_dr6.02_pwv_split.yaml @@ -49,9 +52,11 @@ beams: beams_dr6.02_inout_split: beams_dr6.02_inout_split.yaml beams_dr6.02_t_split: beams_dr6.02_t_split.yaml beams_nemo_v4: beams_nemo_v4.yaml + beams_nemo_v5: beams_nemo_v5.yaml beams_nominal: beams_v4_nominal.yaml beams_leakage: beams_v4_leakage.yaml transfer_func: tf_dr6v4_230523: tf_dr6v4_230523.yaml tf_dr6v4_240416: tf_dr6v4_240416.yaml + tf_dr6v4_240416_lmax10000: tf_dr6v4_240416_lmax10000.yaml diff --git a/sofind/datamodels/act_nemo.yaml b/sofind/datamodels/act_nemo.yaml index de5e0e2..22d17ef 100644 --- a/sofind/datamodels/act_nemo.yaml +++ b/sofind/datamodels/act_nemo.yaml @@ -2,4 +2,5 @@ qids_config: act_nemo.yaml maps: nemo_v3: act_nemov3.yaml - nemo_v4: act_nemov4.yaml \ No newline at end of file + nemo_v4: act_nemov4.yaml + nemo_v5: act_nemov5.yaml \ No newline at end of file diff --git a/sofind/datamodels/planck_npipe.yaml b/sofind/datamodels/planck_npipe.yaml index 8eb6c7e..fcc944b 100644 --- a/sofind/datamodels/planck_npipe.yaml +++ b/sofind/datamodels/planck_npipe.yaml @@ -8,12 +8,13 @@ beams: beams_nemo: beams_nemo.yaml beams_npipe: beams_planck_npipe.yaml -calibrations: - dummy: dummy.yaml +#calibrations: +# dummy: dummy.yaml -transfer_func: - tf_dummy: tf_dummy.yaml +#transfer_func: +# tf_dummy: tf_dummy.yaml masks: lensing_masks_20250513: lensing_masks_20250513.yaml lensing_masks_20240919b: lensing_masks_20240919b.yaml + lensing_masks_20250826: lensing_masks_20250826.yaml \ No newline at end of file diff --git a/sofind/products/beams/beams_v4_leakage.yaml b/sofind/products/beams/beams_v4_leakage.yaml index 73a0767..28ef24a 100644 --- a/sofind/products/beams/beams_v4_leakage.yaml +++ b/sofind/products/beams/beams_v4_leakage.yaml @@ -2,6 +2,7 @@ system_paths: perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/beams/leakage_beams/nominal/ dirac: niagara: + trillium: /scratch/iabril/dr6.02_leakage/nominal/ penn-gpc: souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/leakage_beams/nominal/ diff --git a/sofind/products/noise_models/act_dr6v4_day.yaml b/sofind/products/noise_models/act_dr6v4_day.yaml index e3f1921..9d32ec4 100644 --- a/sofind/products/noise_models/act_dr6v4_day.yaml +++ b/sofind/products/noise_models/act_dr6v4_day.yaml @@ -66,11 +66,15 @@ tile_cmbmask_daydeep: ivar_fwhms: null ivar_lmaxs: null kfilt_lbounds: null - masks_subproduct: lensing_masks - mask_est_name: dr6v4_lensing_20240919_daydeep_enhanced_mask_60.fits + # subject to change! + # masks_subproduct: lensing_masks + # mask_est_name: dr6v4_lensing_20240919_daywide_enhanced_mask_60.fits + masks_subproduct: lensing_masks_20250915 + mask_est_name: dr6v4_lensing_250826_daydeep_enhanced_mask_wtop_rightcutS_shrink1.5_70.fits mask_est_edgecut: 60 mask_est_apodization: 180 - mask_obs_name: dr6v4_lensing_20240919_daydeep_enhanced_mask_60.fits + # mask_obs_name: dr6v4_lensing_20240919_daywide_enhanced_mask_80.fits + mask_obs_name: dr6v4_lensing_250826_daydeep_enhanced_mask_wtop_rightcutS_shrink1.5_70.fits mask_obs_edgecut: 10 model_lim: 0.000001 model_lim0: null @@ -108,11 +112,15 @@ tile_cmbmask_daywide: ivar_fwhms: null ivar_lmaxs: null kfilt_lbounds: null - masks_subproduct: lensing_masks - mask_est_name: dr6v4_lensing_20240919_daywide_enhanced_mask_60.fits + # subject to change! + # masks_subproduct: lensing_masks + # mask_est_name: dr6v4_lensing_20240919_daywide_enhanced_mask_60.fits + masks_subproduct: lensing_masks_20250915 + mask_est_name: dr6v4_lensing_250826_daywide_enhanced_mask_70_South.fits mask_est_edgecut: 60 mask_est_apodization: 180 - mask_obs_name: dr6v4_lensing_20240919_daywide_enhanced_mask_80.fits + # mask_obs_name: dr6v4_lensing_20240919_daywide_enhanced_mask_80.fits + mask_obs_name: dr6v4_lensing_250826_daywide_enhanced_mask_70_South.fits mask_obs_edgecut: 10 model_lim: 0.000001 model_lim0: null From 0e42229d81e6c626d7082d831e5dcc574f5033a4 Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Thu, 23 Oct 2025 22:33:32 -0400 Subject: [PATCH 28/56] nemov5 maps --- sofind/products/maps/act_nemov5.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 sofind/products/maps/act_nemov5.yaml diff --git a/sofind/products/maps/act_nemov5.yaml b/sofind/products/maps/act_nemov5.yaml new file mode 100644 index 0000000..983dc23 --- /dev/null +++ b/sofind/products/maps/act_nemov5.yaml @@ -0,0 +1,12 @@ +system_paths: + trillium: /scratch/jaejoonk/cmodel/nemo/cand_release/cmodel_20251015 + penn-gpc: /data6/depot/cmodel/nemo/cand_release/cmodel_20251015 + +allowed_qids_configs: + - act_nemo.yaml + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +coadd_map_file_template: 'processed_model_f{freq}.fits' From 51f13bae44e37ae49e232cd6acf507403ca9df1d Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Thu, 23 Oct 2025 22:44:27 -0400 Subject: [PATCH 29/56] nemo v5 beams --- sofind/products/beams/beams_nemo_v5.yaml | 68 ++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 sofind/products/beams/beams_nemo_v5.yaml diff --git a/sofind/products/beams/beams_nemo_v5.yaml b/sofind/products/beams/beams_nemo_v5.yaml new file mode 100644 index 0000000..5861696 --- /dev/null +++ b/sofind/products/beams/beams_nemo_v5.yaml @@ -0,0 +1,68 @@ +system_paths: + trillium: /scratch/jaejoonk/cmodel/nemo/cand_release/cmodel_20251015/ + penn-gpc: /scratch/cmodel/nemo/cand_release/cmodel_20251015/ + +allowed_qids_configs: + - act_dr6vX_qids.yaml + - planck_qids.yaml + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + p01: + nemo_freq: 'f090' + p02: + nemo_freq: 'f090' + p03: + nemo_freq: 'f090' + p04: + nemo_freq: 'f090' + p05: + nemo_freq: 'f150' + p06: + nemo_freq: 'f150' + p07: + nemo_freq: 'f150' + p08: + nemo_freq: 'f150' + p09: + nemo_freq: 'f150' + pa4a: + nemo_freq: 'f150' + pa4a_dd: + nemo_freq: 'f150' + pa4a_dw: + nemo_freq: 'f150' + pa4b: + nemo_freq: 'f220' + pa4b_dd: + nemo_freq: 'f220' + pa4b_dw: + nemo_freq: 'f220' + pa5a: + nemo_freq: 'f090' + pa5a_dd: + nemo_freq: 'f090' + pa5a_dw: + nemo_freq: 'f090' + pa5b: + nemo_freq: 'f150' + pa5b_dd: + nemo_freq: 'f150' + pa5b_dw: + nemo_freq: 'f150' + pa6a: + nemo_freq: 'f090' + pa6a_dd: + nemo_freq: 'f090' + pa6a_dw: + nemo_freq: 'f090' + pa6b: + nemo_freq: 'f150' + pa6b_dd: + nemo_freq: 'f150' + pa6b_dw: + nemo_freq: 'f150' + +coadd_beam_file_template: 'beam_dr6_daynight_{nemo_freq}_tfun.txt' +norm: True From 9e31098c75ba7d3c1ace0c17a0500cb3b688f357 Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Thu, 13 Nov 2025 19:24:39 -0500 Subject: [PATCH 30/56] planck beam updates + adding Frank pa5a poleff to act_dr6.02 --- sofind/datamodels/act_dr6v4.yaml | 1 + sofind/datamodels/planck_npipe.yaml | 2 ++ sofind/products/beams/beams_planck_npipe.yaml | 2 +- sofind/products/calibrations/act_dr6.02.yaml | 3 +- .../calibrations/act_dr6.02_hack.yaml | 20 +++++++++++ .../calibrations/dr6v4_cal_frank.yaml | 34 +++++++++++++++++++ .../dr6v4_calday_tonight_daydeep_250826.yaml | 24 +++++++++++++ .../dr6v4_calday_tonight_daywide_250826.yaml | 22 ++++++++++++ 8 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 sofind/products/calibrations/act_dr6.02_hack.yaml create mode 100644 sofind/products/calibrations/dr6v4_cal_frank.yaml create mode 100644 sofind/products/calibrations/dr6v4_calday_tonight_daydeep_250826.yaml create mode 100644 sofind/products/calibrations/dr6v4_calday_tonight_daywide_250826.yaml diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index 7accbf0..ac0cde0 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -41,6 +41,7 @@ calibrations: act_dr6.02_inout_split: act_dr6.02_inout_split.yaml act_dr6.02_pwv_split: act_dr6.02_pwv_split.yaml act_dr6.02_t_split: act_dr6.02_t_split.yaml + act_dr6.02_hack: act_dr6.02_hack.yaml beams: beams_v4_20230130 : beams_v4_20230130.yaml diff --git a/sofind/datamodels/planck_npipe.yaml b/sofind/datamodels/planck_npipe.yaml index fcc944b..1d7532a 100644 --- a/sofind/datamodels/planck_npipe.yaml +++ b/sofind/datamodels/planck_npipe.yaml @@ -6,6 +6,8 @@ maps: beams: beams_nemo: beams_nemo.yaml + beams_nemo_v4: beams_nemo_v4.yaml + beams_nemo_v5: beams_nemo_v5.yaml beams_npipe: beams_planck_npipe.yaml #calibrations: diff --git a/sofind/products/beams/beams_planck_npipe.yaml b/sofind/products/beams/beams_planck_npipe.yaml index e3de51a..b7a12da 100644 --- a/sofind/products/beams/beams_planck_npipe.yaml +++ b/sofind/products/beams/beams_planck_npipe.yaml @@ -16,4 +16,4 @@ allowed_qids_extra_kwargs: split_beam_file_template: 'npipe_DR6_{split_num}x{split_num}/bl_T_npipe_DR6_{split_num}x{split_num}_{freq}{split_num}x{freq}{split_num}.dat' coadd_beam_file_template: 'npipe_DR6_coadd/bl_T_npipe_DR6_coadd_{freq}GHzx{freq}GHz.dat' -norm: False +norm: True diff --git a/sofind/products/calibrations/act_dr6.02.yaml b/sofind/products/calibrations/act_dr6.02.yaml index e206395..eeb37ce 100644 --- a/sofind/products/calibrations/act_dr6.02.yaml +++ b/sofind/products/calibrations/act_dr6.02.yaml @@ -22,5 +22,6 @@ allowed_qids_extra_kwargs: subproduct_kwargs_orders: cals_file_template: act_dr6.02_cals.pkl -poleffs_file_template: act_dr6.02_poleffs.pkl +#poleffs_file_template: act_dr6.02_poleffs.pkl +poleffs_file_template: act_dr6.02_poleffs_pa5a_hack.pkl key_template: dr6_{array}_{freq} diff --git a/sofind/products/calibrations/act_dr6.02_hack.yaml b/sofind/products/calibrations/act_dr6.02_hack.yaml new file mode 100644 index 0000000..5d266a6 --- /dev/null +++ b/sofind/products/calibrations/act_dr6.02_hack.yaml @@ -0,0 +1,20 @@ +system_paths: + # dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/dr6_v4_published/ + # della: /scratch/gpfs/zatkins/data/simonsobs/mnms/calibrations + # perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/simulations/noise/calibrations + # niagara: /scratch/r/rbond/iabril/ACT_DR6_products/calibration/dr6_v4_published/ + trillium: /scratch/jiaqu/calibration/dr6v4_hack/ +allowed_qids_configs: +- act_dr6vX_qids.yaml +allowed_qids: +- pa4a +- pa4b +- pa5a +- pa5b +- pa6a +- pa6b +allowed_qids_extra_kwargs: +subproduct_kwargs_orders: +cals_file_template: +poleffs_file_template: hack_poleff_pa5amf.pkl +key_template: dr6_{array}_{freq} diff --git a/sofind/products/calibrations/dr6v4_cal_frank.yaml b/sofind/products/calibrations/dr6v4_cal_frank.yaml new file mode 100644 index 0000000..2be0ed2 --- /dev/null +++ b/sofind/products/calibrations/dr6v4_cal_frank.yaml @@ -0,0 +1,34 @@ +system_paths: + perlmutter: + della: + dirac: + niagara: + +allowed_qids_configs: +- act_dr6vX_qids.yaml + +allowed_qids: all + +allowed_qids_extra_kwargs: + +# gain factors + +pa4b: + cal: 1.0274146516062883 + poleff: 1.1119 + +pa5a: + cal: 1.009463740391127 + poleff: 0.9625 + +pa5b: + cal: 0.9850773599437168 + poleff: 0.9961 + +pa6a: + cal: 1.0140213622811152 + poleff: 0.9660 + +pa6b: + cal: 0.9686333859482543 + poleff: 0.9764 diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daydeep_250826.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daydeep_250826.yaml new file mode 100644 index 0000000..25ac393 --- /dev/null +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daydeep_250826.yaml @@ -0,0 +1,24 @@ +system_paths: + dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/cal_day_to_nightdefault/ + della: + perlmutter: + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/cal_day_to_nightdefault/ + niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/cal_day_to_nightdefault/ + penn-gpc: /data6/DR6_plus/calibrations/cal_day_to_nightdefault/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/cal_day_to_nightdefault/ + +allowed_qids_configs: +- act_dr6vX_qids.yaml + +allowed_qids: +- pa5a_dd +- pa5b_dd +- pa6a_dd +- pa6b_dd + +allowed_qids_extra_kwargs: + +subproduct_kwargs_orders: +cals_file_template: cal_day_to_nightdefault_daydeep_250826.pkl +poleffs_file_template: +key_template: dr6_{array}_{freq} diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daywide_250826.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daywide_250826.yaml new file mode 100644 index 0000000..fa9409e --- /dev/null +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daywide_250826.yaml @@ -0,0 +1,22 @@ +system_paths: + dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/cal_day_to_nightdefault/ + della: + perlmutter: + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/cal_day_to_nightdefault/ + niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/cal_day_to_nightdefault/ + penn-gpc: /data6/DR6_plus/calibrations/cal_day_to_nightdefault/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/cal_day_to_nightdefault/ + +allowed_qids_configs: +- act_dr6vX_qids.yaml + +allowed_qids: +- pa5a_dw +- pa5b_dw + +allowed_qids_extra_kwargs: + +subproduct_kwargs_orders: +cals_file_template: cal_day_to_nightdefault_daywide_250826.pkl +poleffs_file_template: +key_template: dr6_{array}_{freq} From 6ebefcfc92605e4a865a184c518e154865fd2d6c Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Tue, 2 Dec 2025 10:57:18 -0500 Subject: [PATCH 31/56] added lensing_masks_20250826 --- sofind/products/masks/lensing_masks_20250826.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 sofind/products/masks/lensing_masks_20250826.yaml diff --git a/sofind/products/masks/lensing_masks_20250826.yaml b/sofind/products/masks/lensing_masks_20250826.yaml new file mode 100644 index 0000000..2c0be30 --- /dev/null +++ b/sofind/products/masks/lensing_masks_20250826.yaml @@ -0,0 +1,11 @@ +system_paths: + perlmutter: /global/cfs/cdirs/act/data/iabril/masks/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/masks_250826/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + + From 2ab1a9cc0625b6febf87587fd1d625966160d7cf Mon Sep 17 00:00:00 2001 From: iabrilcabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Wed, 17 Dec 2025 19:19:36 +0100 Subject: [PATCH 32/56] noise models reflect now final masks --- .../products/noise_models/act_dr6v4_day.yaml | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/sofind/products/noise_models/act_dr6v4_day.yaml b/sofind/products/noise_models/act_dr6v4_day.yaml index 9d32ec4..641c20a 100644 --- a/sofind/products/noise_models/act_dr6v4_day.yaml +++ b/sofind/products/noise_models/act_dr6v4_day.yaml @@ -3,7 +3,7 @@ system_paths: perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/simulations/mnms/ # not saving noise sims to disk anymore on trillium - trillium: + trillium: /scratch/iabril/mnms/models/ niagara: /scratch/r/rbond/jaejoonk/dr6plus_noise_sims/ penn-gpc: /data6/DR6_plus/noise_sims/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/simulations/mnms/ @@ -39,8 +39,7 @@ allowed_qids_extra_kwargs: pa5b_dw: num_splits: 4 - -tile_cmbmask_daydeep: +tile_cmbmask_daydeep_250513: noise_model_class: Tiled data_model_name: act_dr6v4 subproduct: day @@ -66,15 +65,11 @@ tile_cmbmask_daydeep: ivar_fwhms: null ivar_lmaxs: null kfilt_lbounds: null - # subject to change! - # masks_subproduct: lensing_masks - # mask_est_name: dr6v4_lensing_20240919_daywide_enhanced_mask_60.fits - masks_subproduct: lensing_masks_20250915 - mask_est_name: dr6v4_lensing_250826_daydeep_enhanced_mask_wtop_rightcutS_shrink1.5_70.fits + masks_subproduct: lensing_masks_20250513 + mask_est_name: dr6v4_lensing_250513_daydeep_enhanced_mask_70.fits mask_est_edgecut: 60 mask_est_apodization: 180 - # mask_obs_name: dr6v4_lensing_20240919_daywide_enhanced_mask_80.fits - mask_obs_name: dr6v4_lensing_250826_daydeep_enhanced_mask_wtop_rightcutS_shrink1.5_70.fits + mask_obs_name: dr6v4_lensing_250513_daydeep_enhanced_mask_70.fits mask_obs_edgecut: 10 model_lim: 0.000001 model_lim0: null @@ -86,7 +81,7 @@ tile_cmbmask_daydeep: model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' -tile_cmbmask_daywide: +tile_cmbmask_daywide_250513: noise_model_class: Tiled data_model_name: act_dr6v4 subproduct: day @@ -95,7 +90,7 @@ tile_cmbmask_daywide: calibrated: false calibrations_subproduct: null catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20220316.csv + catalog_name: union_catalog_regular_20241002.csv differenced: true dtype: f4 enforce_equal_qid_kwargs: @@ -112,15 +107,11 @@ tile_cmbmask_daywide: ivar_fwhms: null ivar_lmaxs: null kfilt_lbounds: null - # subject to change! - # masks_subproduct: lensing_masks - # mask_est_name: dr6v4_lensing_20240919_daywide_enhanced_mask_60.fits - masks_subproduct: lensing_masks_20250915 - mask_est_name: dr6v4_lensing_250826_daywide_enhanced_mask_70_South.fits + masks_subproduct: lensing_masks_20250513 + mask_est_name: dr6v4_lensing_250513_daywide_enhanced_mask_70.fits mask_est_edgecut: 60 mask_est_apodization: 180 - # mask_obs_name: dr6v4_lensing_20240919_daywide_enhanced_mask_80.fits - mask_obs_name: dr6v4_lensing_250826_daywide_enhanced_mask_70_South.fits + mask_obs_name: dr6v4_lensing_250513_daywide_enhanced_mask_70.fits mask_obs_edgecut: 10 model_lim: 0.000001 model_lim0: null From 1ff01d6d791f3bec38935cc1c1c56977602e2b9c Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Wed, 17 Dec 2025 14:04:18 -0500 Subject: [PATCH 33/56] mask subproduct merging --- sofind/products/noise_models/act_dr6v4_day.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sofind/products/noise_models/act_dr6v4_day.yaml b/sofind/products/noise_models/act_dr6v4_day.yaml index 9d32ec4..4d3a07b 100644 --- a/sofind/products/noise_models/act_dr6v4_day.yaml +++ b/sofind/products/noise_models/act_dr6v4_day.yaml @@ -69,7 +69,7 @@ tile_cmbmask_daydeep: # subject to change! # masks_subproduct: lensing_masks # mask_est_name: dr6v4_lensing_20240919_daywide_enhanced_mask_60.fits - masks_subproduct: lensing_masks_20250915 + masks_subproduct: lensing_masks_20250826 mask_est_name: dr6v4_lensing_250826_daydeep_enhanced_mask_wtop_rightcutS_shrink1.5_70.fits mask_est_edgecut: 60 mask_est_apodization: 180 @@ -115,7 +115,7 @@ tile_cmbmask_daywide: # subject to change! # masks_subproduct: lensing_masks # mask_est_name: dr6v4_lensing_20240919_daywide_enhanced_mask_60.fits - masks_subproduct: lensing_masks_20250915 + masks_subproduct: lensing_masks_20250826 mask_est_name: dr6v4_lensing_250826_daywide_enhanced_mask_70_South.fits mask_est_edgecut: 60 mask_est_apodization: 180 From 9e2eec4ada64f0bce4ebe374ebd185b5189570bd Mon Sep 17 00:00:00 2001 From: Joshua Kim <40779225+jaejoonk@users.noreply.github.com> Date: Fri, 6 Feb 2026 18:58:46 -0500 Subject: [PATCH 34/56] Update act_nemov5.yaml --- sofind/products/maps/act_nemov5.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sofind/products/maps/act_nemov5.yaml b/sofind/products/maps/act_nemov5.yaml index 983dc23..73ed3d4 100644 --- a/sofind/products/maps/act_nemov5.yaml +++ b/sofind/products/maps/act_nemov5.yaml @@ -1,6 +1,6 @@ system_paths: trillium: /scratch/jaejoonk/cmodel/nemo/cand_release/cmodel_20251015 - penn-gpc: /data6/depot/cmodel/nemo/cand_release/cmodel_20251015 + penn-gpc: /data6/DR6_plus/depot/cmodel/nemo/cand_release/cmodel_20251015 allowed_qids_configs: - act_nemo.yaml From 62c32485fc64883aa7563b42af22383700f90ccb Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Sat, 7 Feb 2026 18:33:42 -0500 Subject: [PATCH 35/56] gpc path for 250826 masks --- sofind/products/masks/lensing_masks_20250826.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sofind/products/masks/lensing_masks_20250826.yaml b/sofind/products/masks/lensing_masks_20250826.yaml index 2c0be30..c4ab267 100644 --- a/sofind/products/masks/lensing_masks_20250826.yaml +++ b/sofind/products/masks/lensing_masks_20250826.yaml @@ -1,7 +1,8 @@ system_paths: perlmutter: /global/cfs/cdirs/act/data/iabril/masks/ trillium: /scratch/jaejoonk/DR6plus_lensing_data/masks_250826/ - + penn-gpc: /data6/DR6_plus/masks/masks_250826/ + allowed_qids_configs: 'all' allowed_qids: 'all' From fa31b74ab50cb9557092927a29125a39ea24e25b Mon Sep 17 00:00:00 2001 From: Joshua Kim Date: Sat, 7 Feb 2026 18:35:58 -0500 Subject: [PATCH 36/56] get_super_qids_from_qids... function to be compatible with latest mnms versions --- sofind/utils.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/sofind/utils.py b/sofind/utils.py index 442dec3..fc28743 100644 --- a/sofind/utils.py +++ b/sofind/utils.py @@ -5,6 +5,7 @@ import io import os, sys +from itertools import product # adapted from soapack.interfaces def config_from_yaml_file(filename): @@ -202,4 +203,53 @@ def get_protected_fn(*fns, no_fn_collisions=True, write_to_fn_idx=None): # This creates a mapping between Product subclasses and their product tag def get_producttag(product): """Return product.split('.')[-1]""" - return product.split('.')[-1] \ No newline at end of file + return product.split('.')[-1] + +def get_super_qids_from_qids_and_subproduct_kwargs(*qids, **subproduct_kwargs): + """Generate 'super_qids' from qids and a dictionary that maps keys of + subproduct_kwargs to an iterable of their possible values. 'super_qids' + are the outer product of the possible values for each key, and then the + qids (starting with the subproduct_kwargs from left to right in insertion + order). + + For example, if qids is ('pa5a', 'pa5b'), and subproduct_kwargs is + {'el_split': ['el1', 'el2', 'el3'], 'something_else': [0, 1]}, then + the following 12 tuples form the ordered 'super_qids': + + ({'el_split': 'el1', 'something_else': 0}, 'pa5a'), + ({'el_split': 'el1', 'something_else': 0}, 'pa5b'), + ({'el_split': 'el1', 'something_else': 1}, 'pa5a'), + ({'el_split': 'el1', 'something_else': 1}, 'pa5b'), + ({'el_split': 'el2', 'something_else': 0}, 'pa5a'), + ({'el_split': 'el2', 'something_else': 0}, 'pa5b'), + ({'el_split': 'el2', 'something_else': 1}, 'pa5a'), + ({'el_split': 'el2', 'something_else': 1}, 'pa5b'), + ({'el_split': 'el3', 'something_else': 0}, 'pa5a'), + ({'el_split': 'el3', 'something_else': 0}, 'pa5b'), + ({'el_split': 'el3', 'something_else': 1}, 'pa5a'), + ({'el_split': 'el3', 'something_else': 1}, 'pa5b') + + Parameters + ---------- + qids : str or iterable of str + One or more array qids for this model. + subproduct_kwargs : (key, iterable) mapping, optional + Any additional keyword arguments required to load products from + disk, or to format model and sim filenames. The iterable for each + key should list, in order, all possible values the key can take. + + Returns + ------- + list + List of the super_qids. + """ + super_qids = [] + for vals in product(*subproduct_kwargs.values(), qids): + _subproduct_kwargs = dict(zip(subproduct_kwargs.keys(), vals[:-1])) + qid = vals[-1] + + # e.g. ({'inout_split': 'inout1', 'el_split': 1}, 'pa5a') + super_qid = (_subproduct_kwargs, qid) + super_qids.append(super_qid) + + return super_qids \ No newline at end of file From cfd6bbc608ad6a6e0d9bf67ad8c36a505be0fed7 Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:08:26 +0000 Subject: [PATCH 37/56] souk paths, planck dev, some cals --- sofind/datamodels/act_dr6v4.yaml | 7 +- sofind/datamodels/act_nemo.yaml | 3 +- sofind/datamodels/planck_npipe.yaml | 7 ++ sofind/products/beams/beams_nemo_v5.yaml | 70 ++++++++++++++++ sofind/products/beams/beams_planck_npipe.yaml | 1 + .../calibrations/act_dr6.02_hack.yaml | 17 ++++ .../dr6v4_calday_tonight_daydeep_250826.yaml | 22 +++++ .../dr6v4_calday_tonight_daywide_250826.yaml | 20 +++++ sofind/products/maps/__init__.py | 26 +++++- sofind/products/maps/act_nemov3.yaml | 4 +- sofind/products/maps/act_nemov4.yaml | 4 +- sofind/products/maps/act_nemov5.yaml | 16 ++++ sofind/products/maps/planck_npipe.yaml | 4 +- .../maps/planck_npipe_noise_sims.yaml | 3 +- .../masks/lensing_masks_20250826.yaml | 2 +- .../products/noise_models/act_dr6v4_day.yaml | 84 +++++++++++++++++++ .../noise_models/act_dr6v4_inout_split.yaml | 3 +- .../noise_models/act_dr6v4_pwv_split.yaml | 3 +- .../tf_dr6v4_240416_lmax1000.yaml | 21 +++++ sofind/qids/planck_qids.yaml | 9 ++ sofind/systems.py | 3 +- 21 files changed, 318 insertions(+), 11 deletions(-) create mode 100644 sofind/products/beams/beams_nemo_v5.yaml create mode 100644 sofind/products/calibrations/act_dr6.02_hack.yaml create mode 100644 sofind/products/calibrations/dr6v4_calday_tonight_daydeep_250826.yaml create mode 100644 sofind/products/calibrations/dr6v4_calday_tonight_daywide_250826.yaml create mode 100644 sofind/products/maps/act_nemov5.yaml create mode 100644 sofind/products/transfer_func/tf_dr6v4_240416_lmax1000.yaml diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index bab9102..7496261 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -37,6 +37,9 @@ calibrations: dr6v4_poleff_231113: dr6v4_poleff_231113.yaml dr6v4_calday_tonight_daydeep: dr6v4_calday_tonight_daydeep.yaml dr6v4_calday_tonight_daywide: dr6v4_calday_tonight_daywide.yaml + dr6v4_calday_tonight_daydeep_250826: dr6v4_calday_tonight_daydeep_250826.yaml + dr6v4_calday_tonight_daywide_250826: dr6v4_calday_tonight_daywide_250826.yaml + act_dr6.02_hack: act_dr6.02_hack.yaml beams: beams_v4_20230130 : beams_v4_20230130.yaml @@ -44,9 +47,11 @@ beams: beams_v4_day_20240115: beams_v4_day_20240115.yaml beams_nemo: beams_nemo.yaml beams_nemo_v4: beams_nemo_v4.yaml + beams_nemo_v5: beams_nemo_v5.yaml beams_nominal: beams_v4_nominal.yaml beams_leakage: beams_v4_leakage.yaml transfer_func: tf_dr6v4_230523: tf_dr6v4_230523.yaml - tf_dr6v4_240416: tf_dr6v4_240416.yaml \ No newline at end of file + tf_dr6v4_240416: tf_dr6v4_240416.yaml + tf_dr6v4_240416_lmax1000: tf_dr6v4_240416_lmax1000.yaml \ No newline at end of file diff --git a/sofind/datamodels/act_nemo.yaml b/sofind/datamodels/act_nemo.yaml index de5e0e2..22d17ef 100644 --- a/sofind/datamodels/act_nemo.yaml +++ b/sofind/datamodels/act_nemo.yaml @@ -2,4 +2,5 @@ qids_config: act_nemo.yaml maps: nemo_v3: act_nemov3.yaml - nemo_v4: act_nemov4.yaml \ No newline at end of file + nemo_v4: act_nemov4.yaml + nemo_v5: act_nemov5.yaml \ No newline at end of file diff --git a/sofind/datamodels/planck_npipe.yaml b/sofind/datamodels/planck_npipe.yaml index 2efc3cf..691b936 100644 --- a/sofind/datamodels/planck_npipe.yaml +++ b/sofind/datamodels/planck_npipe.yaml @@ -7,3 +7,10 @@ maps: beams: beams_nemo: beams_nemo.yaml beams_npipe: beams_planck_npipe.yaml + +catalogs: + inpaint_catalogs_config: inpaint_catalogs.yaml + +masks: + lensing_masks_20250513: lensing_masks_20250513.yaml + lensing_masks_20250826: lensing_masks_20250826.yaml \ No newline at end of file diff --git a/sofind/products/beams/beams_nemo_v5.yaml b/sofind/products/beams/beams_nemo_v5.yaml new file mode 100644 index 0000000..8dc43d3 --- /dev/null +++ b/sofind/products/beams/beams_nemo_v5.yaml @@ -0,0 +1,70 @@ +system_paths: + dirac: + trillium: /scratch/jaejoonk/cmodel/nemo/cand_release/cmodel_20251015/ + penn-gpc: /scratch/cmodel/nemo/cand_release/cmodel_20251015/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/cmodel_20251015 + +allowed_qids_configs: + - act_dr6vX_qids.yaml + - planck_qids.yaml + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + p01: + nemo_freq: 'f090' + p02: + nemo_freq: 'f090' + p03: + nemo_freq: 'f090' + p04: + nemo_freq: 'f090' + p05: + nemo_freq: 'f150' + p06: + nemo_freq: 'f150' + p07: + nemo_freq: 'f150' + p08: + nemo_freq: 'f150' + p09: + nemo_freq: 'f150' + pa4a: + nemo_freq: 'f150' + pa4a_dd: + nemo_freq: 'f150' + pa4a_dw: + nemo_freq: 'f150' + pa4b: + nemo_freq: 'f220' + pa4b_dd: + nemo_freq: 'f220' + pa4b_dw: + nemo_freq: 'f220' + pa5a: + nemo_freq: 'f090' + pa5a_dd: + nemo_freq: 'f090' + pa5a_dw: + nemo_freq: 'f090' + pa5b: + nemo_freq: 'f150' + pa5b_dd: + nemo_freq: 'f150' + pa5b_dw: + nemo_freq: 'f150' + pa6a: + nemo_freq: 'f090' + pa6a_dd: + nemo_freq: 'f090' + pa6a_dw: + nemo_freq: 'f090' + pa6b: + nemo_freq: 'f150' + pa6b_dd: + nemo_freq: 'f150' + pa6b_dw: + nemo_freq: 'f150' + +coadd_beam_file_template: 'beam_dr6_daynight_{nemo_freq}_tfun.txt' +norm: True diff --git a/sofind/products/beams/beams_planck_npipe.yaml b/sofind/products/beams/beams_planck_npipe.yaml index 237cb52..d0d85ac 100644 --- a/sofind/products/beams/beams_planck_npipe.yaml +++ b/sofind/products/beams/beams_planck_npipe.yaml @@ -3,6 +3,7 @@ system_paths: perlmutter: niagara: /gpfs/fs0/project/r/rbond/jiaqu/actpol/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/beams/planck_beams/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/npipe/ allowed_qids_configs: - planck_qids.yaml diff --git a/sofind/products/calibrations/act_dr6.02_hack.yaml b/sofind/products/calibrations/act_dr6.02_hack.yaml new file mode 100644 index 0000000..9c9669f --- /dev/null +++ b/sofind/products/calibrations/act_dr6.02_hack.yaml @@ -0,0 +1,17 @@ +system_paths: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/dr6v4_hack/ + trillium: /scratch/jiaqu/calibration/dr6v4_hack/ +allowed_qids_configs: +- act_dr6vX_qids.yaml +allowed_qids: +- pa4a +- pa4b +- pa5a +- pa5b +- pa6a +- pa6b +allowed_qids_extra_kwargs: +subproduct_kwargs_orders: +cals_file_template: +poleffs_file_template: hack_poleff_pa5amf.pkl +key_template: dr6_{array}_{freq} \ No newline at end of file diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daydeep_250826.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daydeep_250826.yaml new file mode 100644 index 0000000..13d74e9 --- /dev/null +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daydeep_250826.yaml @@ -0,0 +1,22 @@ +system_paths: + dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/cal_day_to_nightdefault/ + della: + perlmutter: + niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/cal_day_to_nightdefault/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/cal_day_to_nightdefault/ + +allowed_qids_configs: +- act_dr6vX_qids.yaml + +allowed_qids: +- pa5a_dd +- pa5b_dd +- pa6a_dd +- pa6b_dd + +allowed_qids_extra_kwargs: + +subproduct_kwargs_orders: +cals_file_template: cal_day_to_nightdefault_daydeep_250826.pkl +poleffs_file_template: +key_template: dr6_{array}_{freq} \ No newline at end of file diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daywide_250826.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daywide_250826.yaml new file mode 100644 index 0000000..8c7c388 --- /dev/null +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daywide_250826.yaml @@ -0,0 +1,20 @@ +system_paths: + dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/calibration/cal_day_to_nightdefault/ + della: + perlmutter: + niagara: /project/r/rbond/jaejoonk/lensing_pipeline_data/calibration/cal_day_to_nightdefault/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/cal_day_to_nightdefault/ + +allowed_qids_configs: +- act_dr6vX_qids.yaml + +allowed_qids: +- pa5a_dw +- pa5b_dw + +allowed_qids_extra_kwargs: + +subproduct_kwargs_orders: +cals_file_template: cal_day_to_nightdefault_daywide_250826.pkl +poleffs_file_template: +key_template: dr6_{array}_{freq} \ No newline at end of file diff --git a/sofind/products/maps/__init__.py b/sofind/products/maps/__init__.py index 1c8003c..01e8af1 100644 --- a/sofind/products/maps/__init__.py +++ b/sofind/products/maps/__init__.py @@ -111,4 +111,28 @@ def read_map(self, qid, split_num=0, coadd=False, maptag='map', if read_map_kwargs is None: read_map_kwargs = {} - return enmap.read_map(fn, **read_map_kwargs) \ No newline at end of file + return enmap.read_map(fn, **read_map_kwargs) + + def get_if_dfact_map(self, subproduct='default'): + + """Check if the map has already been downgraded + + Parameters + ---------- + subproduct : str, optional + Name of map subproduct to load raw products from, by default + 'default'. + + Returns + ------- + np.int + None if information unavailable + dfact applied otherwise + """ + + subprod_dict = self.get_subproduct_dict(__name__, subproduct) + + # Verify if 'norm' exists on subprod_dict + norm_value = subprod_dict.get('dfact', None) + + return norm_value \ No newline at end of file diff --git a/sofind/products/maps/act_nemov3.yaml b/sofind/products/maps/act_nemov3.yaml index 148bd40..3a8ff8c 100644 --- a/sofind/products/maps/act_nemov3.yaml +++ b/sofind/products/maps/act_nemov3.yaml @@ -11,4 +11,6 @@ allowed_qids: 'all' allowed_qids_extra_kwargs: -coadd_map_file_template: 'nemomodel_dr6_all_clustersSNR5{freq}down2_fejer1.fits' \ No newline at end of file +coadd_map_file_template: 'nemomodel_dr6_all_clustersSNR5{freq}down2_fejer1.fits' + +dfact: 2 \ No newline at end of file diff --git a/sofind/products/maps/act_nemov4.yaml b/sofind/products/maps/act_nemov4.yaml index 148bd40..3a8ff8c 100644 --- a/sofind/products/maps/act_nemov4.yaml +++ b/sofind/products/maps/act_nemov4.yaml @@ -11,4 +11,6 @@ allowed_qids: 'all' allowed_qids_extra_kwargs: -coadd_map_file_template: 'nemomodel_dr6_all_clustersSNR5{freq}down2_fejer1.fits' \ No newline at end of file +coadd_map_file_template: 'nemomodel_dr6_all_clustersSNR5{freq}down2_fejer1.fits' + +dfact: 2 \ No newline at end of file diff --git a/sofind/products/maps/act_nemov5.yaml b/sofind/products/maps/act_nemov5.yaml new file mode 100644 index 0000000..a00c71d --- /dev/null +++ b/sofind/products/maps/act_nemov5.yaml @@ -0,0 +1,16 @@ +system_paths: + dirac: + trillium: /scratch/jaejoonk/cmodel/nemo/cand_release/cmodel_20251015 + penn-gpc: /data6/depot/cmodel/nemo/cand_release/cmodel_20251015 + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/models/cmodel_20251015 + +allowed_qids_configs: + - act_nemo.yaml + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +coadd_map_file_template: 'processed_model_f{freq}.fits' + +dfact: 1 \ No newline at end of file diff --git a/sofind/products/maps/planck_npipe.yaml b/sofind/products/maps/planck_npipe.yaml index ceeb492..97441f2 100644 --- a/sofind/products/maps/planck_npipe.yaml +++ b/sofind/products/maps/planck_npipe.yaml @@ -1,7 +1,9 @@ system_paths: perlmutter: /global/cfs/projectdirs/act/data/planck/npipe/fejer1_equ/ niagara: /gpfs/fs0/project/r/rbond/sigurdkn/actpol/planck/npipe/fejer1_equ/ - + dirac: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/maps/planck/ + allowed_qids_configs: - planck_qids.yaml diff --git a/sofind/products/maps/planck_npipe_noise_sims.yaml b/sofind/products/maps/planck_npipe_noise_sims.yaml index 786a886..f8b901e 100644 --- a/sofind/products/maps/planck_npipe_noise_sims.yaml +++ b/sofind/products/maps/planck_npipe_noise_sims.yaml @@ -1,5 +1,6 @@ system_paths: niagara: /project/r/rbond/jiaqu/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/simulations/ allowed_qids_configs: - planck_qids.yaml @@ -38,6 +39,6 @@ allowed_qids_extra_kwargs: # using maptag for sim index (e.g. "0200") and split_num between "A" and "B" # interfaced by solenspipe.preprocessing so shouldn't have to be manually accessed # THESE ARE HEALPIX MAPS -split_map_file_template: 'npipe6v20{split_num}_sim/{maptag}/npipe6v20{split_num}_{freq}_map.fits' +split_map_file_template: 'npipe6v20{split_num}_sim/{maptag}/residual/residual_npipe6v20{split_num}_{freq}_{maptag}.fits' # should really never be using coadd=True # coadd_map_file_template: 'npipe6v20A_sim/{maptag}/npipe6v20A_{freq}_map.fits' diff --git a/sofind/products/masks/lensing_masks_20250826.yaml b/sofind/products/masks/lensing_masks_20250826.yaml index fbe3f95..465088d 100644 --- a/sofind/products/masks/lensing_masks_20250826.yaml +++ b/sofind/products/masks/lensing_masks_20250826.yaml @@ -1,7 +1,7 @@ system_paths: perlmutter: niagara: - dirac: + dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/masks/dr6v4_lensing_250826_masks/ penn-gpc: souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/masks/dr6v4_lensing_250826_masks/ diff --git a/sofind/products/noise_models/act_dr6v4_day.yaml b/sofind/products/noise_models/act_dr6v4_day.yaml index e793e4e..d88174d 100644 --- a/sofind/products/noise_models/act_dr6v4_day.yaml +++ b/sofind/products/noise_models/act_dr6v4_day.yaml @@ -121,6 +121,90 @@ tile_cmbmask_daywide: model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' +tile_cmbmask_daydeep_250513: + noise_model_class: Tiled + data_model_name: act_dr6v4 + subproduct: day + maps_product: maps + maps_subproduct: default + calibrated: false + calibrations_subproduct: null + catalogs_subproduct: inpaint_catalogs + catalog_name: union_catalog_regular_20241002.csv + differenced: true + dtype: f4 + enforce_equal_qid_kwargs: + - patch + - array + - daynight + - num_splits + filter_kwargs: + lim: 0.000001 + lim0: null + post_filt_rel_downgrade: 2 + iso_filt_method: harmonic + ivar_filt_method: basic + ivar_fwhms: null + ivar_lmaxs: null + kfilt_lbounds: null + masks_subproduct: lensing_masks_20250513 + mask_est_name: dr6v4_lensing_250513_daydeep_enhanced_mask_70.fits + mask_est_edgecut: 60 + mask_est_apodization: 180 + mask_obs_name: dr6v4_lensing_250513_daydeep_enhanced_mask_70.fits + mask_obs_edgecut: 10 + model_lim: 0.000001 + model_lim0: null + qid_names_template: '{array}_{daynight}_{freq}' + srcfree: true + delta_ell_smooth: 400 + height_deg: 4.0 + width_deg: 4.0 + model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' + sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' + +tile_cmbmask_daywide_250513: + noise_model_class: Tiled + data_model_name: act_dr6v4 + subproduct: day + maps_product: maps + maps_subproduct: default + calibrated: false + calibrations_subproduct: null + catalogs_subproduct: inpaint_catalogs + catalog_name: union_catalog_regular_20241002.csv + differenced: true + dtype: f4 + enforce_equal_qid_kwargs: + - patch + - array + - daynight + - num_splits + filter_kwargs: + lim: 0.000001 + lim0: null + post_filt_rel_downgrade: 2 + iso_filt_method: harmonic + ivar_filt_method: basic + ivar_fwhms: null + ivar_lmaxs: null + kfilt_lbounds: null + masks_subproduct: lensing_masks_20250513 + mask_est_name: dr6v4_lensing_250513_daywide_enhanced_mask_70.fits + mask_est_edgecut: 60 + mask_est_apodization: 180 + mask_obs_name: dr6v4_lensing_250513_daywide_enhanced_mask_70.fits + mask_obs_edgecut: 10 + model_lim: 0.000001 + model_lim0: null + qid_names_template: '{array}_{daynight}_{freq}' + srcfree: true + delta_ell_smooth: 400 + height_deg: 4.0 + width_deg: 4.0 + model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' + sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' + tile_cmbmask_daydeep_ivfwhm2: noise_model_class: Tiled data_model_name: act_dr6v4 diff --git a/sofind/products/noise_models/act_dr6v4_inout_split.yaml b/sofind/products/noise_models/act_dr6v4_inout_split.yaml index cc66968..140c062 100644 --- a/sofind/products/noise_models/act_dr6v4_inout_split.yaml +++ b/sofind/products/noise_models/act_dr6v4_inout_split.yaml @@ -1,7 +1,8 @@ system_paths: della: /scratch/gpfs/ACT/dr6v4/simulations/noise perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise - + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/simulations/mnms/ + allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/noise_models/act_dr6v4_pwv_split.yaml b/sofind/products/noise_models/act_dr6v4_pwv_split.yaml index a17533a..026de3b 100644 --- a/sofind/products/noise_models/act_dr6v4_pwv_split.yaml +++ b/sofind/products/noise_models/act_dr6v4_pwv_split.yaml @@ -1,7 +1,8 @@ system_paths: della: /scratch/gpfs/ACT/dr6v4/simulations/noise perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise - + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/simulations/mnms/ + allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/transfer_func/tf_dr6v4_240416_lmax1000.yaml b/sofind/products/transfer_func/tf_dr6v4_240416_lmax1000.yaml new file mode 100644 index 0000000..12caf13 --- /dev/null +++ b/sofind/products/transfer_func/tf_dr6v4_240416_lmax1000.yaml @@ -0,0 +1,21 @@ +system_paths: + perlmutter: /global/cfs/cdirs/act/data/alaposta/transfer_functions/dr6_v4_legacy_240416/ + dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/transfer_functions/dr6_v4_legacy_240416/ + niagara: /scratch/r/rbond/jaejoonk/lensing_pipeline_data/transfer_functions/dr6_v4_legacy_240416/ + penn-gpc: /data5/act/DR6_products/transfer_functions/dr6_v4_legacy_240416/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/transfer_func/dr6_v4_legacy_240416/ + +allowed_qids_configs: +- act_dr6vX_qids.yaml + +allowed_qids: +- pa4a +- pa4b +- pa5a +- pa5b +- pa6a +- pa6b + +allowed_qids_extra_kwargs: + +tf_template: 'tf_fit_dr6_{array}_{freq}_logistic_lmax10000.dat' diff --git a/sofind/qids/planck_qids.yaml b/sofind/qids/planck_qids.yaml index bee14a8..8c73403 100644 --- a/sofind/qids/planck_qids.yaml +++ b/sofind/qids/planck_qids.yaml @@ -1,19 +1,28 @@ 'p01': freq: '030' + daynight: planck 'p02': freq: '044' + daynight: planck 'p03': freq: '070' + daynight: planck 'p04': freq: '100' + daynight: planck 'p05': freq: '143' + daynight: planck 'p06': freq: '217' + daynight: planck 'p07': freq: '353' + daynight: planck 'p08': freq: '545' + daynight: planck 'p09': freq: '857' + daynight: planck diff --git a/sofind/systems.py b/sofind/systems.py index 436a59c..af1da5f 100644 --- a/sofind/systems.py +++ b/sofind/systems.py @@ -6,5 +6,6 @@ 'dirac', 'niagara', 'penn-gpc', - 'souk' + 'souk', + 'trillium' ] From de43d35b02a02877e3820d04416af6de2a478b4c Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Fri, 13 Mar 2026 12:20:27 +0000 Subject: [PATCH 38/56] cals * mflike corrections --- .../products/calibrations/act_dr6.02_wmflike.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 sofind/products/calibrations/act_dr6.02_wmflike.yaml diff --git a/sofind/products/calibrations/act_dr6.02_wmflike.yaml b/sofind/products/calibrations/act_dr6.02_wmflike.yaml new file mode 100644 index 0000000..3be5b33 --- /dev/null +++ b/sofind/products/calibrations/act_dr6.02_wmflike.yaml @@ -0,0 +1,15 @@ +system_paths: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/act_dr6.02_wmflike/ + trillium: /scratch/iabril/calibration/act_dr6.02_wmflike/ +allowed_qids_configs: +- act_dr6vX_qids.yaml +allowed_qids: +- pa5a +- pa5b +- pa6a +- pa6b +allowed_qids_extra_kwargs: +subproduct_kwargs_orders: +cals_file_template: act_dr6.02_cals_wmflike.pkl +poleffs_file_template: act_dr6.02_poleffs_wmflike.pkl +key_template: dr6_{array}_{freq} \ No newline at end of file From d03daa15be6abafa9f8207d0b6a2ea374e60f7de Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Fri, 13 Mar 2026 13:36:38 +0000 Subject: [PATCH 39/56] i forgot to add this to dm --- sofind/datamodels/act_dr6v4.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index 7496261..f0997d8 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -40,6 +40,7 @@ calibrations: dr6v4_calday_tonight_daydeep_250826: dr6v4_calday_tonight_daydeep_250826.yaml dr6v4_calday_tonight_daywide_250826: dr6v4_calday_tonight_daywide_250826.yaml act_dr6.02_hack: act_dr6.02_hack.yaml + act_dr6.02_wmflike: act_dr6.02_wmflike.yaml beams: beams_v4_20230130 : beams_v4_20230130.yaml From 46793c0b9fbdf3030f0bdf3f3382fb1122768c6f Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Fri, 13 Mar 2026 13:38:13 +0000 Subject: [PATCH 40/56] just a typo --- sofind/datamodels/act_dr6v4.yaml | 2 +- ...r6v4_240416_lmax1000.yaml => tf_dr6v4_240416_lmax10000.yaml} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename sofind/products/transfer_func/{tf_dr6v4_240416_lmax1000.yaml => tf_dr6v4_240416_lmax10000.yaml} (100%) diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index f0997d8..1fdafac 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -55,4 +55,4 @@ beams: transfer_func: tf_dr6v4_230523: tf_dr6v4_230523.yaml tf_dr6v4_240416: tf_dr6v4_240416.yaml - tf_dr6v4_240416_lmax1000: tf_dr6v4_240416_lmax1000.yaml \ No newline at end of file + tf_dr6v4_240416_lmax10000: tf_dr6v4_240416_lmax10000.yaml \ No newline at end of file diff --git a/sofind/products/transfer_func/tf_dr6v4_240416_lmax1000.yaml b/sofind/products/transfer_func/tf_dr6v4_240416_lmax10000.yaml similarity index 100% rename from sofind/products/transfer_func/tf_dr6v4_240416_lmax1000.yaml rename to sofind/products/transfer_func/tf_dr6v4_240416_lmax10000.yaml From 22a9206b3d963c852a78558c5a060e6f7eccf3ff Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Thu, 19 Mar 2026 13:32:12 +0000 Subject: [PATCH 41/56] coadd planck beams, trillium --- sofind/products/beams/beams_planck_npipe.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sofind/products/beams/beams_planck_npipe.yaml b/sofind/products/beams/beams_planck_npipe.yaml index d0d85ac..7f2780c 100644 --- a/sofind/products/beams/beams_planck_npipe.yaml +++ b/sofind/products/beams/beams_planck_npipe.yaml @@ -4,6 +4,7 @@ system_paths: niagara: /gpfs/fs0/project/r/rbond/jiaqu/actpol/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/beams/planck_beams/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/npipe/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/planck_beams/planck_beams_from_nersc/ allowed_qids_configs: - planck_qids.yaml @@ -13,6 +14,6 @@ allowed_qids: 'all' allowed_qids_extra_kwargs: split_beam_file_template: 'npipe_DR6_{split_num}x{split_num}/bl_T_npipe_DR6_{split_num}x{split_num}_{freq}{split_num}x{freq}{split_num}.dat' -# coadd_beam_file_template: 'coadd_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' +coadd_beam_file_template: 'npipe_DR6_coadd/bl_T_npipe_DR6_coadd_{freq}GHzx{freq}GHz.dat' norm: False \ No newline at end of file From a2606bda809d76d104baec963a2044c1ae6c175d Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Tue, 7 Apr 2026 18:06:00 +0100 Subject: [PATCH 42/56] tf 10000 --- sofind/products/transfer_func/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sofind/products/transfer_func/__init__.py b/sofind/products/transfer_func/__init__.py index a2dbf66..46663fa 100644 --- a/sofind/products/transfer_func/__init__.py +++ b/sofind/products/transfer_func/__init__.py @@ -90,9 +90,8 @@ def read_tf(self, qid, subproduct='default', if subproduct == 'dummy': # dummy transfer function: - ellt = np.arange(2,3000+1) + ellt = np.arange(2,10000+1) tf = np.ones(len(ellt)) - return np.array([ellt, tf]) # use get_tf_fn and some external library to load the data From ae1d1fe8bc94a811d31d6294959407a1d30f7b6f Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Tue, 7 Apr 2026 18:06:13 +0100 Subject: [PATCH 43/56] mnms update --- sofind/products/noise_models/act_dr6v4_day.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sofind/products/noise_models/act_dr6v4_day.yaml b/sofind/products/noise_models/act_dr6v4_day.yaml index d88174d..d48cc19 100644 --- a/sofind/products/noise_models/act_dr6v4_day.yaml +++ b/sofind/products/noise_models/act_dr6v4_day.yaml @@ -160,6 +160,8 @@ tile_cmbmask_daydeep_250513: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -202,6 +204,8 @@ tile_cmbmask_daywide_250513: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' From 5ebe06b74c176da55837267fb6b5797effd68922 Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Tue, 7 Apr 2026 18:06:27 +0100 Subject: [PATCH 44/56] pa4b --- sofind/products/calibrations/act_dr6.02_wmflike.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/sofind/products/calibrations/act_dr6.02_wmflike.yaml b/sofind/products/calibrations/act_dr6.02_wmflike.yaml index 3be5b33..68379cc 100644 --- a/sofind/products/calibrations/act_dr6.02_wmflike.yaml +++ b/sofind/products/calibrations/act_dr6.02_wmflike.yaml @@ -4,6 +4,7 @@ system_paths: allowed_qids_configs: - act_dr6vX_qids.yaml allowed_qids: +- pa4b - pa5a - pa5b - pa6a From 4c8b38d319b6a812bb543bda7c63cc082759239f Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Tue, 7 Apr 2026 18:07:16 +0100 Subject: [PATCH 45/56] new products --- sofind/datamodels/act_dr6v4.yaml | 4 ++ sofind/datamodels/act_nemo.yaml | 3 +- sofind/datamodels/planck_npipe.yaml | 6 +- .../beams/beams_nemo_v5_20260331.yaml | 71 +++++++++++++++++++ .../catalogs/inpaint_catalogs_260331.yaml | 13 ++++ .../catalogs/inpaint_catalogs_260404.yaml | 14 ++++ sofind/products/maps/act_nemov5_20260331.yaml | 17 +++++ sofind/products/maps/pact_nemo_20260403.yaml | 19 +++++ 8 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 sofind/products/beams/beams_nemo_v5_20260331.yaml create mode 100644 sofind/products/catalogs/inpaint_catalogs_260331.yaml create mode 100644 sofind/products/catalogs/inpaint_catalogs_260404.yaml create mode 100644 sofind/products/maps/act_nemov5_20260331.yaml create mode 100644 sofind/products/maps/pact_nemo_20260403.yaml diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index 1fdafac..ff21025 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -6,6 +6,7 @@ maps: pwv_split_config: act_dr6v4_pwv_split.yaml inout_split_config: act_dr6v4_inout_split.yaml t_split_config: act_dr6v4_t_split.yaml + pact_nemo_20260403: pact_nemo_20260403.yaml masks: mnms_masks_config: mnms_masks.yaml @@ -21,6 +22,8 @@ masks: catalogs: inpaint_catalogs_config: inpaint_catalogs.yaml + inpaint_catalogs_260331: inpaint_catalogs_260331.yaml + inpaint_catalogs_260404: inpaint_catalogs_260404.yaml noise_models: default_config: act_dr6v4.yaml @@ -49,6 +52,7 @@ beams: beams_nemo: beams_nemo.yaml beams_nemo_v4: beams_nemo_v4.yaml beams_nemo_v5: beams_nemo_v5.yaml + beams_nemo_v5_20260331: beams_nemo_v5_20260331.yaml beams_nominal: beams_v4_nominal.yaml beams_leakage: beams_v4_leakage.yaml diff --git a/sofind/datamodels/act_nemo.yaml b/sofind/datamodels/act_nemo.yaml index 22d17ef..b93fc62 100644 --- a/sofind/datamodels/act_nemo.yaml +++ b/sofind/datamodels/act_nemo.yaml @@ -3,4 +3,5 @@ qids_config: act_nemo.yaml maps: nemo_v3: act_nemov3.yaml nemo_v4: act_nemov4.yaml - nemo_v5: act_nemov5.yaml \ No newline at end of file + nemo_v5: act_nemov5.yaml + nemo_v5_20260331: act_nemov5_20260331.yaml \ No newline at end of file diff --git a/sofind/datamodels/planck_npipe.yaml b/sofind/datamodels/planck_npipe.yaml index 691b936..2740201 100644 --- a/sofind/datamodels/planck_npipe.yaml +++ b/sofind/datamodels/planck_npipe.yaml @@ -3,14 +3,18 @@ qids_config: planck_qids.yaml maps: default_config: planck_npipe.yaml noise_sims_config: planck_npipe_noise_sims.yaml + pact_nemo_20260403: pact_nemo_20260403.yaml beams: beams_nemo: beams_nemo.yaml + beams_nemo_v5_20260331: beams_nemo_v5_20260331.yaml beams_npipe: beams_planck_npipe.yaml catalogs: inpaint_catalogs_config: inpaint_catalogs.yaml + inpaint_catalogs_260331: inpaint_catalogs_260331.yaml + inpaint_catalogs_260404: inpaint_catalogs_260404.yaml masks: lensing_masks_20250513: lensing_masks_20250513.yaml - lensing_masks_20250826: lensing_masks_20250826.yaml \ No newline at end of file + lensing_masks_20250826: lensing_masks_20250826.yaml diff --git a/sofind/products/beams/beams_nemo_v5_20260331.yaml b/sofind/products/beams/beams_nemo_v5_20260331.yaml new file mode 100644 index 0000000..94ecfdf --- /dev/null +++ b/sofind/products/beams/beams_nemo_v5_20260331.yaml @@ -0,0 +1,71 @@ +system_paths: + dirac: + trillium: + perlmutter: /global/cfs/projectdirs/act/data/msyriac/cmodel_20260331/ + penn-gpc: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/inpaint_catalogs_20260331/ + +allowed_qids_configs: + - act_dr6vX_qids.yaml + - planck_qids.yaml + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + p01: + nemo_freq: 'f090' + p02: + nemo_freq: 'f090' + p03: + nemo_freq: 'f090' + p04: + nemo_freq: 'f090' + p05: + nemo_freq: 'f150' + p06: + nemo_freq: 'f150' + p07: + nemo_freq: 'f150' + p08: + nemo_freq: 'f150' + p09: + nemo_freq: 'f150' + pa4a: + nemo_freq: 'f150' + pa4a_dd: + nemo_freq: 'f150' + pa4a_dw: + nemo_freq: 'f150' + pa4b: + nemo_freq: 'f220' + pa4b_dd: + nemo_freq: 'f220' + pa4b_dw: + nemo_freq: 'f220' + pa5a: + nemo_freq: 'f090' + pa5a_dd: + nemo_freq: 'f090' + pa5a_dw: + nemo_freq: 'f090' + pa5b: + nemo_freq: 'f150' + pa5b_dd: + nemo_freq: 'f150' + pa5b_dw: + nemo_freq: 'f150' + pa6a: + nemo_freq: 'f090' + pa6a_dd: + nemo_freq: 'f090' + pa6a_dw: + nemo_freq: 'f090' + pa6b: + nemo_freq: 'f150' + pa6b_dd: + nemo_freq: 'f150' + pa6b_dw: + nemo_freq: 'f150' + +coadd_beam_file_template: 'beam_dr6_daynight_{nemo_freq}_tfun.txt' +norm: True diff --git a/sofind/products/catalogs/inpaint_catalogs_260331.yaml b/sofind/products/catalogs/inpaint_catalogs_260331.yaml new file mode 100644 index 0000000..afa5250 --- /dev/null +++ b/sofind/products/catalogs/inpaint_catalogs_260331.yaml @@ -0,0 +1,13 @@ +system_paths: + della: + perlmutter: /global/cfs/projectdirs/act/data/msyriac/cmodel_20260331/ + dirac: + penn-gpc: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/inpaint_catalogs_20260331/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + diff --git a/sofind/products/catalogs/inpaint_catalogs_260404.yaml b/sofind/products/catalogs/inpaint_catalogs_260404.yaml new file mode 100644 index 0000000..fd8fa5e --- /dev/null +++ b/sofind/products/catalogs/inpaint_catalogs_260404.yaml @@ -0,0 +1,14 @@ +system_paths: + della: + perlmutter: /global/cfs/projectdirs/act/data/iabril/cluster_and_inpaint_20260404/ + dirac: + penn-gpc: + trillium: /scratch/iabril/newyear/inpaint_catalogs_20260404/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/inpaint_catalogs_20260404/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + diff --git a/sofind/products/maps/act_nemov5_20260331.yaml b/sofind/products/maps/act_nemov5_20260331.yaml new file mode 100644 index 0000000..a0aa621 --- /dev/null +++ b/sofind/products/maps/act_nemov5_20260331.yaml @@ -0,0 +1,17 @@ +system_paths: + dirac: + trillium: + penn-gpc: + perlmutter: /global/cfs/projectdirs/act/data/msyriac/cmodel_20260331/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/inpaint_catalogs_20260331/ + +allowed_qids_configs: + - act_nemo.yaml + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +coadd_map_file_template: 'processed_model_f{freq}_v20260331.fits' + +dfact: 1 \ No newline at end of file diff --git a/sofind/products/maps/pact_nemo_20260403.yaml b/sofind/products/maps/pact_nemo_20260403.yaml new file mode 100644 index 0000000..4a8441f --- /dev/null +++ b/sofind/products/maps/pact_nemo_20260403.yaml @@ -0,0 +1,19 @@ +system_paths: + dirac: + trillium: /scratch/iabril/newyear/nemo/model_maps/ + penn-gpc: + perlmutter: /global/cfs/projectdirs/act/data/iabril/cluster_and_inpaint_20260404/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/cmodel_20260403/ + +allowed_qids_configs: +# - planck_qids.yaml + - act_dr6vX_qids.yaml + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +coadd_map_file_template: 'candidatesModelMap_{daynight}_{array}_{freq}_coadd_fixedSNR5_ring0_extobj0_dust0_find0.fits' +split_map_file_template: 'candidatesModelMap_{freq}_split{split_num}_fixedSNR5_ring0_extobj0_dust0_find0.fits' + +dfact: 1 \ No newline at end of file From b19f232ae0878af0c5d50535c58cb62944ff2811 Mon Sep 17 00:00:00 2001 From: Joshua Kim <40779225+jaejoonk@users.noreply.github.com> Date: Tue, 7 Apr 2026 14:09:15 -0400 Subject: [PATCH 46/56] nominal beams for trillium + slightly different wmflike cals? --- sofind/products/beams/beams_v4_nominal.yaml | 4 ++-- sofind/products/calibrations/act_dr6.02_wmflike.yaml | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/sofind/products/beams/beams_v4_nominal.yaml b/sofind/products/beams/beams_v4_nominal.yaml index 254b8d4..e47ae71 100644 --- a/sofind/products/beams/beams_v4_nominal.yaml +++ b/sofind/products/beams/beams_v4_nominal.yaml @@ -1,7 +1,7 @@ system_paths: perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/beams/main_beams/nominal/ dirac: - niagara: + trillium: /scratch/jaejoonk/DR6plus_lensing_data/beams_path/main_beams/nominal/ penn-gpc: souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/nominal/ @@ -21,4 +21,4 @@ allowed_qids_extra_kwargs: split_beam_file_template: 'set{split_num}_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' coadd_beam_file_template: 'coadd_{array}_{freq}_{daynight}_beam_tform_jitter_cmb.txt' -norm: True \ No newline at end of file +norm: True diff --git a/sofind/products/calibrations/act_dr6.02_wmflike.yaml b/sofind/products/calibrations/act_dr6.02_wmflike.yaml index 3be5b33..49d190c 100644 --- a/sofind/products/calibrations/act_dr6.02_wmflike.yaml +++ b/sofind/products/calibrations/act_dr6.02_wmflike.yaml @@ -1,15 +1,17 @@ system_paths: souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/act_dr6.02_wmflike/ - trillium: /scratch/iabril/calibration/act_dr6.02_wmflike/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/act_dr6.02_wmflike/ +# trillium: /scratch/iabril/calibration/act_dr6.02_wmflike/ allowed_qids_configs: - act_dr6vX_qids.yaml allowed_qids: +- pa4b - pa5a - pa5b - pa6a - pa6b allowed_qids_extra_kwargs: subproduct_kwargs_orders: -cals_file_template: act_dr6.02_cals_wmflike.pkl -poleffs_file_template: act_dr6.02_poleffs_wmflike.pkl -key_template: dr6_{array}_{freq} \ No newline at end of file +cals_file_template: act_dr6.02_cals_wmflike_v3.pkl +poleffs_file_template: act_dr6.02_poleffs_wmflike_v2.pkl +key_template: dr6_{array}_{freq} From d88ceedb0ed49429d4d6a475178e5a0eb9f93128 Mon Sep 17 00:00:00 2001 From: Joshua Kim <40779225+jaejoonk@users.noreply.github.com> Date: Tue, 7 Apr 2026 14:14:41 -0400 Subject: [PATCH 47/56] fixed values of wmflike cals/poleffs --- sofind/products/calibrations/act_dr6.02_wmflike.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sofind/products/calibrations/act_dr6.02_wmflike.yaml b/sofind/products/calibrations/act_dr6.02_wmflike.yaml index 49d190c..35018a7 100644 --- a/sofind/products/calibrations/act_dr6.02_wmflike.yaml +++ b/sofind/products/calibrations/act_dr6.02_wmflike.yaml @@ -1,7 +1,6 @@ system_paths: souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/act_dr6.02_wmflike/ - trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/act_dr6.02_wmflike/ -# trillium: /scratch/iabril/calibration/act_dr6.02_wmflike/ + trillium: /scratch/iabril/calibration/act_dr6.02_wmflike/ allowed_qids_configs: - act_dr6vX_qids.yaml allowed_qids: @@ -12,6 +11,6 @@ allowed_qids: - pa6b allowed_qids_extra_kwargs: subproduct_kwargs_orders: -cals_file_template: act_dr6.02_cals_wmflike_v3.pkl -poleffs_file_template: act_dr6.02_poleffs_wmflike_v2.pkl +cals_file_template: act_dr6.02_cals_wmflike.pkl +poleffs_file_template: act_dr6.02_poleffs_wmflike.pkl key_template: dr6_{array}_{freq} From 4d2dac30ad800cce7c8c7d33533015b4c623b8db Mon Sep 17 00:00:00 2001 From: Joshua Kim <40779225+jaejoonk@users.noreply.github.com> Date: Tue, 7 Apr 2026 14:20:26 -0400 Subject: [PATCH 48/56] noise model compatibility with mnms + removing duplicate entries in act_dr6v4_day --- sofind/products/noise_models/act_dr6v4.yaml | 8 ++ .../products/noise_models/act_dr6v4_day.yaml | 89 +------------------ .../noise_models/act_dr6v4_el_split.yaml | 8 ++ .../noise_models/act_dr6v4_inout_split.yaml | 8 ++ .../noise_models/act_dr6v4_pwv_split.yaml | 8 ++ 5 files changed, 36 insertions(+), 85 deletions(-) diff --git a/sofind/products/noise_models/act_dr6v4.yaml b/sofind/products/noise_models/act_dr6v4.yaml index 16ed510..680125f 100644 --- a/sofind/products/noise_models/act_dr6v4.yaml +++ b/sofind/products/noise_models/act_dr6v4.yaml @@ -80,6 +80,8 @@ tile_cmbmask: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -122,6 +124,8 @@ tile_cmbmask_ivfwhm2: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -202,6 +206,8 @@ fdw_cmbmask: w_lmax: 10800 w_lmax_j: 5300 w_lmin: 10 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -282,5 +288,7 @@ fdw_cmbmask_ivfwhm2: w_lmax: 10800 w_lmax_j: 5300 w_lmin: 10 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' diff --git a/sofind/products/noise_models/act_dr6v4_day.yaml b/sofind/products/noise_models/act_dr6v4_day.yaml index 7248a0d..ba77635 100644 --- a/sofind/products/noise_models/act_dr6v4_day.yaml +++ b/sofind/products/noise_models/act_dr6v4_day.yaml @@ -39,91 +39,6 @@ allowed_qids_extra_kwargs: pa5b_dw: num_splits: 4 -tile_cmbmask_daydeep_250513: - noise_model_class: Tiled - data_model_name: act_dr6v4 - subproduct: day - maps_product: maps - maps_subproduct: default - calibrated: false - calibrations_subproduct: null - catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20241002.csv - differenced: true - dtype: f4 - enforce_equal_qid_kwargs: - - patch - - array - - daynight - - num_splits - filter_kwargs: - lim: 0.000001 - lim0: null - post_filt_rel_downgrade: 2 - iso_filt_method: harmonic - ivar_filt_method: basic - ivar_fwhms: null - ivar_lmaxs: null - kfilt_lbounds: null - # subject to change! - masks_subproduct: lensing_masks_20250513 - mask_est_name: dr6v4_lensing_250513_daydeep_enhanced_mask_70.fits - mask_est_edgecut: 60 - mask_est_apodization: 180 - mask_obs_name: dr6v4_lensing_250513_daydeep_enhanced_mask_70.fits - mask_obs_edgecut: 10 - model_lim: 0.000001 - model_lim0: null - qid_names_template: '{array}_{daynight}_{freq}' - srcfree: true - delta_ell_smooth: 400 - height_deg: 4.0 - width_deg: 4.0 - model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' - sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' - -tile_cmbmask_daywide_250513: - noise_model_class: Tiled - data_model_name: act_dr6v4 - subproduct: day - maps_product: maps - maps_subproduct: default - calibrated: false - calibrations_subproduct: null - catalogs_subproduct: inpaint_catalogs - catalog_name: union_catalog_regular_20241002.csv - differenced: true - dtype: f4 - enforce_equal_qid_kwargs: - - patch - - array - - daynight - - num_splits - filter_kwargs: - lim: 0.000001 - lim0: null - post_filt_rel_downgrade: 2 - iso_filt_method: harmonic - ivar_filt_method: basic - ivar_fwhms: null - ivar_lmaxs: null - kfilt_lbounds: null - masks_subproduct: lensing_masks_20250513 - mask_est_name: dr6v4_lensing_250513_daywide_enhanced_mask_70.fits - mask_est_edgecut: 60 - mask_est_apodization: 180 - mask_obs_name: dr6v4_lensing_250513_daywide_enhanced_mask_70.fits - mask_obs_edgecut: 10 - model_lim: 0.000001 - model_lim0: null - qid_names_template: '{array}_{daynight}_{freq}' - srcfree: true - delta_ell_smooth: 400 - height_deg: 4.0 - width_deg: 4.0 - model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' - sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' - tile_cmbmask_daydeep_250513: noise_model_class: Tiled data_model_name: act_dr6v4 @@ -251,6 +166,8 @@ tile_cmbmask_daydeep_ivfwhm2: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -293,5 +210,7 @@ tile_cmbmask_daywide_ivfwhm2: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' \ No newline at end of file diff --git a/sofind/products/noise_models/act_dr6v4_el_split.yaml b/sofind/products/noise_models/act_dr6v4_el_split.yaml index 1b97417..29d4e4a 100644 --- a/sofind/products/noise_models/act_dr6v4_el_split.yaml +++ b/sofind/products/noise_models/act_dr6v4_el_split.yaml @@ -71,6 +71,8 @@ tile_cmbmask: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_null_{el_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_null_{el_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -113,6 +115,8 @@ tile_cmbmask_ivfwhm2: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_null_{el_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_null_{el_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -193,6 +197,8 @@ fdw_cmbmask: w_lmax: 10800 w_lmax_j: 5300 w_lmin: 10 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_null_{el_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_null_{el_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -273,5 +279,7 @@ fdw_cmbmask_ivfwhm2: w_lmax: 10800 w_lmax_j: 5300 w_lmin: 10 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_null_{el_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_null_{el_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' diff --git a/sofind/products/noise_models/act_dr6v4_inout_split.yaml b/sofind/products/noise_models/act_dr6v4_inout_split.yaml index 98b3216..85444a7 100644 --- a/sofind/products/noise_models/act_dr6v4_inout_split.yaml +++ b/sofind/products/noise_models/act_dr6v4_inout_split.yaml @@ -72,6 +72,8 @@ tile_cmbmask: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_null_{inout_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_null_{inout_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -114,6 +116,8 @@ tile_cmbmask_ivfwhm2: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_null_{inout_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_null_{inout_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -194,6 +198,8 @@ fdw_cmbmask: w_lmax: 10800 w_lmax_j: 5300 w_lmin: 10 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_null_{inout_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_null_{inout_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -274,5 +280,7 @@ fdw_cmbmask_ivfwhm2: w_lmax: 10800 w_lmax_j: 5300 w_lmin: 10 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_null_{inout_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_null_{inout_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' \ No newline at end of file diff --git a/sofind/products/noise_models/act_dr6v4_pwv_split.yaml b/sofind/products/noise_models/act_dr6v4_pwv_split.yaml index e4ff4c1..3393158 100644 --- a/sofind/products/noise_models/act_dr6v4_pwv_split.yaml +++ b/sofind/products/noise_models/act_dr6v4_pwv_split.yaml @@ -72,6 +72,8 @@ tile_cmbmask: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_null_{pwv_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_null_{pwv_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -114,6 +116,8 @@ tile_cmbmask_ivfwhm2: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_null_{pwv_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_null_{pwv_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -194,6 +198,8 @@ fdw_cmbmask: w_lmax: 10800 w_lmax_j: 5300 w_lmin: 10 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_null_{pwv_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_null_{pwv_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -274,5 +280,7 @@ fdw_cmbmask_ivfwhm2: w_lmax: 10800 w_lmax_j: 5300 w_lmin: 10 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_null_{pwv_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_null_{pwv_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' \ No newline at end of file From 403749d57d70fd169dbb7e9bdd8a0220b438cee9 Mon Sep 17 00:00:00 2001 From: iabrilcabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Wed, 8 Apr 2026 16:37:03 +0100 Subject: [PATCH 49/56] custom tsz for custom --- sofind/datamodels/act_dr6v4.yaml | 3 +++ .../products/maps/pact_nemo_20260403_el.yaml | 18 ++++++++++++++++++ .../maps/pact_nemo_20260403_inout.yaml | 18 ++++++++++++++++++ .../products/maps/pact_nemo_20260403_pwv.yaml | 18 ++++++++++++++++++ 4 files changed, 57 insertions(+) create mode 100644 sofind/products/maps/pact_nemo_20260403_el.yaml create mode 100644 sofind/products/maps/pact_nemo_20260403_inout.yaml create mode 100644 sofind/products/maps/pact_nemo_20260403_pwv.yaml diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index 42550e6..8cf315c 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -7,6 +7,9 @@ maps: inout_split_config: act_dr6v4_inout_split.yaml t_split_config: act_dr6v4_t_split.yaml pact_nemo_20260403: pact_nemo_20260403.yaml + pact_nemo_20260403_el: pact_nemo_20260403_el.yaml + pact_nemo_20260403_inout: pact_nemo_20260403_inout.yaml + pact_nemo_20260403_pwv: pact_nemo_20260403_pwv.yaml masks: mnms_masks_config: mnms_masks.yaml diff --git a/sofind/products/maps/pact_nemo_20260403_el.yaml b/sofind/products/maps/pact_nemo_20260403_el.yaml new file mode 100644 index 0000000..b973dea --- /dev/null +++ b/sofind/products/maps/pact_nemo_20260403_el.yaml @@ -0,0 +1,18 @@ +system_paths: + dirac: + trillium: /scratch/iabril/newyear/nemo/model_maps/ + penn-gpc: + perlmutter: + souk: + +allowed_qids_configs: +# - planck_qids.yaml + - act_dr6vX_qids.yaml + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +coadd_map_file_template: 'candidatesModelMap_{daynight}_{array}_{freq}_coadd_{el_split}_fixedSNR5_ring0_extobj0_dust0_find0.fits' + +dfact: 1 \ No newline at end of file diff --git a/sofind/products/maps/pact_nemo_20260403_inout.yaml b/sofind/products/maps/pact_nemo_20260403_inout.yaml new file mode 100644 index 0000000..590c96f --- /dev/null +++ b/sofind/products/maps/pact_nemo_20260403_inout.yaml @@ -0,0 +1,18 @@ +system_paths: + dirac: + trillium: /scratch/iabril/newyear/nemo/model_maps/ + penn-gpc: + perlmutter: + souk: + +allowed_qids_configs: +# - planck_qids.yaml + - act_dr6vX_qids.yaml + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +coadd_map_file_template: 'candidatesModelMap_{daynight}_{array}_{freq}_coadd_{inout_split}_fixedSNR5_ring0_extobj0_dust0_find0.fits' + +dfact: 1 \ No newline at end of file diff --git a/sofind/products/maps/pact_nemo_20260403_pwv.yaml b/sofind/products/maps/pact_nemo_20260403_pwv.yaml new file mode 100644 index 0000000..21d43ac --- /dev/null +++ b/sofind/products/maps/pact_nemo_20260403_pwv.yaml @@ -0,0 +1,18 @@ +system_paths: + dirac: + trillium: /scratch/iabril/newyear/nemo/model_maps/ + penn-gpc: + perlmutter: + souk: + +allowed_qids_configs: +# - planck_qids.yaml + - act_dr6vX_qids.yaml + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + +coadd_map_file_template: 'candidatesModelMap_{daynight}_{array}_{freq}_coadd_{pwv_split}_fixedSNR5_ring0_extobj0_dust0_find0.fits' + +dfact: 1 \ No newline at end of file From be13762471f10cdaa53d3639421434456b62bf05 Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Mon, 13 Apr 2026 13:43:29 +0100 Subject: [PATCH 50/56] new cals, new tsz --- sofind/datamodels/act_dr6v4.yaml | 6 +++++ .../calibrations/act_dr6.02_wmflike.yaml | 6 +++++ .../dr6v4_calday_tonight_daydeep_260410.yaml | 22 +++++++++++++++++ .../dr6v4_calday_tonight_daywide_260410.yaml | 20 ++++++++++++++++ sofind/products/maps/day_nemo_20260410.yaml | 24 +++++++++++++++++++ sofind/products/maps/pact_nemo_20260403.yaml | 7 ++++-- .../products/maps/pact_nemo_20260403_el.yaml | 21 ++++++++++++++++ .../maps/pact_nemo_20260403_inout.yaml | 21 ++++++++++++++++ .../products/maps/pact_nemo_20260403_pwv.yaml | 21 ++++++++++++++++ 9 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 sofind/products/calibrations/dr6v4_calday_tonight_daydeep_260410.yaml create mode 100644 sofind/products/calibrations/dr6v4_calday_tonight_daywide_260410.yaml create mode 100644 sofind/products/maps/day_nemo_20260410.yaml create mode 100644 sofind/products/maps/pact_nemo_20260403_el.yaml create mode 100644 sofind/products/maps/pact_nemo_20260403_inout.yaml create mode 100644 sofind/products/maps/pact_nemo_20260403_pwv.yaml diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index ff21025..5316f03 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -7,6 +7,10 @@ maps: inout_split_config: act_dr6v4_inout_split.yaml t_split_config: act_dr6v4_t_split.yaml pact_nemo_20260403: pact_nemo_20260403.yaml + pact_nemo_20260403_el: pact_nemo_20260403_el.yaml + pact_nemo_20260403_inout: pact_nemo_20260403_inout.yaml + pact_nemo_20260403_pwv: pact_nemo_20260403_pwv.yaml + day_nemo_20260410: day_nemo_20260410.yaml masks: mnms_masks_config: mnms_masks.yaml @@ -42,6 +46,8 @@ calibrations: dr6v4_calday_tonight_daywide: dr6v4_calday_tonight_daywide.yaml dr6v4_calday_tonight_daydeep_250826: dr6v4_calday_tonight_daydeep_250826.yaml dr6v4_calday_tonight_daywide_250826: dr6v4_calday_tonight_daywide_250826.yaml + dr6v4_calday_tonight_daywide_260410: dr6v4_calday_tonight_daywide_260410.yaml + dr6v4_calday_tonight_daydeep_260410: dr6v4_calday_tonight_daydeep_260410.yaml act_dr6.02_hack: act_dr6.02_hack.yaml act_dr6.02_wmflike: act_dr6.02_wmflike.yaml diff --git a/sofind/products/calibrations/act_dr6.02_wmflike.yaml b/sofind/products/calibrations/act_dr6.02_wmflike.yaml index 68379cc..6f940d5 100644 --- a/sofind/products/calibrations/act_dr6.02_wmflike.yaml +++ b/sofind/products/calibrations/act_dr6.02_wmflike.yaml @@ -9,6 +9,12 @@ allowed_qids: - pa5b - pa6a - pa6b +- pa5a_dw +- pa5b_dw +- pa5a_dd +- pa5b_dd +- pa6a_dd +- pa6b_dd allowed_qids_extra_kwargs: subproduct_kwargs_orders: cals_file_template: act_dr6.02_cals_wmflike.pkl diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daydeep_260410.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daydeep_260410.yaml new file mode 100644 index 0000000..2c65b3b --- /dev/null +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daydeep_260410.yaml @@ -0,0 +1,22 @@ +system_paths: + dirac: + della: + perlmutter: + niagara: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/cal_day_to_nightdefault/ + +allowed_qids_configs: +- act_dr6vX_qids.yaml + +allowed_qids: +- pa5a_dd +- pa5b_dd +- pa6a_dd +- pa6b_dd + +allowed_qids_extra_kwargs: + +subproduct_kwargs_orders: +cals_file_template: cal_day_to_nightdefault_daydeep_260410.pkl +poleffs_file_template: +key_template: dr6_{array}_{freq} \ No newline at end of file diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daywide_260410.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daywide_260410.yaml new file mode 100644 index 0000000..724f39a --- /dev/null +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daywide_260410.yaml @@ -0,0 +1,20 @@ +system_paths: + dirac: + della: + perlmutter: + niagara: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/cal_day_to_nightdefault/ + +allowed_qids_configs: +- act_dr6vX_qids.yaml + +allowed_qids: +- pa5a_dw +- pa5b_dw + +allowed_qids_extra_kwargs: + +subproduct_kwargs_orders: +cals_file_template: cal_day_to_nightdefault_daywide_260410.pkl +poleffs_file_template: +key_template: dr6_{array}_{freq} \ No newline at end of file diff --git a/sofind/products/maps/day_nemo_20260410.yaml b/sofind/products/maps/day_nemo_20260410.yaml new file mode 100644 index 0000000..cfa5356 --- /dev/null +++ b/sofind/products/maps/day_nemo_20260410.yaml @@ -0,0 +1,24 @@ +system_paths: + dirac: + trillium: /scratch/iabril/newyear/nemo/model_maps/ + penn-gpc: + perlmutter: + souk: /cephfs/soukdata/user_data/iabril/dr6plus/daylens_cal/sources/cluster_verify/processed_maps + +allowed_qids_configs: + - act_dr6vX_qids.yaml + +allowed_qids: +- pa5a_dw +- pa5b_dw +- pa5a_dd +- pa5b_dd +- pa6a_dd +- pa6b_dd + +allowed_qids_extra_kwargs: + +coadd_map_file_template: 'tSZ_model_{daynight}_{array}_{freq}_coadd.fits' +split_map_file_template: '' + +dfact: 1 \ No newline at end of file diff --git a/sofind/products/maps/pact_nemo_20260403.yaml b/sofind/products/maps/pact_nemo_20260403.yaml index 4a8441f..b3ea66f 100644 --- a/sofind/products/maps/pact_nemo_20260403.yaml +++ b/sofind/products/maps/pact_nemo_20260403.yaml @@ -6,10 +6,13 @@ system_paths: souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/cmodel_20260403/ allowed_qids_configs: -# - planck_qids.yaml - act_dr6vX_qids.yaml -allowed_qids: 'all' +allowed_qids: +- pa5a +- pa5b +- pa6a +- pa6b allowed_qids_extra_kwargs: diff --git a/sofind/products/maps/pact_nemo_20260403_el.yaml b/sofind/products/maps/pact_nemo_20260403_el.yaml new file mode 100644 index 0000000..ffb2aef --- /dev/null +++ b/sofind/products/maps/pact_nemo_20260403_el.yaml @@ -0,0 +1,21 @@ +system_paths: + dirac: + trillium: /scratch/iabril/newyear/nemo/model_maps/ + penn-gpc: + perlmutter: + souk: + +allowed_qids_configs: + - act_dr6vX_qids.yaml + +allowed_qids: +- pa5a +- pa5b +- pa6a +- pa6b + +allowed_qids_extra_kwargs: + +coadd_map_file_template: 'candidatesModelMap_{array}_{freq}_coadd_{el_split}_fixedSNR5_ring0_extobj0_dust0_find0.fits' + +dfact: 1 \ No newline at end of file diff --git a/sofind/products/maps/pact_nemo_20260403_inout.yaml b/sofind/products/maps/pact_nemo_20260403_inout.yaml new file mode 100644 index 0000000..32c9d11 --- /dev/null +++ b/sofind/products/maps/pact_nemo_20260403_inout.yaml @@ -0,0 +1,21 @@ +system_paths: + dirac: + trillium: /scratch/iabril/newyear/nemo/model_maps/ + penn-gpc: + perlmutter: + souk: + +allowed_qids_configs: + - act_dr6vX_qids.yaml + +allowed_qids: +- pa5a +- pa5b +- pa6a +- pa6b + +allowed_qids_extra_kwargs: + +coadd_map_file_template: 'candidatesModelMap_{array}_{freq}_coadd_{inout_split}_fixedSNR5_ring0_extobj0_dust0_find0.fits' + +dfact: 1 \ No newline at end of file diff --git a/sofind/products/maps/pact_nemo_20260403_pwv.yaml b/sofind/products/maps/pact_nemo_20260403_pwv.yaml new file mode 100644 index 0000000..44ad1e1 --- /dev/null +++ b/sofind/products/maps/pact_nemo_20260403_pwv.yaml @@ -0,0 +1,21 @@ +system_paths: + dirac: + trillium: /scratch/iabril/newyear/nemo/model_maps/ + penn-gpc: + perlmutter: + souk: + +allowed_qids_configs: + - act_dr6vX_qids.yaml + +allowed_qids: +- pa5a +- pa5b +- pa6a +- pa6b + +allowed_qids_extra_kwargs: + +coadd_map_file_template: 'candidatesModelMap_{array}_{freq}_coadd_{pwv_split}_fixedSNR5_ring0_extobj0_dust0_find0.fits' + +dfact: 1 \ No newline at end of file From c43be62ec875b8c70f9bf8dc5d9fa777d7089aba Mon Sep 17 00:00:00 2001 From: Irene Abril Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Thu, 23 Apr 2026 15:56:44 +0100 Subject: [PATCH 51/56] tsz and inpaint --- sofind/datamodels/act_dr6v4.yaml | 1 + sofind/datamodels/planck_npipe.yaml | 2 ++ .../catalogs/inpaint_catalogs_260417.yaml | 14 +++++++++++ .../catalogs/inpaint_catalogs_260421.yaml | 14 +++++++++++ sofind/products/maps/day_nemo_20260420.yaml | 24 +++++++++++++++++++ sofind/products/maps/pact_nemo_20260403.yaml | 11 +++------ .../products/noise_models/act_dr6v4_day.yaml | 8 +++---- 7 files changed, 62 insertions(+), 12 deletions(-) create mode 100644 sofind/products/catalogs/inpaint_catalogs_260417.yaml create mode 100644 sofind/products/catalogs/inpaint_catalogs_260421.yaml create mode 100644 sofind/products/maps/day_nemo_20260420.yaml diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index 5316f03..1cbcd45 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -11,6 +11,7 @@ maps: pact_nemo_20260403_inout: pact_nemo_20260403_inout.yaml pact_nemo_20260403_pwv: pact_nemo_20260403_pwv.yaml day_nemo_20260410: day_nemo_20260410.yaml + day_nemo_20260420: day_nemo_20260420.yaml masks: mnms_masks_config: mnms_masks.yaml diff --git a/sofind/datamodels/planck_npipe.yaml b/sofind/datamodels/planck_npipe.yaml index 2740201..a246e20 100644 --- a/sofind/datamodels/planck_npipe.yaml +++ b/sofind/datamodels/planck_npipe.yaml @@ -14,6 +14,8 @@ catalogs: inpaint_catalogs_config: inpaint_catalogs.yaml inpaint_catalogs_260331: inpaint_catalogs_260331.yaml inpaint_catalogs_260404: inpaint_catalogs_260404.yaml + inpaint_catalogs_260417: inpaint_catalogs_260417.yaml + inpaint_catalogs_260421: inpaint_catalogs_260421.yaml masks: lensing_masks_20250513: lensing_masks_20250513.yaml diff --git a/sofind/products/catalogs/inpaint_catalogs_260417.yaml b/sofind/products/catalogs/inpaint_catalogs_260417.yaml new file mode 100644 index 0000000..8edc45f --- /dev/null +++ b/sofind/products/catalogs/inpaint_catalogs_260417.yaml @@ -0,0 +1,14 @@ +system_paths: + della: + perlmutter: + dirac: + penn-gpc: + trillium: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/inpaint_catalogs_20260417/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + diff --git a/sofind/products/catalogs/inpaint_catalogs_260421.yaml b/sofind/products/catalogs/inpaint_catalogs_260421.yaml new file mode 100644 index 0000000..6f91452 --- /dev/null +++ b/sofind/products/catalogs/inpaint_catalogs_260421.yaml @@ -0,0 +1,14 @@ +system_paths: + della: + perlmutter: + dirac: + penn-gpc: + trillium: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/inpaint_catalogs_20260421/ + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: + diff --git a/sofind/products/maps/day_nemo_20260420.yaml b/sofind/products/maps/day_nemo_20260420.yaml new file mode 100644 index 0000000..ed7bf9d --- /dev/null +++ b/sofind/products/maps/day_nemo_20260420.yaml @@ -0,0 +1,24 @@ +system_paths: + dirac: + trillium: + penn-gpc: + perlmutter: + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/cmodel_20260420/ + +allowed_qids_configs: + - act_dr6vX_qids.yaml + +allowed_qids: +- pa5a_dw +- pa5b_dw +- pa5a_dd +- pa5b_dd +- pa6a_dd +- pa6b_dd + +allowed_qids_extra_kwargs: + +coadd_map_file_template: 'candidatesModelMap_{daynight}_{array}_{freq}_coadd_fixedSNR5_ring0_extobj0_dust0_find0_beamtaperell1.fits' +split_map_file_template: '' + +dfact: 1 \ No newline at end of file diff --git a/sofind/products/maps/pact_nemo_20260403.yaml b/sofind/products/maps/pact_nemo_20260403.yaml index b3ea66f..910d83f 100644 --- a/sofind/products/maps/pact_nemo_20260403.yaml +++ b/sofind/products/maps/pact_nemo_20260403.yaml @@ -5,18 +5,13 @@ system_paths: perlmutter: /global/cfs/projectdirs/act/data/iabril/cluster_and_inpaint_20260404/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/cmodel_20260403/ -allowed_qids_configs: - - act_dr6vX_qids.yaml +allowed_qids_configs: 'all' -allowed_qids: -- pa5a -- pa5b -- pa6a -- pa6b +allowed_qids: 'all' allowed_qids_extra_kwargs: coadd_map_file_template: 'candidatesModelMap_{daynight}_{array}_{freq}_coadd_fixedSNR5_ring0_extobj0_dust0_find0.fits' -split_map_file_template: 'candidatesModelMap_{freq}_split{split_num}_fixedSNR5_ring0_extobj0_dust0_find0.fits' +split_map_file_template: 'candidatesModelMap_{freq}_split{split_num}_fixedSNR5_ring0_extobj0_dust0_find0_goodbeam_tcmbbandcorr.fits' dfact: 1 \ No newline at end of file diff --git a/sofind/products/noise_models/act_dr6v4_day.yaml b/sofind/products/noise_models/act_dr6v4_day.yaml index d48cc19..0509360 100644 --- a/sofind/products/noise_models/act_dr6v4_day.yaml +++ b/sofind/products/noise_models/act_dr6v4_day.yaml @@ -160,8 +160,8 @@ tile_cmbmask_daydeep_250513: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 - inpaint_radius: 6 - inpaint_thumb_width: 120 +# inpaint_radius: 6 +# inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -204,8 +204,8 @@ tile_cmbmask_daywide_250513: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 - inpaint_radius: 6 - inpaint_thumb_width: 120 +# inpaint_radius: 6 +# inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' From a1c40bb91fc5ba7591dd16931fc07ebf22756cf2 Mon Sep 17 00:00:00 2001 From: Irene Abril-Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Thu, 23 Apr 2026 11:49:29 -0400 Subject: [PATCH 52/56] all places --- sofind/products/beams/beams_v4_day_20240115.yaml | 1 + sofind/products/beams/beams_v4_nominal.yaml | 2 +- sofind/products/calibrations/act_dr6.02.yaml | 2 ++ sofind/products/catalogs/inpaint_catalogs_260421.yaml | 2 +- sofind/products/maps/day_nemo_20260420.yaml | 2 +- sofind/products/masks/lensing_masks_20250513.yaml | 5 +++-- sofind/products/masks/lensing_masks_20250826.yaml | 8 ++++---- .../products/transfer_func/tf_dr6v4_240416_lmax10000.yaml | 3 ++- 8 files changed, 15 insertions(+), 10 deletions(-) diff --git a/sofind/products/beams/beams_v4_day_20240115.yaml b/sofind/products/beams/beams_v4_day_20240115.yaml index 4b58e9a..632e568 100644 --- a/sofind/products/beams/beams_v4_day_20240115.yaml +++ b/sofind/products/beams/beams_v4_day_20240115.yaml @@ -1,5 +1,6 @@ system_paths: niagara: /gpfs/fs0/project/r/rbond/sigurdkn/actpol/daybeam_srcs/20240115/release/ + trillium: /project/rrg-rbond-ac/sigurdkn/actpol/daybeam_srcs/20240115/release/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/v4_maps/20240115_day_beams/ penn-gpc: /data5/act/DR6_products/v4_maps/daytime/20240115_day_beams/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/20240115_day_beams/ diff --git a/sofind/products/beams/beams_v4_nominal.yaml b/sofind/products/beams/beams_v4_nominal.yaml index 254b8d4..ec1492f 100644 --- a/sofind/products/beams/beams_v4_nominal.yaml +++ b/sofind/products/beams/beams_v4_nominal.yaml @@ -1,7 +1,7 @@ system_paths: perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/beams/main_beams/nominal/ dirac: - niagara: + trillium: /scratch/jaejoonk/DR6plus_lensing_data/beams_path/main_beams/nominal/ penn-gpc: souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/beams/nominal/ diff --git a/sofind/products/calibrations/act_dr6.02.yaml b/sofind/products/calibrations/act_dr6.02.yaml index bfb5a99..2441618 100644 --- a/sofind/products/calibrations/act_dr6.02.yaml +++ b/sofind/products/calibrations/act_dr6.02.yaml @@ -3,6 +3,8 @@ system_paths: della: /scratch/gpfs/zatkins/data/simonsobs/mnms/calibrations perlmutter: /global/cfs/cdirs/cmb/data/act_dr6/dr6.02/simulations/noise/calibrations niagara: /scratch/r/rbond/iabril/ACT_DR6_products/calibration/dr6_v4_published/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/calibration/dr6.02_calibrations/ + penn-gpc: /data5/act/DR6_products/calibration/dr6_v4_published/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/act_dr6.02/ allowed_qids_configs: diff --git a/sofind/products/catalogs/inpaint_catalogs_260421.yaml b/sofind/products/catalogs/inpaint_catalogs_260421.yaml index 6f91452..e61365c 100644 --- a/sofind/products/catalogs/inpaint_catalogs_260421.yaml +++ b/sofind/products/catalogs/inpaint_catalogs_260421.yaml @@ -3,7 +3,7 @@ system_paths: perlmutter: dirac: penn-gpc: - trillium: + trillium: /scratch/iabril/newyear/inpaint_catalogs_20260421/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/inpaint_catalogs_20260421/ allowed_qids_configs: 'all' diff --git a/sofind/products/maps/day_nemo_20260420.yaml b/sofind/products/maps/day_nemo_20260420.yaml index ed7bf9d..f8caabe 100644 --- a/sofind/products/maps/day_nemo_20260420.yaml +++ b/sofind/products/maps/day_nemo_20260420.yaml @@ -1,6 +1,6 @@ system_paths: dirac: - trillium: + trillium: /scratch/iabril/newyear/nemo/model_maps/ penn-gpc: perlmutter: souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/catalogs/cmodel_20260420/ diff --git a/sofind/products/masks/lensing_masks_20250513.yaml b/sofind/products/masks/lensing_masks_20250513.yaml index 465b379..638f833 100644 --- a/sofind/products/masks/lensing_masks_20250513.yaml +++ b/sofind/products/masks/lensing_masks_20250513.yaml @@ -1,8 +1,9 @@ system_paths: - perlmutter: + perlmutter: /global/cfs/cdirs/act/data/iabril/masks/ + trillium: /project/rrg-rbond-ac/msyriac/masks/dr6v4_lensing_250513_masks/ niagara: /gpfs/fs0/project/r/rbond/msyriac/masks/dr6v4_lensing_250513_masks/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/masks/ - penn-gpc: /data5/act/DR6_products/masks/ + penn-gpc: /data6/DR6_plus/masks/dr6v4_lensing_250513_masks/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/masks/dr6v4_lensing_250513_masks/ allowed_qids_configs: 'all' diff --git a/sofind/products/masks/lensing_masks_20250826.yaml b/sofind/products/masks/lensing_masks_20250826.yaml index 465088d..b8c0ab7 100644 --- a/sofind/products/masks/lensing_masks_20250826.yaml +++ b/sofind/products/masks/lensing_masks_20250826.yaml @@ -1,9 +1,9 @@ system_paths: - perlmutter: - niagara: - dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/masks/dr6v4_lensing_250826_masks/ - penn-gpc: + perlmutter: /global/cfs/cdirs/act/data/iabril/masks/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/masks_250826/ + penn-gpc: /data6/DR6_plus/masks/masks_250826/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/masks/dr6v4_lensing_250826_masks/ + dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/masks/dr6v4_lensing_250826_masks/ allowed_qids_configs: 'all' diff --git a/sofind/products/transfer_func/tf_dr6v4_240416_lmax10000.yaml b/sofind/products/transfer_func/tf_dr6v4_240416_lmax10000.yaml index 12caf13..2b494e2 100644 --- a/sofind/products/transfer_func/tf_dr6v4_240416_lmax10000.yaml +++ b/sofind/products/transfer_func/tf_dr6v4_240416_lmax10000.yaml @@ -1,8 +1,9 @@ system_paths: perlmutter: /global/cfs/cdirs/act/data/alaposta/transfer_functions/dr6_v4_legacy_240416/ + trillium: /scratch/jaejoonk/DR6plus_lensing_data/transfer_functions/dr6_v4_legacy_240416/ + penn-gpc: /data5/act/DR6_products/transfer_functions/dr6_v4_legacy_240416/ dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/DR6_products/transfer_functions/dr6_v4_legacy_240416/ niagara: /scratch/r/rbond/jaejoonk/lensing_pipeline_data/transfer_functions/dr6_v4_legacy_240416/ - penn-gpc: /data5/act/DR6_products/transfer_functions/dr6_v4_legacy_240416/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/transfer_func/dr6_v4_legacy_240416/ allowed_qids_configs: From 02f1646a5462faa72ecf8ec0e042dd4dd7837e15 Mon Sep 17 00:00:00 2001 From: iabrilcabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Thu, 23 Apr 2026 22:01:04 +0100 Subject: [PATCH 53/56] new mnms --- sofind/products/noise_models/act_dr6v4_day.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sofind/products/noise_models/act_dr6v4_day.yaml b/sofind/products/noise_models/act_dr6v4_day.yaml index 37b5666..ba77635 100644 --- a/sofind/products/noise_models/act_dr6v4_day.yaml +++ b/sofind/products/noise_models/act_dr6v4_day.yaml @@ -78,8 +78,8 @@ tile_cmbmask_daydeep_250513: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 -# inpaint_radius: 6 -# inpaint_thumb_width: 120 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' @@ -122,8 +122,8 @@ tile_cmbmask_daywide_250513: delta_ell_smooth: 400 height_deg: 4.0 width_deg: 4.0 -# inpaint_radius: 6 -# inpaint_thumb_width: 120 + inpaint_radius: 6 + inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' sim_file_template: '{config_name}_{noise_model_name}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' From 72c95d4caa217779b1ee80e16d0e0c1096548a75 Mon Sep 17 00:00:00 2001 From: Joshua Kim <40779225+jaejoonk@users.noreply.github.com> Date: Thu, 7 May 2026 15:57:49 -0400 Subject: [PATCH 54/56] Trillium paths to act_dr6v4 (custom) noise models --- sofind/products/noise_models/act_dr6v4.yaml | 2 +- sofind/products/noise_models/act_dr6v4_el_split.yaml | 1 + sofind/products/noise_models/act_dr6v4_inout_split.yaml | 3 ++- sofind/products/noise_models/act_dr6v4_pwv_split.yaml | 3 ++- sofind/products/noise_models/act_dr6v4_t_split.yaml | 3 ++- 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sofind/products/noise_models/act_dr6v4.yaml b/sofind/products/noise_models/act_dr6v4.yaml index b197671..8740fa7 100644 --- a/sofind/products/noise_models/act_dr6v4.yaml +++ b/sofind/products/noise_models/act_dr6v4.yaml @@ -6,7 +6,7 @@ system_paths: #perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise dirac: /rds/project/dirac_vol5/rds-dirac-dp002/AdvACT/simulations/mnms/ # not saving noise sims to disk anymore on trillium - trillium: + trillium: /project/rpp-ahincks/jaejoonk/mnms/ niagara: /scratch/r/rbond/jaejoonk/dr6plus_noise_sims/ penn-gpc: /data6/DR6_plus/noise_sims/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/simulations/mnms/ diff --git a/sofind/products/noise_models/act_dr6v4_el_split.yaml b/sofind/products/noise_models/act_dr6v4_el_split.yaml index b2847a0..0fba283 100644 --- a/sofind/products/noise_models/act_dr6v4_el_split.yaml +++ b/sofind/products/noise_models/act_dr6v4_el_split.yaml @@ -1,6 +1,7 @@ system_paths: della: /scratch/gpfs/ACT/dr6v4/simulations/noise perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise + trillium: /project/rpp-ahincks/jaejoonk/mnms/ allowed_qids_configs: - act_dr6vX_qids.yaml diff --git a/sofind/products/noise_models/act_dr6v4_inout_split.yaml b/sofind/products/noise_models/act_dr6v4_inout_split.yaml index a27660e..791459c 100644 --- a/sofind/products/noise_models/act_dr6v4_inout_split.yaml +++ b/sofind/products/noise_models/act_dr6v4_inout_split.yaml @@ -1,6 +1,7 @@ system_paths: della: /scratch/gpfs/ACT/dr6v4/simulations/noise perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise + trillium: /project/rpp-ahincks/jaejoonk/mnms/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/simulations/mnms/ allowed_qids_configs: @@ -291,4 +292,4 @@ fdw_cmbmask_ivfwhm2: inpaint_radius: 6 inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_null_{inout_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' - sim_file_template: '{config_name}_{noise_model_name}_null_{inout_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' \ No newline at end of file + sim_file_template: '{config_name}_{noise_model_name}_null_{inout_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' diff --git a/sofind/products/noise_models/act_dr6v4_pwv_split.yaml b/sofind/products/noise_models/act_dr6v4_pwv_split.yaml index f7fd5d4..de5b527 100644 --- a/sofind/products/noise_models/act_dr6v4_pwv_split.yaml +++ b/sofind/products/noise_models/act_dr6v4_pwv_split.yaml @@ -1,6 +1,7 @@ system_paths: della: /scratch/gpfs/ACT/dr6v4/simulations/noise perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise + trillium: /project/rpp-ahincks/jaejoonk/mnms/ souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/simulations/mnms/ allowed_qids_configs: @@ -291,4 +292,4 @@ fdw_cmbmask_ivfwhm2: inpaint_radius: 6 inpaint_thumb_width: 120 model_file_template: '{config_name}_{noise_model_name}_null_{pwv_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' - sim_file_template: '{config_name}_{noise_model_name}_null_{pwv_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' \ No newline at end of file + sim_file_template: '{config_name}_{noise_model_name}_null_{pwv_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' diff --git a/sofind/products/noise_models/act_dr6v4_t_split.yaml b/sofind/products/noise_models/act_dr6v4_t_split.yaml index 3534f0a..d32744b 100644 --- a/sofind/products/noise_models/act_dr6v4_t_split.yaml +++ b/sofind/products/noise_models/act_dr6v4_t_split.yaml @@ -1,6 +1,7 @@ system_paths: della: /scratch/gpfs/ACT/dr6v4/simulations/noise perlmutter: /global/cfs/cdirs/act/data/act_dr6v4/simulations/noise + trillium: /project/rpp-ahincks/jaejoonk/mnms/ allowed_qids_configs: - act_dr6vX_qids.yaml @@ -278,4 +279,4 @@ fdw_cmbmask_ivfwhm2: w_lmax_j: 5300 w_lmin: 10 model_file_template: '{config_name}_{noise_model_name}_null_{t_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_model' - sim_file_template: '{config_name}_{noise_model_name}_null_{t_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' \ No newline at end of file + sim_file_template: '{config_name}_{noise_model_name}_null_{t_split}_{qid_names}_lmax{lmax}_{num_splits}way_set{split_num}_noise_sim_{alm_str}{sim_num:04}' From c62dddcc13556556660419bcccca9f709e6bc8ee Mon Sep 17 00:00:00 2001 From: Irene Abril-Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Wed, 20 May 2026 03:01:47 -0400 Subject: [PATCH 55/56] day cals --- sofind/datamodels/act_dr6v4.yaml | 5 ++++- .../dr6v4_calday_tonight_daydeep_260513.yaml | 22 +++++++++++++++++++ ...dr6v4_calday_tonight_daywide_N_260513.yaml | 20 +++++++++++++++++ ...dr6v4_calday_tonight_daywide_S_260513.yaml | 20 +++++++++++++++++ 4 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 sofind/products/calibrations/dr6v4_calday_tonight_daydeep_260513.yaml create mode 100644 sofind/products/calibrations/dr6v4_calday_tonight_daywide_N_260513.yaml create mode 100644 sofind/products/calibrations/dr6v4_calday_tonight_daywide_S_260513.yaml diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index 5fa0e66..4165d83 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -59,7 +59,10 @@ calibrations: dr6v4_calday_tonight_daydeep_260410: dr6v4_calday_tonight_daydeep_260410.yaml act_dr6.02_hack: act_dr6.02_hack.yaml act_dr6.02_wmflike: act_dr6.02_wmflike.yaml - + dr6v4_calday_tonight_daywide_N_260513: dr6v4_calday_tonight_daywide_N_260513.yaml + dr6v4_calday_tonight_daywide_S_260513: dr6v4_calday_tonight_daywide_S_260513.yaml + dr6v4_calday_tonight_daydeep_260513: dr6v4_calday_tonight_daydeep_260513.yaml + beams: beams_v4_20230130 : beams_v4_20230130.yaml beams_v4_20230902 : beams_v4_20230902.yaml diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daydeep_260513.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daydeep_260513.yaml new file mode 100644 index 0000000..95b7d24 --- /dev/null +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daydeep_260513.yaml @@ -0,0 +1,22 @@ +system_paths: + dirac: + della: + perlmutter: + trillium: /scratch/iabril/calibration/cal_day_to_nightdefault/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/cal_day_to_nightdefault/ + +allowed_qids_configs: +- act_dr6vX_qids.yaml + +allowed_qids: +- pa5a_dd +- pa5b_dd +- pa6a_dd +- pa6b_dd + +allowed_qids_extra_kwargs: + +subproduct_kwargs_orders: +cals_file_template: cal_day_to_nightdefault_daydeep_260513.pkl +poleffs_file_template: +key_template: dr6_{array}_{freq} \ No newline at end of file diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daywide_N_260513.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daywide_N_260513.yaml new file mode 100644 index 0000000..2d67c7e --- /dev/null +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daywide_N_260513.yaml @@ -0,0 +1,20 @@ +system_paths: + dirac: + della: + perlmutter: + trillium: /scratch/iabril/calibration/cal_day_to_nightdefault/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/cal_day_to_nightdefault/ + +allowed_qids_configs: +- act_dr6vX_qids.yaml + +allowed_qids: +- pa5a_dw +- pa5b_dw + +allowed_qids_extra_kwargs: + +subproduct_kwargs_orders: +cals_file_template: cal_day_to_nightdefault_daywide_N_260513.pkl +poleffs_file_template: +key_template: dr6_{array}_{freq} \ No newline at end of file diff --git a/sofind/products/calibrations/dr6v4_calday_tonight_daywide_S_260513.yaml b/sofind/products/calibrations/dr6v4_calday_tonight_daywide_S_260513.yaml new file mode 100644 index 0000000..d6c888a --- /dev/null +++ b/sofind/products/calibrations/dr6v4_calday_tonight_daywide_S_260513.yaml @@ -0,0 +1,20 @@ +system_paths: + dirac: + della: + perlmutter: + trillium: /scratch/iabril/calibration/cal_day_to_nightdefault/ + souk: /cephfs/soukdata/shared_projects/SO_lensing/AdvACT/calibrations/cal_day_to_nightdefault/ + +allowed_qids_configs: +- act_dr6vX_qids.yaml + +allowed_qids: +- pa5a_dw +- pa5b_dw + +allowed_qids_extra_kwargs: + +subproduct_kwargs_orders: +cals_file_template: cal_day_to_nightdefault_daywide_S_260513.pkl +poleffs_file_template: +key_template: dr6_{array}_{freq} \ No newline at end of file From 1854b985b7dc6a5b21c746fb01c43aa5c08f31a1 Mon Sep 17 00:00:00 2001 From: Irene Abril-Cabezas <79768254+iabrilcabezas@users.noreply.github.com> Date: Wed, 17 Jun 2026 12:49:43 -0400 Subject: [PATCH 56/56] more masks woop woop --- sofind/datamodels/act_dr6v4.yaml | 2 ++ sofind/datamodels/planck_npipe.yaml | 2 ++ sofind/products/masks/lensing_masks_20260530.yaml | 12 ++++++++++++ sofind/products/masks/lensing_masks_20260608.yaml | 12 ++++++++++++ 4 files changed, 28 insertions(+) create mode 100644 sofind/products/masks/lensing_masks_20260530.yaml create mode 100644 sofind/products/masks/lensing_masks_20260608.yaml diff --git a/sofind/datamodels/act_dr6v4.yaml b/sofind/datamodels/act_dr6v4.yaml index 4165d83..63d5c61 100644 --- a/sofind/datamodels/act_dr6v4.yaml +++ b/sofind/datamodels/act_dr6v4.yaml @@ -28,6 +28,8 @@ masks: lensing_masks_20250513: lensing_masks_20250513.yaml lensing_masks_20250826: lensing_masks_20250826.yaml lensing_masks_20250915: lensing_masks_20250915.yaml + lensing_masks_20260530: lensing_masks_20260530.yaml + lensing_masks_20260608: lensing_masks_20260608.yaml catalogs: inpaint_catalogs_config: inpaint_catalogs.yaml diff --git a/sofind/datamodels/planck_npipe.yaml b/sofind/datamodels/planck_npipe.yaml index 6430448..b7343b5 100644 --- a/sofind/datamodels/planck_npipe.yaml +++ b/sofind/datamodels/planck_npipe.yaml @@ -23,3 +23,5 @@ masks: lensing_masks_20250513: lensing_masks_20250513.yaml lensing_masks_20240919b: lensing_masks_20240919b.yaml lensing_masks_20250826: lensing_masks_20250826.yaml + lensing_masks_20260530: lensing_masks_20260530.yaml + lensing_masks_20260608: lensing_masks_20260608.yaml \ No newline at end of file diff --git a/sofind/products/masks/lensing_masks_20260530.yaml b/sofind/products/masks/lensing_masks_20260530.yaml new file mode 100644 index 0000000..7f92faa --- /dev/null +++ b/sofind/products/masks/lensing_masks_20260530.yaml @@ -0,0 +1,12 @@ +system_paths: + perlmutter: + trillium: /project/rrg-rbond-ac/msyriac/masks/dr6v4_lensing_260530_masks/ + niagara: + dirac: + penn-gpc: + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: diff --git a/sofind/products/masks/lensing_masks_20260608.yaml b/sofind/products/masks/lensing_masks_20260608.yaml new file mode 100644 index 0000000..c0e3712 --- /dev/null +++ b/sofind/products/masks/lensing_masks_20260608.yaml @@ -0,0 +1,12 @@ +system_paths: + perlmutter: + trillium: /project/rrg-rbond-ac/msyriac/masks/dr6v4_lensing_260608_masks/ + niagara: + dirac: + penn-gpc: + +allowed_qids_configs: 'all' + +allowed_qids: 'all' + +allowed_qids_extra_kwargs: