From 90ae85e8463758f664ff2faba74e9ea87186753a Mon Sep 17 00:00:00 2001 From: Erin Howard Date: Thu, 17 Sep 2026 10:00:13 -0700 Subject: [PATCH 1/2] Update template pipelines for cell based coadds. --- bps/bps_PromptTemplate.yaml | 9 ++++++--- bps/bps_QuickTemplate.yaml | 6 +++++- bps/clustering/clustering_PromptTemplate.yaml | 12 ++++++------ bps/clustering/clustering_QuickTemplate.yaml | 12 ++++++------ pipelines/LSSTCam/PromptTemplate.yaml | 12 ++++++------ pipelines/LSSTCam/QuickTemplate.yaml | 5 +++++ pipelines/_ingredients/QuickTemplate.yaml | 13 +++++++++++++ 7 files changed, 47 insertions(+), 22 deletions(-) diff --git a/bps/bps_PromptTemplate.yaml b/bps/bps_PromptTemplate.yaml index 4dbfa744..29f91c05 100644 --- a/bps/bps_PromptTemplate.yaml +++ b/bps/bps_PromptTemplate.yaml @@ -18,8 +18,11 @@ site: profile: condor: +Walltime: 43200 + +# Include BPS clustering to group compatible pipeline tasks for more efficient batch +# processing. Use QuickTemplate's clustering file. includeConfigs: - - ${AP_PIPE_DIR}/bps/clustering/clustering_PromptTemplate.yaml + - ${AP_PIPE_DIR}/bps/clustering/clustering_QuickTemplate.yaml # Memory allocated for each quantum, in MBs; can be overridden on a per-task basis. # The bps default is 2048 MB, the same as this example, but it's kept here as a reminder. @@ -37,12 +40,12 @@ payload: # The operator variable is used at facilities beyond bps and it defaults to the value of USER. output : 'u/{operator}/{payloadName}' # UPDATE THIS to specify what data to process - dataQuery: "instrument='LSSTCam' AND skymap='lsst_cells_v1 AND band='g'" + dataQuery: "instrument='LSSTCam' AND skymap='lsst_cells_v2 AND band='g'" # Various things for bps to customize about each pipeline task pipetask: # Option to list other pipeline tasks being run here with default overrides - makeWarp: + makeDirectWarp: requestMemory: 8192 # Set the appropriate wms service class for the batch system diff --git a/bps/bps_QuickTemplate.yaml b/bps/bps_QuickTemplate.yaml index 31019c1b..e5fc6a65 100644 --- a/bps/bps_QuickTemplate.yaml +++ b/bps/bps_QuickTemplate.yaml @@ -18,8 +18,12 @@ site: profile: condor: +Walltime: 43200 + +# Include BPS clustering to group compatible pipeline tasks for more efficient batch +# processing. includeConfigs: - ${AP_PIPE_DIR}/bps/clustering/clustering_QuickTemplate.yaml + # if you are running on data that needs skyCorr, such as HSC, comment out the previous line # and use this one instead: # - ${AP_PIPE_DIR}/bps/clustering/clustering_HSC_QuickTemplate.yaml @@ -45,7 +49,7 @@ payload: # Various things for bps to customize about each pipeline task pipetask: # Option to list other pipeline tasks being run here with default overrides - makeWarp: + makeDirectWarp: requestMemory: 8192 # Set the appropriate wms service class for the batch system diff --git a/bps/clustering/clustering_PromptTemplate.yaml b/bps/clustering/clustering_PromptTemplate.yaml index d57e651b..0fcabaaf 100644 --- a/bps/clustering/clustering_PromptTemplate.yaml +++ b/bps/clustering/clustering_PromptTemplate.yaml @@ -18,14 +18,14 @@ cluster: pipetasks: makeDirectWarp,makePsfMatchedWarp dimensions: patch,visit selectAndAssemble: - pipetasks: selectTemplateCoaddVisits,assembleTemplateCoadd + pipetasks: selectTemplateCoaddVisits,assembleTemplateCoadd,assembleCellCoadd dimensions: band,tract,patch - make_analysis_images: - pipetasks: makeBinnedTemplateCoaddNImage,makeBinnedTemplateCoaddImage - dimensions: tract,band - make_tract_analysis_images: - pipetasks: makeWholeTractTemplateCoaddNImage,makeWholeTractTemplateCoaddImage + tract_band_analysis: + pipetasks: makeTemplateCoaddInputSummaryTract,aggregateTemplateCoaddMaskFractions,makeBinnedTemplateCoaddNImage,makeBinnedTemplateCoaddImage,makeWholeTractTemplateCoaddNImage,makeWholeTractTemplateCoaddImage dimensions: tract,band + mask_fraction_global: + pipetasks: makeAggregatedTemplateCoaddMaskFractionsTable,aggregatedTemplateCoaddMaskFractionsWholeSkyPlot + dimensions: band make_metrics: pipetasks: analyzeCoaddDepthCore,coaddDepthMetricTract dimensions: tract diff --git a/bps/clustering/clustering_QuickTemplate.yaml b/bps/clustering/clustering_QuickTemplate.yaml index 194ae850..74ef8350 100644 --- a/bps/clustering/clustering_QuickTemplate.yaml +++ b/bps/clustering/clustering_QuickTemplate.yaml @@ -22,14 +22,14 @@ cluster: pipetasks: makeDirectWarp,makePsfMatchedWarp dimensions: patch,visit selectAndAssemble: - pipetasks: selectTemplateCoaddVisits,assembleTemplateCoadd + pipetasks: selectTemplateCoaddVisits,assembleTemplateCoadd,assembleCellCoadd dimensions: band,tract,patch - make_analysis_images: - pipetasks: makeBinnedTemplateCoaddNImage,makeBinnedTemplateCoaddImage - dimensions: tract,band - make_tract_analysis_images: - pipetasks: makeWholeTractTemplateCoaddNImage,makeWholeTractTemplateCoaddImage + tract_band_analysis: + pipetasks: makeTemplateCoaddInputSummaryTract,aggregateTemplateCoaddMaskFractions,makeBinnedTemplateCoaddNImage,makeBinnedTemplateCoaddImage,makeWholeTractTemplateCoaddNImage,makeWholeTractTemplateCoaddImage dimensions: tract,band + mask_fraction_global: + pipetasks: makeAggregatedTemplateCoaddMaskFractionsTable,aggregatedTemplateCoaddMaskFractionsWholeSkyPlot + dimensions: band make_metrics: pipetasks: analyzeCoaddDepthCore,coaddDepthMetricTract dimensions: tract diff --git a/pipelines/LSSTCam/PromptTemplate.yaml b/pipelines/LSSTCam/PromptTemplate.yaml index 439d519e..badea985 100644 --- a/pipelines/LSSTCam/PromptTemplate.yaml +++ b/pipelines/LSSTCam/PromptTemplate.yaml @@ -1,5 +1,5 @@ description: The LSSTCam AP template building pipeline with preliminary_visit_images as inputs. -# Important Note: Use corresponding BPS clustering file when running with BPS. +# Important Note: Use QuickTemplate BPS clustering file when running with BPS. instrument: lsst.obs.lsst.LsstCam @@ -8,13 +8,13 @@ imports: exclude: - isr - calibrateImage - - selectTemplateCoaddVisits + labeledSubsetModifyMode: EDIT + +parameters: + image_type: future + use_cell_coadds: true tasks: - consolidateVisitSummary: - class: lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask - config: - full: True selectTemplateCoaddVisits: # Class changed from BestSeeingQuantileSelectVisitsTask due to different template # expectations for incremental templates. diff --git a/pipelines/LSSTCam/QuickTemplate.yaml b/pipelines/LSSTCam/QuickTemplate.yaml index f32b3468..12e0298f 100644 --- a/pipelines/LSSTCam/QuickTemplate.yaml +++ b/pipelines/LSSTCam/QuickTemplate.yaml @@ -1,6 +1,11 @@ description: The LSSTCam AP template building pipeline +# Important Note: Use QuickTemplate BPS clustering file when running with BPS. instrument: lsst.obs.lsst.LsstCam imports: - location: $AP_PIPE_DIR/pipelines/_ingredients/QuickTemplate.yaml + +parameters: + image_type: future + use_cell_coadds: true diff --git a/pipelines/_ingredients/QuickTemplate.yaml b/pipelines/_ingredients/QuickTemplate.yaml index 768e7c36..1824bbf5 100644 --- a/pipelines/_ingredients/QuickTemplate.yaml +++ b/pipelines/_ingredients/QuickTemplate.yaml @@ -10,6 +10,7 @@ imports: - makeDirectWarp - makePsfMatchedWarp - assembleTemplateCoadd + - assembleCellCoadd - makeTemplateCoaddInputSummaryTract - makeTemplateCoaddInputSummary - makeBinnedTemplateCoaddImage @@ -53,6 +54,17 @@ tasks: maxFractionEpochsHigh: 0.2 badMaskPlanes: ["NO_DATA", "BAD", "SAT", "EDGE", "SPIKE"] prefilterArtifactsMaskPlanes: ["NO_DATA", "BAD", "SUSPECT", "SPIKE"] + doWriteArtifactMasks: true + python: | + config.doInputMap = not parameters.use_cell_coadds + assembleCellCoadd: + class: lsst.drp.tasks.assemble_cell_coadd.AssembleCellCoaddTask + config: + connections.inputWarps: direct_warp + connections.maskedFractionWarps: direct_warp_masked_fraction + connections.visitSummaryList: preliminary_visit_summary + connections.multipleCellCoadd: template_coadd_cell + connections.inputMap: template_coadd_input_map subsets: singleFrame: @@ -68,6 +80,7 @@ subsets: - makeDirectWarp - makePsfMatchedWarp - assembleTemplateCoadd + - assembleCellCoadd description: > Tasks to run to build good seeing coadds for use as templates. makeMetrics: From f85b67ba9a4a35912b30606a5063626265ac2884 Mon Sep 17 00:00:00 2001 From: Erin Howard Date: Tue, 29 Sep 2026 10:04:11 -0700 Subject: [PATCH 2/2] Remove unnecessary PromptTemplate clustering file. --- bps/clustering/clustering_PromptTemplate.yaml | 31 ------------------- 1 file changed, 31 deletions(-) delete mode 100644 bps/clustering/clustering_PromptTemplate.yaml diff --git a/bps/clustering/clustering_PromptTemplate.yaml b/bps/clustering/clustering_PromptTemplate.yaml deleted file mode 100644 index 0fcabaaf..00000000 --- a/bps/clustering/clustering_PromptTemplate.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# This is a prescription for quantum clustering with BPS, suitable for any -# concrete pipeline based on the AP template pipeline that does not run SkyCorr. -# -# Use it by adding: -# -# includeConfigs: -# - ${AP_PIPE_DIR}/bps/clustering/clustering_PromptTemplate.yaml -# -# (with no outer indentation) to your BPS config file. -# - -clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering -cluster: - consolidate: - pipetasks: consolidateVisitSummary - dimensions: visit - makeWarp: - pipetasks: makeDirectWarp,makePsfMatchedWarp - dimensions: patch,visit - selectAndAssemble: - pipetasks: selectTemplateCoaddVisits,assembleTemplateCoadd,assembleCellCoadd - dimensions: band,tract,patch - tract_band_analysis: - pipetasks: makeTemplateCoaddInputSummaryTract,aggregateTemplateCoaddMaskFractions,makeBinnedTemplateCoaddNImage,makeBinnedTemplateCoaddImage,makeWholeTractTemplateCoaddNImage,makeWholeTractTemplateCoaddImage - dimensions: tract,band - mask_fraction_global: - pipetasks: makeAggregatedTemplateCoaddMaskFractionsTable,aggregatedTemplateCoaddMaskFractionsWholeSkyPlot - dimensions: band - make_metrics: - pipetasks: analyzeCoaddDepthCore,coaddDepthMetricTract - dimensions: tract